diff options
Diffstat (limited to 'doc')
643 files changed, 27685 insertions, 9502 deletions
diff --git a/doc/.vale/gitlab/Acronyms.yml b/doc/.vale/gitlab/Acronyms.yml index 5176a18e2b6..d166e71491c 100644 --- a/doc/.vale/gitlab/Acronyms.yml +++ b/doc/.vale/gitlab/Acronyms.yml @@ -3,7 +3,7 @@ # # Checks for unexpanded acronyms. # -# For a list of all options, see https://errata-ai.github.io/vale/styles/ +# For a list of all options, see https://errata-ai.gitbook.io/vale/getting-started/styles extends: conditional message: '"%s" has no definition.' link: https://about.gitlab.com/handbook/marketing/growth-marketing/content/editorial-team/#acronyms @@ -18,12 +18,15 @@ exceptions: - ARN - ASCII - AWS + - CLI - CNAME - CPU + - CORE - CSS - CSV - DNS - EKS + - FAQ - GDK - GET - GNU @@ -33,21 +36,28 @@ exceptions: - HTTP - HTTPS - IAM + - IBM - IDE + - IRC + - ISO - JSON - LDAP - LDAPS - LESS - LFS + - LRU - NFS - NGINX - NOTE + - NPM - ONLY + - PDF - PGP - PHP - POST - PUT - RPC + - RAM - RSA - RSS - SAML @@ -58,13 +68,17 @@ exceptions: - SSH - SSL - SSO + - SVN + - TCP - TIP - TLS - TODO - TOML - UNIX + - USB - URI - URL + - UUID - VPC - WIP - XML diff --git a/doc/.vale/gitlab/AlertBoxStyle.yml b/doc/.vale/gitlab/AlertBoxStyle.yml new file mode 100644 index 00000000000..831d5395fc8 --- /dev/null +++ b/doc/.vale/gitlab/AlertBoxStyle.yml @@ -0,0 +1,16 @@ +--- +# Error: gitlab.AlertBoxStyle +# +# Makes sure alert boxes follow standard formatting. +# +# For a list of all options, see https://errata-ai.gitbook.io/vale/getting-started/styles +extends: existence +message: 'Alert box "%s" must use the formatting detailed in the documentation style guide.' +link: https://docs.gitlab.com/ee/development/documentation/styleguide.html#alert-boxes +level: error +scope: raw +raw: + - '((NOTE|TIP|CAUTION|DANGER): \*\*[^:]*\*\*)|' + - '((NOTE: \*\*NOTE:\*\*)|(TIP: \*\*TIP:\*\*)|(CAUTION: \*\*CAUTION:\*\*)|(DANGER: \*\*DANGER:\*\*))|' + - '((NOTE: \*\*note:\*\*)|(TIP: \*\*tip:\*\*)|(CAUTION: \*\*caution:\*\*)|(DANGER: \*\*danger:\*\*))|' + - '((NOTE|TIP|CAUTION|DANGER): \*\*.*\*\*.+)' diff --git a/doc/.vale/gitlab/BadgeCapitalization.yml b/doc/.vale/gitlab/BadgeCapitalization.yml index c9e9da3b6ce..caf7143e254 100644 --- a/doc/.vale/gitlab/BadgeCapitalization.yml +++ b/doc/.vale/gitlab/BadgeCapitalization.yml @@ -3,7 +3,7 @@ # # Verifies that badges are not mixed case, which won't render properly. # -# For a list of all options, see https://errata-ai.github.io/vale/styles/ +# For a list of all options, see https://errata-ai.gitbook.io/vale/getting-started/styles extends: existence message: 'Badge "%s" must be capitalized.' link: https://docs.gitlab.com/ee/development/documentation/styleguide.html#product-badges diff --git a/doc/.vale/gitlab/British.yml b/doc/.vale/gitlab/British.yml index 1e5841d3648..3a0cb321f93 100644 --- a/doc/.vale/gitlab/British.yml +++ b/doc/.vale/gitlab/British.yml @@ -3,7 +3,7 @@ # # Checks that US spelling is used over British spelling. # -# For a list of all options, see https://errata-ai.github.io/vale/styles/ +# For a list of all options, see https://errata-ai.gitbook.io/vale/getting-started/styles extends: substitution message: 'Use the US spelling "%s" instead of the British "%s".' link: https://about.gitlab.com/handbook/communication/#top-misused-terms diff --git a/doc/.vale/gitlab/CodeblockFences.yml b/doc/.vale/gitlab/CodeblockFences.yml index 8b61a1a3c16..7258a8ef475 100644 --- a/doc/.vale/gitlab/CodeblockFences.yml +++ b/doc/.vale/gitlab/CodeblockFences.yml @@ -3,7 +3,7 @@ # # Ensures all codeblock language tags use the full name, not aliases. # -# For a list of all options, see https://errata-ai.github.io/vale/styles/ +# For a list of all options, see https://errata-ai.gitbook.io/vale/getting-started/styles extends: existence message: 'Syntax highlighting hint "%s" must be one of: yaml, ruby, plaintext, markdown, javascript, shell, golang, python, dockerfile, or typescript.' link: https://docs.gitlab.com/ee/development/documentation/styleguide.html#code-blocks diff --git a/doc/.vale/gitlab/Contractions.yml b/doc/.vale/gitlab/Contractions.yml index 45212945c53..dc48b876f40 100644 --- a/doc/.vale/gitlab/Contractions.yml +++ b/doc/.vale/gitlab/Contractions.yml @@ -3,7 +3,7 @@ # # Checks for use of common and uncommon contractions. # -# For a list of all options, see https://errata-ai.github.io/vale/styles/ +# For a list of all options, see https://errata-ai.gitbook.io/vale/getting-started/styles extends: substitution message: 'Use "%s" instead of "%s", for a friendly, informal tone.' link: https://docs.gitlab.com/ee/development/documentation/styleguide.html#language @@ -20,7 +20,6 @@ swap: have not: haven't that is: that's we are: we're - will not: won't would not: wouldn't you are: you're you have: you've @@ -31,25 +30,16 @@ swap: didn't: did not doesn't: does not hasn't: has not - how'll: how will how's: how is isn't: is not - it'll: it will shouldn't: should not - that'll: that will - they'll: they will they're: they are wasn't: was not weren't: were not - we'll: we will we've: we have what's: what is - what'll: what will when's: when is - when'll: when will where's: where is - where'll: where will who's: who is - who'll: who will why's: why is - why'll: why will + diff --git a/doc/.vale/gitlab/CurlStringsQuoted.yml b/doc/.vale/gitlab/CurlStringsQuoted.yml index 4fcb0069423..39ee9372947 100644 --- a/doc/.vale/gitlab/CurlStringsQuoted.yml +++ b/doc/.vale/gitlab/CurlStringsQuoted.yml @@ -3,7 +3,7 @@ # # Ensures all codeblocks using curl quote any URL strings. # -# For a list of all options, see https://errata-ai.github.io/vale/styles/ +# For a list of all options, see https://errata-ai.gitbook.io/vale/getting-started/styles extends: existence message: 'Curl commands must wrap URLs in double quotes ("): %s' link: https://docs.gitlab.com/ee/development/documentation/styleguide.html#curl-commands diff --git a/doc/.vale/gitlab/CurrentStatus.yml b/doc/.vale/gitlab/CurrentStatus.yml new file mode 100644 index 00000000000..7368310eee8 --- /dev/null +++ b/doc/.vale/gitlab/CurrentStatus.yml @@ -0,0 +1,13 @@ +--- +# Suggestion: gitlab.CurrentStatus +# +# Checks for words that indicate a product or feature may change in the future. +# +# For a list of all options, see https://errata-ai.gitbook.io/vale/getting-started/styles +extends: existence +message: 'Avoid words like "%s" that promise future changes.' +level: suggestion +ignorecase: true +link: https://docs.gitlab.com/ee/development/documentation/styleguide.html#language-to-avoid +tokens: + - currently diff --git a/doc/.vale/gitlab/FirstPerson.yml b/doc/.vale/gitlab/FirstPerson.yml index 6db89dd4758..d247f137501 100644 --- a/doc/.vale/gitlab/FirstPerson.yml +++ b/doc/.vale/gitlab/FirstPerson.yml @@ -3,7 +3,7 @@ # # Checks for use of first person pronouns. # -# For a list of all options, see https://errata-ai.github.io/vale/styles/ +# For a list of all options, see https://errata-ai.gitbook.io/vale/getting-started/styles extends: existence message: '"%s" is a first-person pronoun. Use second- or third-person pronouns (like we, you, us, one) instead.' level: warning diff --git a/doc/.vale/gitlab/FutureTense.yml b/doc/.vale/gitlab/FutureTense.yml new file mode 100644 index 00000000000..a53a7dd29cc --- /dev/null +++ b/doc/.vale/gitlab/FutureTense.yml @@ -0,0 +1,17 @@ +--- +# Suggestion: gitlab.FutureTense +# +# Checks for use of future tense in sentences. Present tense is preferred as +# much as possible. +# +# For a list of all options, see https://errata-ai.gitbook.io/vale/getting-started/styles +extends: existence +message: 'Avoid using future tense: "%s"' +ignorecase: true +level: warning +link: https://docs.gitlab.com/ee/development/documentation/styleguide.html#language-to-avoid +raw: + - "(going to( |\n|[[:punct:]])[a-zA-Z]*|" + - "will( |\n|[[:punct:]])[a-zA-Z]*|" + - "won't( |\n|[[:punct:]])[a-zA-Z]*|" + - "[a-zA-Z]*'ll( |\n|[[:punct:]])[a-zA-Z]*)" diff --git a/doc/.vale/gitlab/InternalLinkExtension.yml b/doc/.vale/gitlab/InternalLinkExtension.yml index 94a935196a7..61a08e4a86c 100644 --- a/doc/.vale/gitlab/InternalLinkExtension.yml +++ b/doc/.vale/gitlab/InternalLinkExtension.yml @@ -3,7 +3,7 @@ # # Checks that internal links have .md extenstion and not .html extension. # -# For a list of all options, see https://errata-ai.github.io/vale/styles/ +# For a list of all options, see https://errata-ai.gitbook.io/vale/getting-started/styles extends: existence message: 'Link "%s" must use the .md file extension.' link: https://docs.gitlab.com/ee/development/documentation/styleguide.html#links-to-internal-documentation diff --git a/doc/.vale/gitlab/LatinTerms.yml b/doc/.vale/gitlab/LatinTerms.yml index a2d024fb1ec..26dba42839a 100644 --- a/doc/.vale/gitlab/LatinTerms.yml +++ b/doc/.vale/gitlab/LatinTerms.yml @@ -3,7 +3,7 @@ # # Checks for use of latin terms. # -# For a list of all options, see https://errata-ai.github.io/vale/styles/ +# For a list of all options, see https://errata-ai.gitbook.io/vale/getting-started/styles extends: substitution message: 'Use "%s" instead of "%s", but consider rewriting the sentence.' link: https://docs.gitlab.com/ee/development/documentation/styleguide.html#language diff --git a/doc/.vale/gitlab/MeaningfulLinkWords.yml b/doc/.vale/gitlab/MeaningfulLinkWords.yml index 1931112ab3e..4a255e5aae4 100644 --- a/doc/.vale/gitlab/MeaningfulLinkWords.yml +++ b/doc/.vale/gitlab/MeaningfulLinkWords.yml @@ -3,7 +3,7 @@ # # Checks for the presence of semantically unhelpful words in link text. # -# For a list of all options, see https://errata-ai.github.io/vale/styles/ +# For a list of all options, see https://errata-ai.gitbook.io/vale/getting-started/styles extends: existence message: 'Improve SEO and accessibility by rewriting "%s" in the link text.' level: warning diff --git a/doc/.vale/gitlab/MergeConflictMarkers.yml b/doc/.vale/gitlab/MergeConflictMarkers.yml index 4d733c856e5..4f216ac34c5 100644 --- a/doc/.vale/gitlab/MergeConflictMarkers.yml +++ b/doc/.vale/gitlab/MergeConflictMarkers.yml @@ -3,7 +3,7 @@ # # Checks for the presence of merge conflict markers. # -# For a list of all options, see https://errata-ai.github.io/vale/styles/ +# For a list of all options, see https://errata-ai.gitbook.io/vale/getting-started/styles extends: existence message: 'Merge conflict marker "%s" found.' link: https://docs.gitlab.com/ee/development/code_review.html#merging-a-merge-request diff --git a/doc/.vale/gitlab/OutdatedVersions.yml b/doc/.vale/gitlab/OutdatedVersions.yml new file mode 100644 index 00000000000..3252481a523 --- /dev/null +++ b/doc/.vale/gitlab/OutdatedVersions.yml @@ -0,0 +1,21 @@ +--- +# Warning: gitlab.OutdatedVersions +# +# Checks for references to versions of GitLab that are no longer supported. +# +# For a list of all options, see https://errata-ai.github.io/vale/styles/ +extends: existence +message: 'Can this reference to "%s" be refactored?' +link: https://docs.gitlab.com/ee/development/documentation/styleguide.html#importance-of-referencing-gitlab-versions-and-tiers +level: warning +nonword: true +ignorecase: true +tokens: + - "GitLab (v)?2." + - "GitLab (v)?3." + - "GitLab (v)?4." + - "GitLab (v)?5." + - "GitLab (v)?6." + - "GitLab (v)?7." + - "GitLab (v)?8." + - "GitLab (v)?9." diff --git a/doc/.vale/gitlab/OxfordComma.yml b/doc/.vale/gitlab/OxfordComma.yml index e04d209d960..334db5d0388 100644 --- a/doc/.vale/gitlab/OxfordComma.yml +++ b/doc/.vale/gitlab/OxfordComma.yml @@ -4,7 +4,7 @@ # Checks for the lack of an Oxford comma. In some cases, will catch overly # complex sentence structures with lots of commas. # -# For a list of all options, see https://errata-ai.github.io/vale/styles/ +# For a list of all options, see https://errata-ai.gitbook.io/vale/getting-started/styles extends: existence message: 'Use a comma before the last "and" or "or" in a list of four or more items.' link: https://docs.gitlab.com/ee/development/documentation/styleguide.html#punctuation diff --git a/doc/.vale/gitlab/ReferenceLinks.yml b/doc/.vale/gitlab/ReferenceLinks.yml index 8a3b6940187..49e6ed5a531 100644 --- a/doc/.vale/gitlab/ReferenceLinks.yml +++ b/doc/.vale/gitlab/ReferenceLinks.yml @@ -3,7 +3,7 @@ # # Checks for the presence of reference-style links that must be inline. # -# For a list of all options, see https://errata-ai.github.io/vale/styles/ +# For a list of all options, see https://errata-ai.gitbook.io/vale/getting-started/styles extends: existence message: 'Link "%s" must be inline.' link: https://docs.gitlab.com/ee/development/documentation/styleguide.html#basic-link-criteria diff --git a/doc/.vale/gitlab/RelativeLinks.yml b/doc/.vale/gitlab/RelativeLinks.yml index de24d0608e7..f7407375b84 100644 --- a/doc/.vale/gitlab/RelativeLinks.yml +++ b/doc/.vale/gitlab/RelativeLinks.yml @@ -3,7 +3,7 @@ # # Checks for the presence of absolute hyperlinks that should be relative. # -# For a list of all options, see https://errata-ai.github.io/vale/styles/ +# For a list of all options, see https://errata-ai.gitbook.io/vale/getting-started/styles extends: existence message: 'Link "%s" must be relative.' link: https://docs.gitlab.com/ee/development/documentation/styleguide.html#links-to-internal-documentation diff --git a/doc/.vale/gitlab/Repetition.yml b/doc/.vale/gitlab/Repetition.yml index 76afb7bb5ab..c4b0cc14192 100644 --- a/doc/.vale/gitlab/Repetition.yml +++ b/doc/.vale/gitlab/Repetition.yml @@ -3,7 +3,7 @@ # # Checks for duplicate words, like `the the` or `and and`. # -# For a list of all options, see https://errata-ai.github.io/vale/styles/ +# For a list of all options, see https://errata-ai.gitbook.io/vale/getting-started/styles extends: repetition message: '"%s" is repeated.' level: error diff --git a/doc/.vale/gitlab/SentenceLength.yml b/doc/.vale/gitlab/SentenceLength.yml index b19b76723a6..da9fa052584 100644 --- a/doc/.vale/gitlab/SentenceLength.yml +++ b/doc/.vale/gitlab/SentenceLength.yml @@ -3,7 +3,7 @@ # # Counts words in a sentence and alerts if a sentence exceeds 25 words. # -# For a list of all options, see https://errata-ai.github.io/vale/styles/ +# For a list of all options, see https://errata-ai.gitbook.io/vale/getting-started/styles extends: occurrence message: 'Shorter sentences improve readability (max 25 words).' scope: sentence diff --git a/doc/.vale/gitlab/SentenceSpacing.yml b/doc/.vale/gitlab/SentenceSpacing.yml index c460ef3ae65..e6da920222c 100644 --- a/doc/.vale/gitlab/SentenceSpacing.yml +++ b/doc/.vale/gitlab/SentenceSpacing.yml @@ -1,12 +1,12 @@ --- # Error: gitlab.SentenceSpacing # -# Check for the following in common content scenarios: +# Checks for the following in common content scenarios: # # - No spaces. # - More than one space. # -# For a list of all options, see https://errata-ai.github.io/vale/styles/ +# For a list of all options, see https://errata-ai.gitbook.io/vale/getting-started/styles extends: existence message: '"%s" must contain one and only one space.' link: https://docs.gitlab.com/ee/development/documentation/styleguide.html#punctuation diff --git a/doc/.vale/gitlab/Spelling.yml b/doc/.vale/gitlab/Spelling.yml index 7bf0f085f5c..602b7cd11e6 100644 --- a/doc/.vale/gitlab/Spelling.yml +++ b/doc/.vale/gitlab/Spelling.yml @@ -9,7 +9,7 @@ # Commands, like `git clone` must use backticks, and must not be added to the # exceptions. # -# For a list of all options, see https://errata-ai.github.io/vale/styles/ +# For a list of all options, see https://errata-ai.gitbook.io/vale/getting-started/styles extends: spelling message: 'Spelling check: "%s"?' level: warning diff --git a/doc/.vale/gitlab/SubstitutionWarning.yml b/doc/.vale/gitlab/SubstitutionWarning.yml index 8c5f7705417..5324a48e38c 100644 --- a/doc/.vale/gitlab/SubstitutionWarning.yml +++ b/doc/.vale/gitlab/SubstitutionWarning.yml @@ -4,7 +4,7 @@ # Warns against using common shorthand for terms. # For substitutions flagged as errors, see Substitutions.yml # -# For a list of all options, see https://errata-ai.github.io/vale/styles/ +# For a list of all options, see https://errata-ai.gitbook.io/vale/getting-started/styles extends: substitution message: 'If possible, use "%s" instead of "%s".' link: https://about.gitlab.com/handbook/communication/#top-misused-terms @@ -13,8 +13,11 @@ ignorecase: true swap: admin: administrator blacklist(ed|ing)?: denylist + code base: codebase config: configuration distro: distribution + file name: filename + filesystem: file system info: information repo: repository whitelist(ed|ing)?: allowlist diff --git a/doc/.vale/gitlab/Substitutions.yml b/doc/.vale/gitlab/Substitutions.yml index 156ff3a53f0..77536ea0b33 100644 --- a/doc/.vale/gitlab/Substitutions.yml +++ b/doc/.vale/gitlab/Substitutions.yml @@ -4,7 +4,7 @@ # Checks for use of some of the top misused terms at GitLab. # For substitutions only flagged as warnings, see SubstitutionWarning.yml # -# For a list of all options, see https://errata-ai.github.io/vale/styles/ +# For a list of all options, see https://errata-ai.gitbook.io/vale/getting-started/styles extends: substitution message: 'Use "%s" instead of "%s".' link: https://about.gitlab.com/handbook/communication/#top-misused-terms diff --git a/doc/.vale/gitlab/VersionText.yml b/doc/.vale/gitlab/VersionText.yml index 9a05103cc39..3723170b169 100644 --- a/doc/.vale/gitlab/VersionText.yml +++ b/doc/.vale/gitlab/VersionText.yml @@ -13,7 +13,7 @@ # immediately on the next line is ok. However, this will often highlight where multi-line version # text is attempted without `-` characters. # -# For a list of all options, see https://errata-ai.github.io/vale/styles/ +# For a list of all options, see https://errata-ai.gitbook.io/vale/getting-started/styles extends: existence message: '"%s" is not formatted correctly.' link: https://docs.gitlab.com/ee/development/documentation/styleguide.html#text-for-documentation-requiring-version-text diff --git a/doc/.vale/gitlab/spelling-exceptions.txt b/doc/.vale/gitlab/spelling-exceptions.txt index b56fa2861ca..28da80557ec 100644 --- a/doc/.vale/gitlab/spelling-exceptions.txt +++ b/doc/.vale/gitlab/spelling-exceptions.txt @@ -83,6 +83,7 @@ compilable composable Conda Consul +Contentful Corosync cron crons @@ -224,6 +225,7 @@ LDAP ldapsearch Leiningen Libravatar +liveness Lograge Logstash lookahead @@ -247,6 +249,7 @@ memoization memoize memoized memoizing +Memorystore mergeable Microsoft middleware @@ -334,8 +337,7 @@ Puma Python Qualys Rackspace -Raketask -Raketasks +Raspbian reachability rebase rebased @@ -423,6 +425,7 @@ storages strace strikethrough strikethroughs +stunnel subpath subfolder subfolders diff --git a/doc/README.md b/doc/README.md index fe38b6cb419..115026dae6e 100644 --- a/doc/README.md +++ b/doc/README.md @@ -11,7 +11,7 @@ description: 'Learn how to use and administer GitLab, the most scalable Git-base # GitLab Docs -Welcome to [GitLab](https://about.gitlab.com/) Documentation. +Welcome to [GitLab](https://about.gitlab.com/) documentation. Here you can access the complete documentation for GitLab, the single application for the [entire DevOps lifecycle](#the-entire-devops-lifecycle). @@ -20,30 +20,32 @@ Here you can access the complete documentation for GitLab, the single applicatio No matter how you use GitLab, we have documentation for you. -| Essential Documentation | Essential Documentation | -|:-------------------------------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------| -| [**User Documentation**](user/index.md)<br/>Discover features and concepts for GitLab users. | [**Administrator documentation**](administration/index.md)<br/>Everything GitLab self-managed administrators need to know. | -| [**Contributing to GitLab**](#contributing-to-gitlab)<br/>At GitLab, everyone can contribute! | [**New to Git and GitLab?**](#new-to-git-and-gitlab)<br/>We have the resources to get you started. | -| [**Building an integration with GitLab?**](#building-an-integration-with-gitlab)<br/>Consult our automation and integration documentation. | [**Coming to GitLab from another platform?**](#coming-to-gitlab-from-another-platform)<br/>Consult our handy guides. | -| [**Install GitLab**](https://about.gitlab.com/install/)<br/>Installation options for different platforms. | [**Customers**](subscriptions/index.md)<br/>Information for new and existing customers. | -| [**Update GitLab**](update/README.md)<br/>Update your GitLab self-managed instance to the latest version. | [**Reference Architectures**](administration/reference_architectures/index.md)<br/>GitLab's reference architectures | -| [**GitLab Releases**](https://about.gitlab.com/releases/)<br/>What's new in GitLab. | | - -## Popular Documentation - -Have a look at some of our most popular documentation resources: - -| Popular Topic | Description | -|:----------------------------------------------------------------|:-----------------------------------------------------------------| -| [Configuring `.gitlab-ci.yml`](ci/yaml/README.md) | Complete syntax documentation for configuring your CI pipelines. | -| [GitLab CI/CD examples](ci/examples/README.md) | Get up to speed quickly with common CI/CD scenarios. | -| [GitLab Container Registry](user/packages/container_registry/index.md) | Host Docker images within GitLab. | -| [GitLab Pages](user/project/pages/index.md) | Host static websites for your projects with GitLab. | -| [GitLab.com settings](user/gitlab_com/index.md) | Settings for GitLab.com. | -| [Kubernetes integration](user/project/clusters/index.md) | Use GitLab with Kubernetes. | -| [SSH authentication](ssh/README.md) | Secure your network communications. | -| [Using Docker images](ci/docker/using_docker_images.md) | Build and test your applications with Docker. | -| [GraphQL](api/graphql/index.md) | Explore GitLab's GraphQL API. | +| Essential documentation | Essential documentation | +|:-------------------------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------| +| [**User Documentation**](user/index.md)<br/>Discover features and concepts for GitLab users. | [**Administrator documentation**](administration/index.md)<br/>Everything GitLab self-managed administrators need to know. | +| [**Contributing to GitLab**](#contributing-to-gitlab)<br/>At GitLab, everyone can contribute! | [**New to Git and GitLab?**](#new-to-git-and-gitlab)<br/>We have the resources to get you started. | +| [**Build an integration with GitLab?**](#build-an-integration-with-gitlab)<br/>Consult our automation and integration documentation. | [**Coming to GitLab from another platform?**](#coming-to-gitlab-from-another-platform)<br/>Consult our handy guides. | +| [**Install GitLab**](https://about.gitlab.com/install/)<br/>Installation options for different platforms. | [**Customers**](subscriptions/index.md)<br/>Information for new and existing customers. | +| [**Update GitLab**](update/README.md)<br/>Update your GitLab self-managed instance to the latest version. | [**Reference Architectures**](administration/reference_architectures/index.md)<br/>GitLab's reference architectures | +| [**GitLab Releases**](https://about.gitlab.com/releases/)<br/>What's new in GitLab. | | + +## Popular topics + +Have a look at some of our most popular topics: + +| Popular topic | Description | +|:-----------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------| +| [Two-factor authentication](user/profile/account/two_factor_authentication.md) | Improve the security of your GitLab account. | +| [GitLab groups](user/group/index.md) | Manage projects together. | +| [GitLab CI/CD pipeline configuration reference](ci/yaml/README.md) | Available configuration options for `.gitlab-ci.yml` files. | +| [Activate GitLab EE with a license](user/admin_area/license.md) **(STARTER ONLY)** | Activate GitLab Enterprise Edition functionality with a license. | +| [Back up and restore GitLab](raketasks/backup_restore.md) **(CORE ONLY)** | Rake tasks for backing up and restoring GitLab self-managed instances. | +| [GitLab release and maintenance policy](policy/maintenance.md) | Policies for version naming and cadence, and also upgrade recommendations. | +| [Elasticsearch integration](integration/elasticsearch.md) **(STARTER ONLY)** | Integrate Elasticsearch with GitLab to enable advanced searching. | +| [Omnibus GitLab database settings](https://docs.gitlab.com/omnibus/settings/database.html) **(CORE ONLY)** | Database settings for Omnibus GitLab self-managed instances. | +| [Omnibus GitLab NGINX settings](https://docs.gitlab.com/omnibus/settings/nginx.html) **(CORE ONLY)** | NGINX settings for Omnibus GitLab self-managed instances. | +| [Omnibus GitLab SSL configuration](https://docs.gitlab.com/omnibus/settings/ssl.html) **(CORE ONLY)** | SSL settings for Omnibus GitLab self-managed instances. | +| [GitLab.com settings](user/gitlab_com/index.md) | Settings used for GitLab.com. | ## The entire DevOps Lifecycle @@ -62,7 +64,7 @@ than ever. The following sections provide links to documentation for each DevOps stage: -| DevOps Stage | Documentation for | +| DevOps stage | Documentation for | |:------------------------|:------------------------------------------------------------| | [Manage](#manage) | Statistics and analytics features. | | [Plan](#plan) | Project planning and management features. | @@ -86,7 +88,7 @@ GitLab provides statistics and insight into ways you can maximize the value of G The following documentation relates to the DevOps **Manage** stage: -| Manage Topics | Description | +| Manage topics | Description | |:--------------------------------------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | [Authentication and<br/>Authorization](administration/auth/README.md) **(CORE ONLY)** | Supported authentication and authorization providers. | | [GitLab Value Stream Analytics](user/project/cycle_analytics.md) | Measure the time it takes to go from an [idea to production](https://about.gitlab.com/blog/2016/08/05/continuous-integration-delivery-and-deployment-with-gitlab/#from-idea-to-production-with-gitlab) for each project you have. | @@ -107,7 +109,7 @@ management tools. The following documentation relates to the DevOps **Plan** stage: -| Plan Topics | Description | +| Plan topics | Description | |:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------| | [Burndown Charts](user/project/milestones/burndown_charts.md) **(STARTER)** | Watch your project's progress throughout a specific milestone. | | [Discussions](user/discussions/index.md) | Threads, comments, and resolvable threads in issues, commits, and merge requests. | @@ -121,7 +123,7 @@ The following documentation relates to the DevOps **Plan** stage: | [Related Issues](user/project/issues/related_issues.md) **(STARTER)** | Create a relationship between issues. | | [Requirements Management](user/project/requirements/index.md) **(ULTIMATE)** | Check your products against a set of criteria. | | [Roadmap](user/group/roadmap/index.md) **(ULTIMATE)** | Visualize epic timelines. | -| [Service Desk](user/project/service_desk.md) **(PREMIUM)** | A simple way to allow people to create issues in your GitLab instance without needing their own user account. | +| [Service Desk](user/project/service_desk.md) | A simple way to allow people to create issues in your GitLab instance without needing their own user account. | | [Time Tracking](user/project/time_tracking.md) | Track time spent on issues and merge requests. | | [Todos](user/todos.md) | Keep track of work requiring attention with a chronological list displayed on a simple dashboard. | @@ -136,7 +138,7 @@ The following documentation relates to the DevOps **Plan** stage: Consolidate source code into a single [distributed version control system](https://en.wikipedia.org/wiki/Distributed_version_control) that’s easily managed and controlled without disrupting your workflow. -GitLab’s Git repositories come complete with branching tools and access +GitLab repositories come complete with branching tools and access controls, providing a scalable, single source of truth for collaborating on projects and code. @@ -144,7 +146,7 @@ The following documentation relates to the DevOps **Create** stage: #### Projects and Groups -| Create Topics - Projects and Groups | Description | +| Create topics - Projects and Groups | Description | |:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------| | [Advanced global search](user/search/advanced_global_search.md) **(STARTER)** | Leverage Elasticsearch for faster, more advanced code search across your entire GitLab instance. | | [Advanced syntax search](user/search/advanced_search_syntax.md) **(STARTER)** | Use advanced queries for more targeted search results. | @@ -169,7 +171,7 @@ The following documentation relates to the DevOps **Create** stage: #### Repositories -| Create Topics - Repositories | Description | +| Create topics - Repositories | Description | |:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:--------------------------------------------------------------------------------| | [Branches](user/project/repository/branches/index.md) and the [default branch](user/project/repository/branches/index.md#default-branch) | How to use branches in GitLab. | | [Commits](user/project/repository/index.md#commits) and [signing commits](user/project/repository/gpg_signed_commits/index.md) | Work with commits, and use GPG to sign your commits. | @@ -192,13 +194,13 @@ The following documentation relates to the DevOps **Create** stage: #### Merge Requests -| Create Topics - Merge Requests | Description | +| Create topics - Merge Requests | Description | |:--------------------------------------------------------------------------------------------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------| | [Checking out merge requests locally](user/project/merge_requests/reviewing_and_managing_merge_requests.md#checkout-merge-requests-locally) | Tips for working with merge requests locally. | | [Cherry-picking](user/project/merge_requests/cherry_pick_changes.md) | Use GitLab for cherry-picking changes. | | [Merge request thread resolution](user/discussions/index.md#moving-a-single-thread-to-a-new-issue) | Resolve threads, move threads in a merge request to an issue, and only allow merge requests to be merged if all threads are resolved. | | [Merge requests](user/project/merge_requests/index.md) | Merge request management. | -| [Work In Progress "WIP" merge requests](user/project/merge_requests/work_in_progress_merge_requests.md) | Prevent merges of work-in-progress merge requests. | +| [**Draft** merge requests](user/project/merge_requests/work_in_progress_merge_requests.md) | Prevent merges of draft merge requests. | <div align="right"> <a type="button" class="btn btn-default" href="#overview"> @@ -208,7 +210,7 @@ The following documentation relates to the DevOps **Create** stage: #### Integration and Automation -| Create Topics - Integration and Automation | Description | +| Create topics - Integration and Automation | Description | |:------------------------------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------| | [GitLab API](api/README.md) | Integrate GitLab via a simple and powerful API. | | [GitLab Integration](integration/README.md) | Integrate with multiple third-party services with GitLab to allow external issue trackers and external authentication. | @@ -235,9 +237,9 @@ scales to run your tests faster. The following documentation relates to the DevOps **Verify** stage: -| Verify Topics | Description | +| Verify topics | Description | |:----------------------------------------------------------------------------------|:--------------------------------------------------------------------------------------------------------| -| [Code Quality reports](user/project/merge_requests/code_quality.md) **(STARTER)** | Analyze source code quality. | +| [Code Quality reports](user/project/merge_requests/code_quality.md) | Analyze source code quality. | | [GitLab CI/CD](ci/README.md) | Explore the features and capabilities of Continuous Integration with GitLab. | | [JUnit test reports](ci/junit_test_reports.md) | Display JUnit test reports on merge requests. | | [Multi-project pipelines](ci/multi_project_pipelines.md) **(PREMIUM)** | Visualize entire pipelines that span multiple projects, including all cross-project inter-dependencies. | @@ -258,7 +260,7 @@ packages, which can be easily consumed as a dependency in downstream projects. The following documentation relates to the DevOps **Package** stage: -| Package Topics | Description | +| Package topics | Description | |:----------------------------------------------------------------|:-------------------------------------------------------| | [Container Registry](user/packages/container_registry/index.md) | The GitLab Container Registry enables every project in GitLab to have its own space to store [Docker](https://www.docker.com/) images. | | [Dependency Proxy](user/packages/dependency_proxy/index.md) **(PREMIUM)** | The GitLab Dependency Proxy sets up a local proxy for frequently used upstream images/packages. | @@ -280,7 +282,7 @@ confidently and securely with GitLab’s built-in Continuous Delivery and Deploy The following documentation relates to the DevOps **Release** stage: -| Release Topics | Description | +| Release topics | Description | |:------------------------------------------------------------------------------------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------| | [Auto Deploy](topics/autodevops/stages.md#auto-deploy) | Configure GitLab for the deployment of your application. | | [Canary Deployments](user/project/canary_deployments.md) **(PREMIUM)** | Employ a popular CI strategy where a small portion of the fleet is updated to the new version first. | @@ -306,19 +308,19 @@ configuration. Then customize everything from buildpacks to CI/CD. The following documentation relates to the DevOps **Configure** stage: -| Configure Topics | Description | +| Configure topics | Description | |:-----------------------------------------------------------------------------------------------------------------------------------|:--------------------------------------------------------------------------| | [Auto DevOps](topics/autodevops/index.md) | Automatically employ a complete DevOps lifecycle. | | [Create Kubernetes clusters](user/project/clusters/add_remove_clusters.md#create-new-cluster) | Use Kubernetes and GitLab. | | [Executable Runbooks](user/project/clusters/runbooks/index.md) | Documented procedures that explain how to carry out particular processes. | | [GitLab ChatOps](ci/chatops/README.md) | Interact with CI/CD jobs through chat services. | -| [Installing Applications](user/project/clusters/index.md#installing-applications) | Deploy Helm, Ingress, and Prometheus on Kubernetes. | +| [Installing Applications](user/project/clusters/index.md#installing-applications) | Install Helm charts such as Ingress and Prometheus on Kubernetes. | | [Mattermost slash commands](user/project/integrations/mattermost_slash_commands.md) | Enable and use slash commands from within Mattermost. | -| [Multiple Kubernetes Clusters](user/project/clusters/index.md#multiple-kubernetes-clusters-premium) **(PREMIUM)** | Associate more than one Kubernetes clusters to your project. | +| [Multiple Kubernetes Clusters](user/project/clusters/index.md#multiple-kubernetes-clusters) | Associate more than one Kubernetes clusters to your project. | | [Protected variables](ci/variables/README.md#protect-a-custom-variable) | Restrict variables to protected branches and tags. | | [Serverless](user/project/clusters/serverless/index.md) | Run serverless workloads on Kubernetes. | | [Slack slash commands](user/project/integrations/slack_slash_commands.md) | Enable and use slash commands from within Slack. | -| [Manage your infrastructure with Terraform](user/infrastructure/index.md) | Manage your infrastructure as you run your CI/CD pipeline. | +| [Manage your infrastructure with Terraform](user/infrastructure/index.md) | Manage your infrastructure as you run your CI/CD pipeline. | <div align="right"> <a type="button" class="btn btn-default" href="#overview"> @@ -335,7 +337,7 @@ instant how code changes impact your production environment. The following documentation relates to the DevOps **Monitor** stage: -| Monitor Topics | Description | +| Monitor topics | Description | |:------------------------------------------------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------| | [GitLab Performance Monitoring](administration/monitoring/performance/index.md) **(CORE ONLY)** | Use Prometheus and Grafana to monitor the performance of your GitLab instance. | | [GitLab Prometheus](administration/monitoring/prometheus/index.md) **(CORE ONLY)** | Configure the bundled Prometheus to collect various metrics from your GitLab instance. | @@ -360,7 +362,7 @@ high-level view on projects and groups, and start remediation processes when nee The following documentation relates to the DevOps **Secure** stage: -| Secure Topics | Description | +| Secure topics | Description | |:------------------------------------------------------------------------------------------------------|:-----------------------------------------------------------------------| | [Compliance Dashboard](user/compliance/compliance_dashboard/index.md) **(ULTIMATE)** | View the most recent Merge Request activity in a group. | | [Container Scanning](user/application_security/container_scanning/index.md) **(ULTIMATE)** | Use Clair to scan Docker images for known vulnerabilities. | @@ -442,7 +444,7 @@ If you are coming to GitLab from another platform, you'll find the following inf </a> </div> -## Building an integration with GitLab +## Build an integration with GitLab There are many ways to integrate with GitLab, including: diff --git a/doc/administration/audit_events.md b/doc/administration/audit_events.md index cc94d756f99..c85fb2d2e47 100644 --- a/doc/administration/audit_events.md +++ b/doc/administration/audit_events.md @@ -56,9 +56,7 @@ From there, you can see the following actions: - User sign-in via [Group SAML](../user/group/saml_sso/index.md) - Permissions changes of a user assigned to a group - Removed user from group -- Project imported in to group -- Project added to group and with which visibility level -- Project removed from group +- Project repository imported into group - [Project shared with group](../user/project/members/share_project_with_groups.md) and with which [permissions](../user/permissions.md) - Removal of a previously shared group with a project @@ -80,7 +78,7 @@ To view a project's audit events, navigate to **Project > Settings > Audit Event From there, you can see the following actions: - Added or removed deploy keys -- Project created, deleted, renamed, moved(transferred), changed path +- Project created, deleted, renamed, moved (transferred), changed path - Project changed visibility level - User was added to project and with which [permissions](../user/permissions.md) - Permission changes of a user assigned to a project @@ -96,6 +94,7 @@ From there, you can see the following actions: - Permission to approve merge requests by committers was updated ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7531) in GitLab 12.9) - Permission to approve merge requests by authors was updated ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7531) in GitLab 12.9) - Number of required approvals was updated ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7531) in GitLab 12.9) +- Added or removed users and groups from project approval groups ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/213603) in GitLab 13.2) Project events can also be accessed via the [Project Audit Events API](../api/audit_events.md#project-audit-events-starter) diff --git a/doc/administration/auth/jwt.md b/doc/administration/auth/jwt.md index 29b192a4845..d3a77fdaca5 100644 --- a/doc/administration/auth/jwt.md +++ b/doc/administration/auth/jwt.md @@ -62,7 +62,8 @@ JWT will provide you with a secret key for you to use. } ``` - NOTE: **Note:** For more information on each configuration option refer to + NOTE: **Note:** + For more information on each configuration option refer to the [OmniAuth JWT usage documentation](https://github.com/mbleigh/omniauth-jwt#usage). 1. Change `YOUR_APP_SECRET` to the client secret and set `auth_url` to your redirect URL. diff --git a/doc/administration/auth/ldap/google_secure_ldap.md b/doc/administration/auth/ldap/google_secure_ldap.md index 2271ce93b6f..1f8fca33811 100644 --- a/doc/administration/auth/ldap/google_secure_ldap.md +++ b/doc/administration/auth/ldap/google_secure_ldap.md @@ -34,7 +34,8 @@ The steps below cover: 'Entire domain (GitLab)' or 'Selected organizational units' for both 'Verify user credentials' and 'Read user information'. Select 'Add LDAP Client' - TIP: **Tip:** If you plan to use GitLab [LDAP Group Sync](index.md#group-sync-starter-only) + TIP: **Tip:** + If you plan to use GitLab [LDAP Group Sync](index.md#group-sync-starter-only) , turn on 'Read group information'.  diff --git a/doc/administration/auth/ldap/index.md b/doc/administration/auth/ldap/index.md index 4a7a972596f..aef6c70ff92 100644 --- a/doc/administration/auth/ldap/index.md +++ b/doc/administration/auth/ldap/index.md @@ -53,7 +53,7 @@ are already logged in or are using Git over SSH will still be able to access GitLab for up to one hour. Manually block the user in the GitLab Admin Area to immediately block all access. -NOTE: **Note**: +NOTE: **Note:** GitLab Enterprise Edition Starter supports a [configurable sync time](#adjusting-ldap-user-sync-schedule-starter-only). @@ -99,7 +99,7 @@ Normally, if you specify `simple_tls` it will be on port 636, while `start_tls` would be on port 389. `plain` also operates on port 389. Removed values: `tls` was replaced with `start_tls` and `ssl` was replaced with `simple_tls`. NOTE: **Note:** -LDAP users must have an email address set, regardless of whether it is used to log in. +LDAP users must have an email address set, regardless of whether it is used to sign-in. ### Example Configurations **(CORE ONLY)** @@ -169,7 +169,7 @@ production: | Setting | Description | Required | Examples | | ------- | ----------- | -------- | -------- | -| `label` | A human-friendly name for your LDAP server. It will be displayed on your login page. | yes | `'Paris'` or `'Acme, Ltd.'` | +| `label` | A human-friendly name for your LDAP server. It will be displayed on your sign-in page. | yes | `'Paris'` or `'Acme, Ltd.'` | | `host` | IP address or domain name of your LDAP server. | yes | `'ldap.mydomain.com'` | | `port` | The port to connect with on your LDAP server. Always an integer, not a string. | yes | `389` or `636` (for SSL) | | `uid` | LDAP attribute for username. Should be the attribute, not the value that maps to the `uid`. | yes | `'sAMAccountName'`, `'uid'`, `'userPrincipalName'` | @@ -179,7 +179,7 @@ production: | `verify_certificates` | Enables SSL certificate verification if encryption method is `start_tls` or `simple_tls`. Defaults to true. | no | boolean | | `timeout` | Set a timeout, in seconds, for LDAP queries. This helps avoid blocking a request if the LDAP server becomes unresponsive. A value of 0 means there is no timeout. | no | `10` or `30` | | `active_directory` | This setting specifies if LDAP server is Active Directory LDAP server. For non-AD servers it skips the AD specific queries. If your LDAP server is not AD, set this to false. | no | boolean | -| `allow_username_or_email_login` | If enabled, GitLab will ignore everything after the first `@` in the LDAP username submitted by the user on login. If you are using `uid: 'userPrincipalName'` on ActiveDirectory you need to disable this setting, because the userPrincipalName contains an `@`. | no | boolean | +| `allow_username_or_email_login` | If enabled, GitLab will ignore everything after the first `@` in the LDAP username submitted by the user on sign-in. If you are using `uid: 'userPrincipalName'` on ActiveDirectory you need to disable this setting, because the userPrincipalName contains an `@`. | no | boolean | | `block_auto_created_users` | To maintain tight control over the number of active users on your GitLab installation, enable this setting to keep new users blocked until they have been cleared by the admin (default: false). | no | boolean | | `base` | Base where we can search for users. | yes | `'ou=people,dc=gitlab,dc=example'` or `'DC=mydomain,DC=com'` | | `user_filter` | Filter LDAP users. Format: [RFC 4515](https://tools.ietf.org/search/rfc4515) Note: GitLab does not support `omniauth-ldap`'s custom filter syntax. | no | `'(employeeType=developer)'` or `'(&(objectclass=user)(|(samaccountname=momo)(samaccountname=toto)))'` | @@ -197,7 +197,7 @@ production: ### Attribute Configuration Settings **(CORE ONLY)** -LDAP attributes that GitLab will use to create an account for the LDAP user. The specified attribute can either be the attribute name as a string (e.g. `'mail'`), or an array of attribute names to try in order (e.g. `['mail', 'email']`). Note that the user's LDAP login will always be the attribute specified as `uid` above. +LDAP attributes that GitLab will use to create an account for the LDAP user. The specified attribute can either be the attribute name as a string (e.g. `'mail'`), or an array of attribute names to try in order (e.g. `['mail', 'email']`). Note that the user's LDAP sign-in will always be the attribute specified as `uid` above. | Setting | Description | Required | Examples | | ------- | ----------- | -------- | -------- | @@ -396,7 +396,7 @@ Be sure to choose a different provider ID made of letters a-z and numbers 0-9. This ID will be stored in the database so that GitLab can remember which LDAP server a user belongs to. - + Based on the example illustrated on the image above, our `gitlab.rb` configuration would look like: @@ -424,7 +424,7 @@ gitlab_rails['ldap_servers'] = { 'port' => 636, ... } - + } ``` @@ -450,7 +450,7 @@ has bit 2 set. See <https://ctovswild.com/2009/09/03/bitmask-searches-in-ldap/> for more information. The user will be set to `ldap_blocked` state in GitLab if the above conditions -fail. This means the user will not be able to login or push/pull code. +fail. This means the user will not be able to sign-in or push/pull code. The process will also update the following user information: @@ -605,6 +605,12 @@ When enabled, the following applies: - Users are not allowed to share project with other groups or invite members to a project created in a group. +To enable it you need to: + +1. [Enable LDAP](#configuration-core-only) +1. Navigate to **(admin)** **Admin Area > Settings -> Visibility and access controls**. +1. Make sure the "Lock memberships to LDAP synchronization" checkbox is enabled. + ### Adjusting LDAP group sync schedule **(STARTER ONLY)** NOTE: **Note:** diff --git a/doc/administration/auth/ldap/ldap-troubleshooting.md b/doc/administration/auth/ldap/ldap-troubleshooting.md index 909802b5dec..75183f54990 100644 --- a/doc/administration/auth/ldap/ldap-troubleshooting.md +++ b/doc/administration/auth/ldap/ldap-troubleshooting.md @@ -27,7 +27,7 @@ Could not authenticate you from Ldapmain because "Connection timed out - user sp ``` If your configured LDAP provider and/or endpoint is offline or otherwise -unreachable by GitLab, no LDAP user will be able to authenticate and log in. +unreachable by GitLab, no LDAP user will be able to authenticate and sign-in. GitLab does not cache or store credentials for LDAP users to provide authentication during an LDAP outage. @@ -81,7 +81,7 @@ adapter.ldap_search(options) For examples of how this is run, [review the `Adapter` module](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/ee/gitlab/auth/ldap/adapter.rb). -### User logins +### User sign-ins #### No users are found @@ -97,24 +97,24 @@ In this case, you con confirm which of the above is true using [ldapsearch](#ldapsearch) with the existing LDAP configuration in your `/etc/gitlab/gitlab.rb`. -#### User(s) cannot login +#### User(s) cannot sign-in -A user can have trouble logging in for any number of reasons. To get started, +A user can have trouble signing in for any number of reasons. To get started, here are some questions to ask yourself: - Does the user fall under the [configured `base`](index.md#configuration-core-only) in - LDAP? The user must fall under this `base` to login. + LDAP? The user must fall under this `base` to sign-in. - Does the user pass through the [configured `user_filter`](index.md#set-up-ldap-user-filter-core-only)? If one is not configured, this question can be ignored. If it is, then the - user must also pass through this filter to be allowed to login. + user must also pass through this filter to be allowed to sign-in. - Refer to our docs on [debugging the `user_filter`](#debug-ldap-user-filter). If the above are both okay, the next place to look for the problem is the logs themselves while reproducing the issue. -- Ask the user to login and let it fail. +- Ask the user to sign-in and let it fail. - [Look through the output](#gitlab-logs) for any errors or other - messages about the login. You may see one of the other error messages on + messages about the sign-in. You may see one of the other error messages on this page, in which case that section can help resolve the issue. If the logs don't lead to the root of the problem, use the @@ -125,9 +125,9 @@ It can also be helpful to [debug a user sync](#sync-all-users-starter-only) to investigate further. -#### Invalid credentials on login +#### Invalid credentials on sign-in -If that the login credentials used are accurate on LDAP, ensure the following +If that the sign-in credentials used are accurate on LDAP, ensure the following are true for the user in question: - Make sure the user you are binding with has enough permissions to read the user's @@ -138,7 +138,7 @@ are true for the user in question: #### Email has already been taken -A user tries to login with the correct LDAP credentials, is denied access, +A user tries to sign-in with the correct LDAP credentials, is denied access, and the [production.log](../../logs.md#productionlog) shows an error that looks like this: ```plaintext @@ -163,7 +163,7 @@ user.username This will show you which user has this email address. One of two steps will have to be taken here: -- To create a new GitLab user/username for this user when logging in with LDAP, +- To create a new GitLab user/username for this user when signing in with LDAP, remove the secondary email to remove the conflict. - To use an existing GitLab user/username for this user to use with LDAP, remove this email as a secondary email and make it a primary one so GitLab @@ -235,7 +235,7 @@ uid: John There's a lot here, so let's go over what could be helpful when debugging. First, GitLab will look for all users that have previously -logged in with LDAP and iterate on them. Each user's sync will start with +signed in with LDAP and iterate on them. Each user's sync will start with the following line that contains the user's username and email, as they exist in GitLab now: @@ -244,7 +244,7 @@ Syncing user John, email@example.com ``` If you don't find a particular user's GitLab email in the output, then that -user hasn't logged in with LDAP yet. +user hasn't signed in with LDAP yet. Next, GitLab searches its `identities` table for the existing link between this user and the configured LDAP provider(s): @@ -313,7 +313,7 @@ things to check to debug the situation. 1. Search for the user 1. Open the user, by clicking on their name. Do not click 'Edit'. 1. Navigate to the **Identities** tab. There should be an LDAP identity with - an LDAP DN as the 'Identifier'. If not, this user hasn't logged in with + an LDAP DN as the 'Identifier'. If not, this user hasn't signed in with LDAP yet and must do so first. - You've waited an hour or [the configured interval](index.md#adjusting-ldap-group-sync-schedule-starter-only) for the group to @@ -346,7 +346,7 @@ the following are true: - A [`group_base` is also configured](index.md#group-sync-starter-only). - The configured `admin_group` in the `gitlab.rb` is a CN, rather than a DN or an array. - This CN falls under the scope of the configured `group_base`. -- The members of the `admin_group` have already logged into GitLab with their LDAP +- The members of the `admin_group` have already signed into GitLab with their LDAP credentials. GitLab will only grant this admin access to the users whose accounts are already connected to LDAP. @@ -357,7 +357,7 @@ GitLab syncs the `admin_group`. #### Sync all groups **(STARTER ONLY)** -NOTE: **NOTE:** +NOTE: **Note:** To sync all groups manually when debugging is unnecessary, [use the Rake task](../../raketasks/ldap.md#run-a-group-sync-starter-only) instead. @@ -571,7 +571,7 @@ If a user account is blocked or unblocked due to the LDAP configuration, a message will be [logged to `application.log`](../../logs.md#applicationlog). If there is an unexpected error during an LDAP lookup (configuration error, -timeout), the login is rejected and a message will be [logged to +timeout), the sign-in is rejected and a message will be [logged to `production.log`](../../logs.md#productionlog). ### ldapsearch @@ -653,7 +653,7 @@ adfind -h ad.example.org:636 -ssl -u "CN=GitLabSRV,CN=Users,DC=GitLab,DC=org" -u ### Rails console -CAUTION: **CAUTION:** +CAUTION: **Caution:** Please note that it is very easy to create, read, modify, and destroy data on the rails console, so please be sure to run commands exactly as listed. diff --git a/doc/administration/auth/okta.md b/doc/administration/auth/okta.md index f7ab60ab56b..78b10aedb77 100644 --- a/doc/administration/auth/okta.md +++ b/doc/administration/auth/okta.md @@ -159,8 +159,7 @@ You might want to try this out on an incognito browser window. >**Note:** Make sure the groups exist and are assigned to the Okta app. -You can take a look of the [SAML documentation](../../integration/saml.md#marking-users-as-external-based-on-saml-groups) on external groups since -it works the same. +You can take a look of the [SAML documentation](../../integration/saml.md#saml-groups) on configuring groups. <!-- ## Troubleshooting diff --git a/doc/administration/auth/smartcard.md b/doc/administration/auth/smartcard.md index 9ad1e0641f6..80d2efbad84 100644 --- a/doc/administration/auth/smartcard.md +++ b/doc/administration/auth/smartcard.md @@ -208,7 +208,7 @@ attribute. As a prerequisite, you must use an LDAP server that: client_certificate_required_port: 3443 ``` - NOTE: **Note** + NOTE: **Note:** Assign a value to at least one of the following variables: `client_certificate_required_host` or `client_certificate_required_port`. diff --git a/doc/administration/feature_flags.md b/doc/administration/feature_flags.md index 9e87eed4508..678ab6c5d7b 100644 --- a/doc/administration/feature_flags.md +++ b/doc/administration/feature_flags.md @@ -103,10 +103,28 @@ Some feature flags can be enabled or disabled on a per project basis: Feature.enable(:<feature flag>, Project.find(<project id>)) ``` -For example, to enable the [`:release_evidence_collection`](../ci/junit_test_reports.md#enabling-the-feature) feature flag for project `1234`: +For example, to enable the [`:junit_pipeline_view`](../ci/junit_test_reports.md#enabling-the-junit-test-reports-feature-core-only) feature flag for project `1234`: ```ruby -Feature.enable(:release_evidence_collection, Project.find(1234)) +Feature.enable(:junit_pipeline_view, Project.find(1234)) +``` + +`Feature.enable` and `Feature.disable` always return `nil`, this is not an indication that the command failed: + +```ruby +irb(main):001:0> Feature.enable(:release_evidence_collection) +=> nil +``` + +To check if a flag is enabled or disabled you can use `Feature.enabled?` or `Feature.disabled?`: + +```ruby +Feature.enable(:release_evidence_collection) +=> nil +Feature.enabled?(:release_evidence_collection) +=> true +Feature.disabled?(:release_evidence_collection) +=> false ``` When the feature is ready, GitLab will remove the feature flag, the option for diff --git a/doc/administration/geo/disaster_recovery/bring_primary_back.md b/doc/administration/geo/disaster_recovery/bring_primary_back.md index b19e55595e7..3b7c7fd549c 100644 --- a/doc/administration/geo/disaster_recovery/bring_primary_back.md +++ b/doc/administration/geo/disaster_recovery/bring_primary_back.md @@ -32,13 +32,15 @@ To bring the former **primary** node up to date: sudo gitlab-ctl start ``` - NOTE: **Note:** If you [disabled the **primary** node permanently](index.md#step-2-permanently-disable-the-primary-node), + NOTE: **Note:** + If you [disabled the **primary** node permanently](index.md#step-2-permanently-disable-the-primary-node), you need to undo those steps now. For Debian/Ubuntu you just need to run `sudo systemctl enable gitlab-runsvdir`. For CentOS 6, you need to install the GitLab instance from scratch and set it up as a **secondary** node by following [Setup instructions](../replication/index.md#setup-instructions). In this case, you don't need to follow the next step. - NOTE: **Note:** If you [changed the DNS records](index.md#step-4-optional-updating-the-primary-domain-dns-record) + NOTE: **Note:** + If you [changed the DNS records](index.md#step-4-optional-updating-the-primary-domain-dns-record) for this node during disaster recovery procedure you may need to [block all the writes to this node](planned_failover.md#prevent-updates-to-the-primary-node) during this procedure. diff --git a/doc/administration/geo/disaster_recovery/index.md b/doc/administration/geo/disaster_recovery/index.md index f690ec63cf9..2d837ebb369 100644 --- a/doc/administration/geo/disaster_recovery/index.md +++ b/doc/administration/geo/disaster_recovery/index.md @@ -122,18 +122,31 @@ Note the following when promoting a secondary: roles ['geo_secondary_role'] ``` -1. Promote the **secondary** node to the **primary** node. Execute: +1. Promote the **secondary** node to the **primary** node. + + Before promoting a secondary node to primary, preflight checks should be run. They can be run separately or along with the promotion script. + + To promote the secondary node to primary along with preflight checks: ```shell gitlab-ctl promote-to-primary-node ``` - If you have already run the [preflight checks](planned_failover.md#preflight-checks), you can skip them with: + CAUTION: **Warning:** + Skipping preflight checks will promote the secondary to a primary without any further confirmation! + + If you have already run the [preflight checks](planned_failover.md#preflight-checks) or don't want to run them, you can skip preflight checks with: ```shell gitlab-ctl promote-to-primary-node --skip-preflight-check ``` + You can also run preflight checks separately: + + ```shell + gitlab-ctl promotion-preflight-checks + ``` + 1. Verify you can connect to the newly promoted **primary** node using the URL used previously for the **secondary** node. 1. If successful, the **secondary** node has now been promoted to the **primary** node. @@ -261,7 +274,7 @@ secondary domain, like changing Git remotes and API URLs. external_url 'https://<new_external_url>' ``` - NOTE: **Note** + NOTE: **Note:** Changing `external_url` won't prevent access via the old secondary URL, as long as the secondary DNS records are still intact. diff --git a/doc/administration/geo/disaster_recovery/planned_failover.md b/doc/administration/geo/disaster_recovery/planned_failover.md index 0ce1325a537..ef0e67434d0 100644 --- a/doc/administration/geo/disaster_recovery/planned_failover.md +++ b/doc/administration/geo/disaster_recovery/planned_failover.md @@ -45,8 +45,19 @@ be found in `/var/opt/gitlab/gitlab-rails/shared/pages` if using Omnibus). ## Preflight checks -Follow these steps before scheduling a planned failover to ensure the process -will go smoothly. +Run this command to list out all preflight checks and automatically check if replication and verification are complete before scheduling a planned failover to ensure the process will go smoothly: + +```shell +gitlab-ctl promotion-preflight-checks +``` + +You can run this command in `force` mode to promote to primary even if preflight checks fail: + +```shell +sudo gitlab-ctl promotion-preflight-checks --force +``` + +Each step is described in more detail below. ### Object storage diff --git a/doc/administration/geo/replication/database.md b/doc/administration/geo/replication/database.md index 3f2d46ba457..02f51e79907 100644 --- a/doc/administration/geo/replication/database.md +++ b/doc/administration/geo/replication/database.md @@ -130,7 +130,8 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o connect to the **primary** node's database. For this reason, we need the address of each node. - NOTE: **Note:** For external PostgreSQL instances, see [additional instructions](external_database.md). + NOTE: **Note:** + For external PostgreSQL instances, see [additional instructions](external_database.md). If you are using a cloud provider, you can lookup the addresses for each Geo node through your cloud provider's management console. @@ -419,7 +420,8 @@ data before running `pg_basebackup`. 1. Execute the command below to start a backup/restore and begin the replication - CAUTION: **Warning:** Each Geo **secondary** node must have its own unique replication slot name. + CAUTION: **Warning:** + Each Geo **secondary** node must have its own unique replication slot name. Using the same slot name between two secondaries will break PostgreSQL replication. ```shell diff --git a/doc/administration/geo/replication/datatypes.md b/doc/administration/geo/replication/datatypes.md index f50da27d82f..5636ff79189 100644 --- a/doc/administration/geo/replication/datatypes.md +++ b/doc/administration/geo/replication/datatypes.md @@ -45,6 +45,8 @@ verification methods: | Blobs | Archived CI build traces _(object storage)_ | Geo with API/Managed (*2*) | _Not implemented_ | | Blobs | Container registry _(filesystem)_ | Geo with API/Docker API | _Not implemented_ | | Blobs | Container registry _(object storage)_ | Geo with API/Managed/Docker API (*2*) | _Not implemented_ | +| Blobs | Package registry _(filesystem)_ | Geo with API | _Not implemented_ | +| Blobs | Package registry _(object storage)_ | Geo with API/Managed (*2*) | _Not implemented_ | - (*1*): Redis replication can be used as part of HA with Redis sentinel. It's not used between Geo nodes. - (*2*): Object storage replication can be performed by Geo or by your object storage provider/appliance @@ -124,41 +126,41 @@ these epics/issues: - [Unreplicated Data Types](https://gitlab.com/groups/gitlab-org/-/epics/893) - [Verify all replicated data](https://gitlab.com/groups/gitlab-org/-/epics/1430) -DANGER: **DANGER** +DANGER: **Danger:** Features not on this list, or with **No** in the **Replicated** column, are not replicated on the **secondary** node. Failing over without manually replicating data from those features will cause the data to be **lost**. If you wish to use those features on a **secondary** node, or to execute a failover successfully, you must replicate their data using some other means. -| Feature | Replicated | Verified | Notes | +| Feature | Replicated (added in GitLab version) | Verified (added in GitLab version) | Notes | |:---------------------------------------------------------------------|:---------------------------------------------------------|:--------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------| -| Application data in PostgreSQL | **Yes** | **Yes** | | -| Project repository | **Yes** | **Yes** | | -| Project wiki repository | **Yes** | **Yes** | | -| Project designs repository | **Yes** | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/32467) | | -| Uploads | **Yes** | [No](https://gitlab.com/groups/gitlab-org/-/epics/1817) | Verified only on transfer, or manually (*1*) | -| LFS objects | **Yes** | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/8922) | Verified only on transfer, or manually (*1*). Unavailable for new LFS objects in 11.11.x and 12.0.x (*2*). | -| CI job artifacts (other than traces) | **Yes** | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/8923) | Verified only manually (*1*) | -| Archived traces | **Yes** | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/8923) | Verified only on transfer, or manually (*1*) | -| Personal snippets | **Yes** | **Yes** | | +| Application data in PostgreSQL | **Yes** (10.2) | **Yes** (10.2) | | +| Project repository | **Yes** (10.2) | **Yes** (10.7) | | +| Project wiki repository | **Yes** (10.2) | **Yes** (10.7) | | +| Project designs repository | **Yes** (12.7) | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/32467) | | +| Uploads | **Yes** (10.2) | [No](https://gitlab.com/groups/gitlab-org/-/epics/1817) | Verified only on transfer, or manually (*1*) | +| LFS objects | **Yes** (10.2) | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/8922) | Verified only on transfer, or manually (*1*). Unavailable for new LFS objects in 11.11.x and 12.0.x (*2*). | +| CI job artifacts (other than traces) | **Yes** (10.4) | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/8923) | Verified only manually (*1*) | +| Archived traces | **Yes** (10.4) | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/8923) | Verified only on transfer, or manually (*1*) | +| Personal snippets | **Yes** (10.2) | **Yes** (10.2) | | | [Versioned snippets](../../../user/snippets.md#versioned-snippets) | [No](https://gitlab.com/groups/gitlab-org/-/epics/2809) | [No](https://gitlab.com/groups/gitlab-org/-/epics/2810) | | -| Project snippets | **Yes** | **Yes** | | -| Object pools for forked project deduplication | **Yes** | No | | -| [Server-side Git Hooks](../../custom_hooks.md) | No | No | | +| Project snippets | **Yes** (10.2) | **Yes** (10.2) | | +| Object pools for forked project deduplication | **Yes** | No | | +| [Server-side Git hooks](../../server_hooks.md) | No | No | | | [Elasticsearch integration](../../../integration/elasticsearch.md) | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/1186) | No | | | [GitLab Pages](../../pages/index.md) | [No](https://gitlab.com/groups/gitlab-org/-/epics/589) | No | | -| [Container Registry](../../packages/container_registry.md) | **Yes** | No | | -| [NPM Registry](../../../user/packages/npm_registry/index.md) | [No](https://gitlab.com/groups/gitlab-org/-/epics/2346) | No | | -| [Maven Repository](../../../user/packages/maven_repository/index.md) | [No](https://gitlab.com/groups/gitlab-org/-/epics/2346) | No | | -| [Conan Repository](../../../user/packages/conan_repository/index.md) | [No](https://gitlab.com/groups/gitlab-org/-/epics/2346) | No | | -| [NuGet Repository](../../../user/packages/nuget_repository/index.md) | [No](https://gitlab.com/groups/gitlab-org/-/epics/2346) | No | | -| [PyPi Repository](../../../user/packages/pypi_repository/index.md) | [No](https://gitlab.com/groups/gitlab-org/-/epics/2554) | No | | -| [Composer Repository](../../../user/packages/composer_repository/index.md) | [No](https://gitlab.com/groups/gitlab-org/-/epics/3096) | No | | +| [Container Registry](../../packages/container_registry.md) | **Yes** (12.3) | No | | +| [NPM Registry](../../../user/packages/npm_registry/index.md) | **Yes** (13.2) | No | | +| [Maven Repository](../../../user/packages/maven_repository/index.md) | **Yes** (13.2) | No | | +| [Conan Repository](../../../user/packages/conan_repository/index.md) | **Yes** (13.2) | No | | +| [NuGet Repository](../../../user/packages/nuget_repository/index.md) | **Yes** (13.2) | No | | +| [PyPi Repository](../../../user/packages/pypi_repository/index.md) | **Yes** (13.2) | No | | +| [Composer Repository](../../../user/packages/composer_repository/index.md) | **Yes** (13.2) | No | | | [External merge request diffs](../../merge_request_diffs.md) | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/33817) | No | | | [Terraform State](../../terraform_state.md) | [No](https://gitlab.com/groups/gitlab-org/-/epics/3112)(*3*) | No | | -| [Vulnerability Export](../../../user/application_security/security_dashboard/#export-vulnerabilities-1) | [No](https://gitlab.com/groups/gitlab-org/-/epics/3111)(*3*) | No | | | -| Content in object storage | **Yes** | No | | +| [Vulnerability Export](../../../user/application_security/security_dashboard/#export-vulnerabilities) | [No](https://gitlab.com/groups/gitlab-org/-/epics/3111)(*3*) | No | | | +| Content in object storage | **Yes** (12.4) | No | | - (*1*): The integrity can be verified manually using [Integrity Check Rake Task](../../raketasks/check.md) on both nodes and comparing diff --git a/doc/administration/geo/replication/disable_geo.md b/doc/administration/geo/replication/disable_geo.md new file mode 100644 index 00000000000..f53b4c1b796 --- /dev/null +++ b/doc/administration/geo/replication/disable_geo.md @@ -0,0 +1,93 @@ +--- +stage: Enablement +group: Geo +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: howto +--- + +# Disabling Geo **(PREMIUM ONLY)** + +If you want to revert to a regular Omnibus setup after a test, or you have encountered a Disaster Recovery +situation and you want to disable Geo momentarily, you can use these instructions to disable your +Geo setup. + +There should be no functional difference between disabling Geo and having an active Geo setup with +no secondary Geo nodes if you remove them correctly. + +To disable Geo, follow these steps: + +1. [Remove all secondary Geo nodes](#remove-all-secondary-geo-nodes). +1. [Remove the primary node from the UI](#remove-the-primary-node-from-the-ui). +1. [Remove secondary replication slots](#remove-secondary-replication-slots). +1. [Remove Geo-related configuration](#remove-geo-related-configuration). +1. [(Optional) Revert PostgreSQL settings to use a password and listen on an IP](#optional-revert-postgresql-settings-to-use-a-password-and-listen-on-an-ip). + +## Remove all secondary Geo nodes + +To disable Geo, you need to first remove all your secondary Geo nodes, which means replication will not happen +anymore on these nodes. You can follow our docs to [remove your secondary Geo nodes](./remove_geo_node.md). + +If the current node that you want to keep using is a secondary node, you need to first promote it to primary. +You can use our steps on [how to promote a secondary node](../disaster_recovery/#step-3-promoting-a-secondary-node) +in order to do that. + +## Remove the primary node from the UI + +1. Go to **{admin}** **Admin Area >** **{location-dot}** **Geo** (`/admin/geo/nodes`). +1. Click the **Remove** button for the **primary** node. +1. Confirm by clicking **Remove** when the prompt appears. + +## Remove secondary replication slots + +To remove secondary replication slots, run one of the following queries on your primary +Geo node in a PostgreSQL console (`sudo gitlab-psql`): + +- If you already have a PostgreSQL cluster, drop individual replication slots by name to prevent + removing your secondary databases from the same cluster. You can use the following to get + all names and then drop each individual slot: + + ```sql + SELECT slot_name, slot_type, active FROM pg_replication_slots; -- view present replication slots + SELECT pg_drop_replication_slot('slot_name'); -- where slot_name is the one expected from above + ``` + +- To remove all secondary replication slots: + + ```sql + SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots; + ``` + +## Remove Geo-related configuration + +1. SSH into your primary Geo node and log in as root: + + ```shell + sudo -i + ``` + +1. Edit `/etc/gitlab/gitlab.rb` and remove the Geo related configuration by + removing any lines that enabled `geo_primary_role`: + + ```ruby + ## In pre-11.5 documentation, the role was enabled as follows. Remove this line. + geo_primary_role['enable'] = true + + ## In 11.5+ documentation, the role was enabled as follows. Remove this line. + roles ['geo_primary_role'] + ``` + +1. After making these changes, [reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure) + for the changes to take effect. + +## (Optional) Revert PostgreSQL settings to use a password and listen on an IP + +If you want to remove the PostgreSQL-specific settings and revert +to the defaults (using a socket instead), you can safely remove the following +lines from the `/etc/gitlab/gitlab.rb` file: + +```ruby +postgresql['sql_user_password'] = '...' +gitlab_rails['db_password'] = '...' +postgresql['listen_address'] = '...' +postgresql['md5_auth_cidr_addresses'] = ['...', '...'] +``` diff --git a/doc/administration/geo/replication/docker_registry.md b/doc/administration/geo/replication/docker_registry.md index bea6528dc9b..c34732cba67 100644 --- a/doc/administration/geo/replication/docker_registry.md +++ b/doc/administration/geo/replication/docker_registry.md @@ -18,7 +18,7 @@ Registry on the **primary** node, you can use the same storage for a **secondary Docker Registry as well. For more information, read the [Load balancing considerations](https://docs.docker.com/registry/deploying/#load-balancing-considerations) when deploying the Registry, and how to set up the storage driver for GitLab's -integrated [Container Registry](../../packages/container_registry.md#container-registry-storage-driver). +integrated [Container Registry](../../packages/container_registry.md#use-object-storage). ## Replicating Docker Registry diff --git a/doc/administration/geo/replication/external_database.md b/doc/administration/geo/replication/external_database.md index 491b3278ead..e85a82fa414 100644 --- a/doc/administration/geo/replication/external_database.md +++ b/doc/administration/geo/replication/external_database.md @@ -270,7 +270,8 @@ the tracking database on port 5432. query_exec "GRANT USAGE ON FOREIGN SERVER gitlab_secondary TO ${GEO_DB_USER};" ``` - NOTE: **Note:** The script template above uses `gitlab-psql` as it's intended to be executed from the Geo machine, + NOTE: **Note:** + The script template above uses `gitlab-psql` as it's intended to be executed from the Geo machine, but you can change it to `psql` and run it from any machine that has access to the database. We also recommend using `psql` for AWS RDS. diff --git a/doc/administration/geo/replication/geo_validation_tests.md b/doc/administration/geo/replication/geo_validation_tests.md index 7b186d15fae..0255e5c9883 100644 --- a/doc/administration/geo/replication/geo_validation_tests.md +++ b/doc/administration/geo/replication/geo_validation_tests.md @@ -16,20 +16,53 @@ This section contains a journal of recent validation tests and links to the rele The following are GitLab upgrade validation tests we performed. +### July 2020 + +[Upgrade Geo multi-node installation](https://gitlab.com/gitlab-org/gitlab/-/issues/225359): + +- Description: Tested upgrading from GitLab 12.10.12 to 13.0.10 package in a multi-node + configuration. As part of the issue to [Fix zero-downtime upgrade process/instructions for multi-node Geo deployments](https://gitlab.com/gitlab-org/gitlab/-/issues/22568), we monitored for downtime using the looping pipeline, HAProxy stats dashboards, and a script to log readiness status on both nodes. +- Outcome: Partial success because we observed downtime during the upgrade of the primary and secondary sites. +- Follow up issues/actions: + - [Investigate why `reconfigure` and `hup` cause downtime on multi-node Geo deployments](https://gitlab.com/gitlab-org/gitlab/-/issues/228898) + - [Geo multi-node deployment upgrade: investigate order when upgrading non-deploy nodes](https://gitlab.com/gitlab-org/gitlab/-/issues/228954) + +### June 2020 + +[Upgrade Geo multi-node installation](https://gitlab.com/gitlab-org/gitlab/-/issues/223284): + +- Description: Tested upgrading from GitLab 12.9.10 to 12.10.12 package in a multi-node + configuration. Monitored for downtime using the looping pipeline and HAProxy stats dashboards. +- Outcome: Partial success because we observed downtime during the upgrade of the primary and secondary sites. +- Follow up issues/actions: + - [Fix zero-downtime upgrade process/instructions for multi-node Geo deployments](https://gitlab.com/gitlab-org/gitlab/-/issues/225684) + - [Geo:check Rake task: Exclude AuthorizedKeysCommand check if node not running Puma/Unicorn](https://gitlab.com/gitlab-org/gitlab/-/issues/225454) + - [Update instructions in the next upgrade issue to include monitoring HAProxy dashboards](https://gitlab.com/gitlab-org/gitlab/-/issues/225359) + +[Upgrade Geo multi-node installation](https://gitlab.com/gitlab-org/gitlab/-/issues/208104): + +- Description: Tested upgrading from GitLab 12.8.1 to 12.9.10 package in a multi-node + configuration. +- Outcome: Partial success because we did not run the looping pipeline during the demo to validate + zero-downtime. +- Follow up issues: + - [Clarify hup Puma/Unicorn should include deploy node](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5460) + - [Investigate MR creation failure after upgrade to 12.9.10](https://gitlab.com/gitlab-org/gitlab/-/issues/223282) Closed as false positive. + ### February 2020 -[Upgrade Geo multi-server installation](https://gitlab.com/gitlab-org/gitlab/-/issues/201837): +[Upgrade Geo multi-node installation](https://gitlab.com/gitlab-org/gitlab/-/issues/201837): -- Description: Tested upgrading from GitLab 12.7.5 to the latest GitLab 12.8 package in a multi-server +- Description: Tested upgrading from GitLab 12.7.5 to the latest GitLab 12.8 package in a multi-node configuration. - Outcome: Partial success because we did not run the looping pipeline during the demo to monitor downtime. ### January 2020 -[Upgrade Geo multi-server installation](https://gitlab.com/gitlab-org/gitlab/-/issues/200085): +[Upgrade Geo multi-node installation](https://gitlab.com/gitlab-org/gitlab/-/issues/200085): -- Description: Tested upgrading from GitLab 12.6.x to the latest GitLab 12.7 package in a multi-server +- Description: Tested upgrading from GitLab 12.6.x to the latest GitLab 12.7 package in a multi-node configuration. - Outcome: Upgrade test was successful. - Follow up issues: @@ -37,16 +70,16 @@ The following are GitLab upgrade validation tests we performed. - [Add more logging to Geo end-to-end tests](https://gitlab.com/gitlab-org/gitlab/-/issues/201830). - [Excess service restarts during zero-downtime upgrade](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5047). -[Upgrade Geo multi-server installation](https://gitlab.com/gitlab-org/gitlab/-/issues/199836): +[Upgrade Geo multi-node installation](https://gitlab.com/gitlab-org/gitlab/-/issues/199836): -- Description: Tested upgrading from GitLab 12.5.7 to GitLab 12.6.6 in a multi-server configuration. +- Description: Tested upgrading from GitLab 12.5.7 to GitLab 12.6.6 in a multi-node configuration. - Outcome: Upgrade test was successful. - Follow up issue: [Update documentation for zero-downtime upgrades to ensure deploy node it not in use](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5046). -[Upgrade Geo multi-server installation](https://gitlab.com/gitlab-org/gitlab/-/issues/37044): +[Upgrade Geo multi-node installation](https://gitlab.com/gitlab-org/gitlab/-/issues/37044): -- Description: Tested upgrading from GitLab 12.4.x to the latest GitLab 12.5 package in a multi-server +- Description: Tested upgrading from GitLab 12.4.x to the latest GitLab 12.5 package in a multi-node configuration. - Outcome: Upgrade test was successful. - Follow up issues: @@ -55,17 +88,17 @@ The following are GitLab upgrade validation tests we performed. ### October 2019 -[Upgrade Geo multi-server installation](https://gitlab.com/gitlab-org/gitlab/-/issues/35262): +[Upgrade Geo multi-node installation](https://gitlab.com/gitlab-org/gitlab/-/issues/35262): -- Description: Tested upgrading from GitLab 12.3.5 to GitLab 12.4.1 in a multi-server configuration. +- Description: Tested upgrading from GitLab 12.3.5 to GitLab 12.4.1 in a multi-node configuration. - Outcome: Upgrade test was successful. -[Upgrade Geo multi-server installation](https://gitlab.com/gitlab-org/gitlab/-/issues/32437): +[Upgrade Geo multi-node installation](https://gitlab.com/gitlab-org/gitlab/-/issues/32437): - Description: Tested upgrading from GitLab 12.2.8 to GitLab 12.3.5. - Outcome: Upgrade test was successful. -[Upgrade Geo multi-server installation](https://gitlab.com/gitlab-org/gitlab/-/issues/32435): +[Upgrade Geo multi-node installation](https://gitlab.com/gitlab-org/gitlab/-/issues/32435): - Description: Tested upgrading from GitLab 12.1.9 to GitLab 12.2.8. - Outcome: Partial success due to possible misconfiguration issues. @@ -80,7 +113,7 @@ The following are PostgreSQL upgrade validation tests we performed. - Description: Prior to making PostgreSQL 11 the default version of PostgreSQL in GitLab 12.10, we tested upgrading to PostgreSQL 11 in Geo deployments on GitLab 12.9. -- Outcome: Partially successful. Issues were discovered in multi-server configurations with a separate +- Outcome: Partially successful. Issues were discovered in multi-node configurations with a separate tracking database and concerns were raised about allowing automatic upgrades when Geo enabled. - Follow up issues: - [`replicate-geo-database` incorrectly tries to back up repositories](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5241). @@ -102,6 +135,6 @@ The following are PostgreSQL upgrade validation tests we performed. various upgrade scenarios from GitLab 11.11.5 through to GitLab 12.1.8. - Outcome: Multiple issues were found when upgrading and addressed in follow-up issues. - Follow up issues: - - [`gitlab-ctl` reconfigure fails on Redis node in multi-server Geo setup](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/4706). - - [Geo multi-server upgrade from 12.0.9 to 12.1.9 does not upgrade PostgreSQL](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/4705). - - [Refresh foreign tables fails on app server in multi-server setup after upgrade to 12.1.9](https://gitlab.com/gitlab-org/gitlab/-/issues/32119). + - [`gitlab-ctl` reconfigure fails on Redis node in multi-node Geo setup](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/4706). + - [Geo multi-node upgrade from 12.0.9 to 12.1.9 does not upgrade PostgreSQL](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/4705). + - [Refresh foreign tables fails on app server in multi-node setup after upgrade to 12.1.9](https://gitlab.com/gitlab-org/gitlab/-/issues/32119). diff --git a/doc/administration/geo/replication/index.md b/doc/administration/geo/replication/index.md index 5b4b476bfa8..51c831abaf3 100644 --- a/doc/administration/geo/replication/index.md +++ b/doc/administration/geo/replication/index.md @@ -9,7 +9,7 @@ type: howto > - Introduced in GitLab Enterprise Edition 8.9. > - Using Geo in combination with -> [multi-server architectures](../../reference_architectures/index.md) +> [multi-node architectures](../../reference_architectures/index.md) > is considered **Generally Available** (GA) in > [GitLab Premium](https://about.gitlab.com/pricing/) 10.4. @@ -213,9 +213,29 @@ For information on configuring Geo, see [Geo configuration](configuration.md). For information on how to update your Geo nodes to the latest GitLab version, see [Updating the Geo nodes](updating_the_geo_nodes.md). -### Configuring Geo for multiple servers +### Pausing and resuming replication -For information on configuring Geo for multiple servers, see [Geo for multiple servers](multiple_servers.md). +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/35913) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.2. + +In some circumstances, like during [upgrades](updating_the_geo_nodes.md) or a [planned failover](../disaster_recovery/planned_failover.md), it is desirable to pause replication between the primary and secondary. + +Pausing and resuming replication is done via a command line tool from the secondary node. + +**To Pause: (from secondary)** + +```shell +gitlab-ctl geo-replication-pause +``` + +**To Resume: (from secondary)** + +```shell +gitlab-ctl geo-replication-resume +``` + +### Configuring Geo for multiple nodes + +For information on configuring Geo for multiple nodes, see [Geo for multiple servers](multiple_servers.md). ### Configuring Geo with Object Storage @@ -245,6 +265,10 @@ For an example of how to set up a location-aware Git remote URL with AWS Route53 For more information on removing a Geo node, see [Removing **secondary** Geo nodes](remove_geo_node.md). +## Disable Geo + +To find out how to disable Geo, see [Disabling Geo](disable_geo.md). + ## Current limitations CAUTION: **Caution:** diff --git a/doc/administration/geo/replication/location_aware_git_url.md b/doc/administration/geo/replication/location_aware_git_url.md index 49c83ee1718..8b086e3ff5f 100644 --- a/doc/administration/geo/replication/location_aware_git_url.md +++ b/doc/administration/geo/replication/location_aware_git_url.md @@ -18,7 +18,7 @@ Though these instructions use [AWS Route53](https://aws.amazon.com/route53/), other services such as [Cloudflare](https://www.cloudflare.com/) could be used as well. -NOTE: **Note** +NOTE: **Note:** You can also use a load balancer to distribute web UI or API traffic to [multiple Geo **secondary** nodes](../../../user/admin_area/geo_nodes.md#multiple-secondary-nodes-behind-a-load-balancer). Importantly, the **primary** node cannot yet be included. See the feature request diff --git a/doc/administration/geo/replication/multiple_servers.md b/doc/administration/geo/replication/multiple_servers.md index 2747aaeb105..d8f04e07fb0 100644 --- a/doc/administration/geo/replication/multiple_servers.md +++ b/doc/administration/geo/replication/multiple_servers.md @@ -5,15 +5,15 @@ info: To determine the technical writer assigned to the Stage/Group associated w type: howto --- -# Geo for multiple servers **(PREMIUM ONLY)** +# Geo for multiple nodes **(PREMIUM ONLY)** This document describes a minimal reference architecture for running Geo -in a multi-server configuration. If your multi-server setup differs from the one +in a multi-node configuration. If your multi-node setup differs from the one described, it is possible to adapt these instructions to your needs. ## Architecture overview - + _[diagram source - GitLab employees only](https://docs.google.com/drawings/d/1z0VlizKiLNXVVVaERFwgsIOuEgjcUqDTWPdQYsE7Z4c/edit)_ @@ -30,36 +30,36 @@ The only external way to access the two Geo deployments is by HTTPS at NOTE: **Note:** The **primary** and **secondary** Geo deployments must be able to communicate to each other over HTTPS. -## Redis and PostgreSQL for multiple servers +## Redis and PostgreSQL for multiple nodes Geo supports: -- Redis and PostgreSQL on the **primary** node configured for multiple servers. -- Redis on **secondary** nodes configured for multiple servers. +- Redis and PostgreSQL on the **primary** node configured for multiple nodes. +- Redis on **secondary** nodes configured for multiple nodes. NOTE: **Note:** -Support for PostgreSQL on **secondary** nodes in multi-server configuration +Support for PostgreSQL on **secondary** nodes in multi-node configuration [is planned](https://gitlab.com/groups/gitlab-org/-/epics/2536). Because of the additional complexity involved in setting up this configuration -for PostgreSQL and Redis, it is not covered by this Geo multi-server documentation. +for PostgreSQL and Redis, it is not covered by this Geo multi-node documentation. -For more information about setting up a multi-server PostgreSQL cluster and Redis cluster using the omnibus package see the multi-server documentation for +For more information about setting up a multi-node PostgreSQL cluster and Redis cluster using the omnibus package see the multi-node documentation for [PostgreSQL](../../postgresql/replication_and_failover.md) and -[Redis](../../high_availability/redis.md), respectively. +[Redis](../../redis/replication_and_failover.md), respectively. NOTE: **Note:** It is possible to use cloud hosted services for PostgreSQL and Redis, but this is beyond the scope of this document. -## Prerequisites: Two working GitLab multi-server clusters +## Prerequisites: Two working GitLab multi-node clusters One cluster will serve as the **primary** node. Use the -[GitLab multi-server documentation](../../reference_architectures/index.md) to set this up. If +[GitLab multi-node documentation](../../reference_architectures/index.md) to set this up. If you already have a working GitLab instance that is in-use, it can be used as a **primary**. The second cluster will serve as the **secondary** node. Again, use the -[GitLab multi-server documentation](../../reference_architectures/index.md) to set this up. +[GitLab multi-node documentation](../../reference_architectures/index.md) to set this up. It's a good idea to log in and test it, however, note that its data will be wiped out as part of the process of replicating from the **primary**. @@ -90,12 +90,13 @@ The following steps enable a GitLab cluster to serve as the **primary** node. After making these changes, [reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure) so the changes take effect. -NOTE: **Note:** PostgreSQL and Redis should have already been disabled on the +NOTE: **Note:** +PostgreSQL and Redis should have already been disabled on the application servers, and connections from the application servers to those -services on the backend servers configured, during normal GitLab multi-server set up. See -multi-server configuration documentation for +services on the backend servers configured, during normal GitLab multi-node set up. See +multi-node configuration documentation for [PostgreSQL](../../postgresql/replication_and_failover.md#configuring-the-application-nodes) -and [Redis](../../high_availability/redis.md#example-configuration-for-the-gitlab-application). +and [Redis](../../redis/replication_and_failover.md#example-configuration-for-the-gitlab-application). ### Step 2: Configure the **primary** database @@ -110,7 +111,7 @@ and [Redis](../../high_availability/redis.md#example-configuration-for-the-gitla ## Configure a **secondary** node -A **secondary** cluster is similar to any other GitLab multi-server cluster, with two +A **secondary** cluster is similar to any other GitLab multi-node cluster, with two major differences: - The main PostgreSQL database is a read-only replica of the **primary** node's @@ -119,8 +120,8 @@ major differences: called the "tracking database", which tracks the synchronization state of various resources. -Therefore, we will set up the multi-server components one-by-one, and include deviations -from the normal multi-server setup. However, we highly recommend first configuring a +Therefore, we will set up the multi-node components one-by-one, and include deviations +from the normal multi-node setup. However, we highly recommend first configuring a brand-new cluster as if it were not part of a Geo setup so that it can be tested and verified as a working cluster. And only then should it be modified for use as a Geo **secondary**. This helps to separate problems that are related @@ -128,10 +129,10 @@ and are not related to Geo setup. ### Step 1: Configure the Redis and Gitaly services on the **secondary** node -Configure the following services, again using the non-Geo multi-server +Configure the following services, again using the non-Geo multi-node documentation: -- [Configuring Redis for GitLab](../../high_availability/redis.md) for multiple servers. +- [Configuring Redis for GitLab](../../redis/replication_and_failover.md#example-configuration-for-the-gitlab-application) for multiple nodes. - [Gitaly](../../high_availability/gitaly.md), which will store data that is synchronized from the **primary** node. @@ -141,8 +142,9 @@ recommended. ### Step 2: Configure the main read-only replica PostgreSQL database on the **secondary** node -NOTE: **Note:** The following documentation assumes the database will be run on -a single node only. Multi-server PostgreSQL on **secondary** nodes is +NOTE: **Note:** +The following documentation assumes the database will be run on +a single node only. Multi-node PostgreSQL on **secondary** nodes is [not currently supported](https://gitlab.com/groups/gitlab-org/-/epics/2536). Configure the [**secondary** database](database.md) as a read-only replica of @@ -206,7 +208,8 @@ If using an external PostgreSQL instance, refer also to ### Step 3: Configure the tracking database on the **secondary** node -NOTE: **Note:** This documentation assumes the tracking database will be run on +NOTE: **Note:** +This documentation assumes the tracking database will be run on only a single machine, rather than as a PostgreSQL cluster. Configure the tracking database. @@ -282,7 +285,7 @@ application services. These services are enabled selectively in the configuration. Configure the application servers following -[Configuring GitLab for multiple servers](../../high_availability/gitlab.md), then make the +[Configuring GitLab for multiple nodes](../../high_availability/gitlab.md), then make the following modifications: 1. Edit `/etc/gitlab/gitlab.rb` on each application server in the **secondary** @@ -370,13 +373,13 @@ application servers. In this topology, a load balancer is required at each geographic location to route traffic to the application servers. -See [Load Balancer for GitLab with multiple servers](../../high_availability/load_balancer.md) for +See [Load Balancer for GitLab with multiple nodes](../../high_availability/load_balancer.md) for more information. ### Step 6: Configure the backend application servers on the **secondary** node The minimal reference architecture diagram above shows all application services -running together on the same machines. However, for multiple servers we +running together on the same machines. However, for multiple nodes we [strongly recommend running all services separately](../../reference_architectures/index.md). For example, a Sidekiq server could be configured similarly to the frontend diff --git a/doc/administration/geo/replication/troubleshooting.md b/doc/administration/geo/replication/troubleshooting.md index b03a2dae971..c2f8aa35d2d 100644 --- a/doc/administration/geo/replication/troubleshooting.md +++ b/doc/administration/geo/replication/troubleshooting.md @@ -5,7 +5,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w type: howto --- -# Geo Troubleshooting **(PREMIUM ONLY)** +# Troubleshooting Geo **(PREMIUM ONLY)** Setting up Geo requires careful attention to details and sometimes it's easy to miss a step. @@ -452,13 +452,13 @@ to start again from scratch, there are a few steps that can help you: chown git:git /var/opt/gitlab/git-data/repositories ``` - TIP: **Tip** + TIP: **Tip:** You may want to remove the `/var/opt/gitlab/git-data/repositories.old` in the future as soon as you confirmed that you don't need it anymore, to save disk space. 1. _(Optional)_ Rename other data folders and create new ones - CAUTION: **Caution**: + CAUTION: **Caution:** You may still have files on the **secondary** node that have been removed from **primary** node but removal have not been reflected. If you skip this step, they will never be removed from this Geo node. @@ -701,7 +701,8 @@ To check the configuration: Description | ``` - NOTE: **Note:** Pay particular attention to the host and port under + NOTE: **Note:** + Pay particular attention to the host and port under FDW options. That configuration should point to the Geo secondary database. diff --git a/doc/administration/geo/replication/updating_the_geo_nodes.md b/doc/administration/geo/replication/updating_the_geo_nodes.md index 6c2778ad0fe..c0b1bc6688c 100644 --- a/doc/administration/geo/replication/updating_the_geo_nodes.md +++ b/doc/administration/geo/replication/updating_the_geo_nodes.md @@ -36,15 +36,18 @@ different steps. ## General update steps -NOTE: **Note:** These general update steps are not intended for [high-availability deployments](https://docs.gitlab.com/omnibus/update/README.html#multi-node--ha-deployment), and will cause downtime. If you want to avoid downtime, consider using [zero downtime updates](https://docs.gitlab.com/omnibus/update/README.html#zero-downtime-updates). +NOTE: **Note:** +These general update steps are not intended for [high-availability deployments](https://docs.gitlab.com/omnibus/update/README.html#multi-node--ha-deployment), and will cause downtime. If you want to avoid downtime, consider using [zero downtime updates](https://docs.gitlab.com/omnibus/update/README.html#zero-downtime-updates). To update the Geo nodes when a new GitLab version is released, update **primary** and all **secondary** nodes: +1. **Optional:** [Pause replication on each **secondary** node.](./index.md#pausing-and-resuming-replication) 1. Log into the **primary** node. 1. [Update GitLab on the **primary** node using Omnibus](https://docs.gitlab.com/omnibus/update/README.html). 1. Log into each **secondary** node. 1. [Update GitLab on each **secondary** node using Omnibus](https://docs.gitlab.com/omnibus/update/README.html). +1. If you paused replication in step 1, [resume replication on each **secondary**](./index.md#pausing-and-resuming-replication) 1. [Test](#check-status-after-updating) **primary** and **secondary** nodes, and check version in each. ### Check status after updating diff --git a/doc/administration/git_annex.md b/doc/administration/git_annex.md index c4c38f8e683..ed6218ede91 100644 --- a/doc/administration/git_annex.md +++ b/doc/administration/git_annex.md @@ -4,9 +4,9 @@ disqus_identifier: 'https://docs.gitlab.com/ee/workflow/git_annex.html' # Git annex -> **Warning:** GitLab has [completely -removed](https://gitlab.com/gitlab-org/gitlab/-/issues/1648) in GitLab 9.0 (2017/03/22). -Read through the [migration guide from git-annex to Git LFS](../topics/git/lfs/migrate_from_git_annex_to_git_lfs.md). +CAUTION: **Warning:** +[Git Annex support was removed](https://gitlab.com/gitlab-org/gitlab/-/issues/1648) +in GitLab 9.0. Read through the [migration guide from git-annex to Git LFS](../topics/git/lfs/migrate_from_git_annex_to_git_lfs.md). The biggest limitation of Git, compared to some older centralized version control systems has been the maximum size of the repositories. @@ -198,7 +198,7 @@ can cause `git-annex` to raise unpredicted warnings and errors. Consult the [Annex upgrade page](https://git-annex.branchable.com/upgrades/) for more information about the differences between versions. You can find out which version is installed -on your server by navigating to <https://pkgs.org/download/git-annex> and +on your server by navigating to `https://pkgs.org/download/git-annex` and searching for your distribution. Although there is no general guide for `git-annex` errors, there are a few tips diff --git a/doc/administration/gitaly/img/praefect_storage_v12_10.png b/doc/administration/gitaly/img/praefect_storage_v12_10.png Binary files differdeleted file mode 100644 index f60a56fa1fb..00000000000 --- a/doc/administration/gitaly/img/praefect_storage_v12_10.png +++ /dev/null diff --git a/doc/administration/gitaly/index.md b/doc/administration/gitaly/index.md index 1469ed64004..057d0559c14 100644 --- a/doc/administration/gitaly/index.md +++ b/doc/administration/gitaly/index.md @@ -19,12 +19,13 @@ In the Gitaly documentation: - [GitLab Shell](https://gitlab.com/gitlab-org/gitlab-shell). - [GitLab Workhorse](https://gitlab.com/gitlab-org/gitlab-workhorse). -GitLab end users do not have direct access to Gitaly. +GitLab end users do not have direct access to Gitaly. Gitaly only manages Git +repository access for GitLab. Other types of GitLab data aren't accessed using Gitaly. CAUTION: **Caution:** -From GitLab 13.0, using NFS for Git repositories is deprecated. In GitLab 14.0, -support for NFS for Git repositories is scheduled to be removed. Upgrade to -[Gitaly Cluster](praefect.md) as soon as possible. +From GitLab 13.0, Gitaly support for NFS is deprecated. In GitLab 14.0, Gitaly support +for NFS is scheduled to be removed. Upgrade to [Gitaly Cluster](praefect.md) as soon as +possible. ## Architecture @@ -49,6 +50,12 @@ To change Gitaly settings: 1. Edit `/home/git/gitaly/config.toml` and add or change the [Gitaly settings](https://gitlab.com/gitlab-org/gitaly/blob/master/config.toml.example). 1. Save the file and [restart GitLab](../restart_gitlab.md#installations-from-source). +The following configuration options are also available: + +- Enabling [TLS support](#enable-tls-support). +- Configuring the [number of `gitaly-ruby` workers](#configure-number-of-gitaly-ruby-workers). +- Limiting [RPC concurrency](#limit-rpc-concurrency). + ## Run Gitaly on its own server By default, Gitaly is run on the same server as Gitaly clients and is @@ -72,6 +79,7 @@ The process for setting up Gitaly on its own server is: 1. [Configure authentication](#configure-authentication). 1. [Configure Gitaly servers](#configure-gitaly-servers). 1. [Configure Gitaly clients](#configure-gitaly-clients). +1. [Disable Gitaly where not required](#disable-gitaly-where-not-required-optional) (optional). When running Gitaly on its own server, note the following regarding GitLab versions: @@ -116,7 +124,7 @@ The following list depicts the network architecture of Gitaly: DANGER: **Danger:** Gitaly servers must not be exposed to the public internet as Gitaly's network traffic is unencrypted by default. The use of firewall is highly recommended to restrict access to the Gitaly server. -Another option is to [use TLS](#tls-support). +Another option is to [use TLS](#enable-tls-support). In the following sections, we describe how to configure two Gitaly servers with secret token `abc123secret`: @@ -380,20 +388,10 @@ Gitaly makes the following assumptions: clients, and that Gitaly server can read and write to `/mnt/gitlab/storage2`. - Your `gitaly1.internal` and `gitaly2.internal` Gitaly servers can reach each other. -Note you can't a use mixed setup, with at least one of your Gitaly servers configured as a local -server with the `path` setting provided. This is because other Gitaly instances can't communicate -with it. The following setup is _incorrect_, because: - -- You must replace `path` with `gitaly_address` containing a proper value. -- The address must be reachable from the other two addresses provided. - -```ruby -git_data_dirs({ - 'default' => { 'gitaly_address' => 'tcp://gitaly1.internal:8075' }, - 'storage1' => { 'path' => '/var/opt/gitlab/git-data' }, - 'storage2' => { 'gitaly_address' => 'tcp://gitaly2.internal:8075' }, -}) -``` +You can't define Gitaly servers with some as a local Gitaly server +(without `gitaly_address`) and some as remote +server (with `gitaly_address`) unless you setup with special +[mixed configuration](#mixed-configuration). **For Omnibus GitLab** @@ -457,15 +455,47 @@ If you have [server hooks](../server_hooks.md) configured, either per repository must move these to the Gitaly servers. If you have multiple Gitaly servers, copy your server hooks to all Gitaly servers. -### Disabling the Gitaly service in a cluster environment +#### Mixed configuration + +GitLab can reside on the same server as one of many Gitaly servers, but doesn't support +configuration that mixes local and remote configuration. The following setup is incorrect, because: + +- All addresses must be reachable from the other Gitaly servers. +- `storage1` will be assigned a Unix socket for `gitaly_address` which is + invalid for some of the Gitaly servers. + +```ruby +git_data_dirs({ + 'default' => { 'gitaly_address' => 'tcp://gitaly1.internal:8075' }, + 'storage1' => { 'path' => '/mnt/gitlab/git-data' }, + 'storage2' => { 'gitaly_address' => 'tcp://gitaly2.internal:8075' }, +}) +``` + +To combine local and remote Gitaly servers, use an external address for the local Gitaly server. For +example: + +```ruby +git_data_dirs({ + 'default' => { 'gitaly_address' => 'tcp://gitaly1.internal:8075' }, + # Address of the GitLab server that has Gitaly running on it + 'storage1' => { 'gitaly_address' => 'tcp://gitlab.internal:8075', 'path' => '/mnt/gitlab/git-data' }, + 'storage2' => { 'gitaly_address' => 'tcp://gitaly2.internal:8075' }, +}) +``` + +`path` can only be included for storage shards on the local Gitaly server. +If it's excluded, default Git storage directory will be used for that storage shard. -If you are running Gitaly [as a remote -service](#run-gitaly-on-its-own-server) you may want to disable -the local Gitaly service that runs on your GitLab server by default. -Disabling Gitaly only makes sense when you run GitLab in a custom -cluster configuration, where different services run on different -machines. Disabling Gitaly on all machines in the cluster is not a -valid configuration. +### Disable Gitaly where not required (optional) + +If you are running Gitaly [as a remote service](#run-gitaly-on-its-own-server) you may want to +disable the local Gitaly service that runs on your GitLab server by default, leaving it only running +where required. + +Disabling Gitaly on the GitLab instance only makes sense when you run GitLab in a custom cluster configuration, where +Gitaly runs on a separate machine from the GitLab instance. Disabling Gitaly on all machines in the cluster is not +a valid configuration (some machines much act as Gitaly servers). To disable Gitaly on a GitLab server: @@ -489,43 +519,44 @@ To disable Gitaly on a GitLab server: 1. Save the file and [restart GitLab](../restart_gitlab.md#installations-from-source). -## TLS support +## Enable TLS support > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/22602) in GitLab 11.8. -Gitaly supports TLS encryption. To be able to communicate -with a Gitaly instance that listens for secure connections you will need to use `tls://` URL -scheme in the `gitaly_address` of the corresponding storage entry in the GitLab configuration. +Gitaly supports TLS encryption. To communicate with a Gitaly instance that listens for secure +connections, you must use `tls://` URL scheme in the `gitaly_address` of the corresponding +storage entry in the GitLab configuration. -You will need to bring your own certificates as this isn't provided automatically. -The certificate corresponding to each Gitaly server will need to be installed -on that Gitaly server. +You must supply your own certificates as this isn't provided automatically. The certificate +corresponding to each Gitaly server must be installed on that Gitaly server. -Additionally the certificate, or its certificate authority, must be installed on all Gitaly servers -(including the Gitaly server using the certificate) and on all Gitaly clients -that communicate with it following the procedure described in -[GitLab custom certificate configuration](https://docs.gitlab.com/omnibus/settings/ssl.html#install-custom-public-certificates) (and repeated below). +Additionally, the certificate (or its certificate authority) must be installed on all: -NOTE: **Note** -The certificate must specify the address you use to access the -Gitaly server. If you are addressing the Gitaly server by a hostname, you can -either use the Common Name field for this, or add it as a Subject Alternative -Name. If you are addressing the Gitaly server by its IP address, you must add it -as a Subject Alternative Name to the certificate. -[gRPC does not support using an IP address as Common Name in a certificate](https://github.com/grpc/grpc/issues/2691). +- Gitaly servers, including the Gitaly server using the certificate. +- Gitaly clients that communicate with it. -NOTE: **Note:** -It is possible to configure Gitaly servers with both an -unencrypted listening address `listen_addr` and an encrypted listening -address `tls_listen_addr` at the same time. This allows you to do a -gradual transition from unencrypted to encrypted traffic, if necessary. +The process is documented in the +[GitLab custom certificate configuration](https://docs.gitlab.com/omnibus/settings/ssl.html#install-custom-public-certificates) +and repeated below. + +Note the following: + +- The certificate must specify the address you use to access the Gitaly server. If you are: + - Addressing the Gitaly server by a hostname, you can either use the Common Name field for this, + or add it as a Subject Alternative Name. + - Addressing the Gitaly server by its IP address, you must add it as a Subject Alternative Name to + the certificate. [gRPC does not support using an IP address as Common Name in a certificate](https://github.com/grpc/grpc/issues/2691). +- You can configure Gitaly servers with both an unencrypted listening address `listen_addr` and an + encrypted listening address `tls_listen_addr` at the same time. This allows you to gradually + transition from unencrypted to encrypted traffic if necessary. To configure Gitaly with TLS: **For Omnibus GitLab** 1. Create certificates for Gitaly servers. -1. On the Gitaly clients, copy the certificates, or their certificate authority, into the `/etc/gitlab/trusted-certs`: +1. On the Gitaly clients, copy the certificates (or their certificate authority) into + `/etc/gitlab/trusted-certs`: ```shell sudo cp cert.pem /etc/gitlab/trusted-certs/ @@ -542,7 +573,8 @@ To configure Gitaly with TLS: ``` 1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). -1. On the Gitaly servers, create the `/etc/gitlab/ssl` directory and copy your key and certificate there: +1. On the Gitaly servers, create the `/etc/gitlab/ssl` directory and copy your key and certificate + there: ```shell sudo mkdir -p /etc/gitlab/ssl @@ -551,8 +583,9 @@ To configure Gitaly with TLS: sudo chmod 644 key.pem cert.pem ``` -1. Copy all Gitaly server certificates, or their certificate authority, to `/etc/gitlab/trusted-certs` so Gitaly server will trust the certificate when -calling into itself or other Gitaly servers: +1. Copy all Gitaly server certificates (or their certificate authority) to + `/etc/gitlab/trusted-certs` so that Gitaly servers will trust the certificate when calling into themselves + or other Gitaly servers: ```shell sudo cp cert1.pem cert2.pem /etc/gitlab/trusted-certs/ @@ -572,10 +605,13 @@ calling into itself or other Gitaly servers: ``` 1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). -1. (Optional) After [verifying that all Gitaly traffic is being served over TLS](#observe-type-of-gitaly-connections), - you can improve security by disabling non-TLS connections by commenting out - or deleting `gitaly['listen_addr']` in `/etc/gitlab/gitlab.rb`, saving the file, - and [reconfiguring GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). +1. Verify Gitaly traffic is being served over TLS by + [observing the types of Gitaly connections](#observe-type-of-gitaly-connections). +1. (Optional) Improve security by: + 1. Disabling non-TLS connections by commenting out or deleting `gitaly['listen_addr']` in + `/etc/gitlab/gitlab.rb`. + 1. Saving the file. + 1. [Reconfiguring GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). **For installations from source** @@ -605,9 +641,9 @@ calling into itself or other Gitaly servers: ``` NOTE: **Note:** - `/some/dummy/path` should be set to a local folder that exists, however no - data will be stored in this folder. This will no longer be necessary after - [this issue](https://gitlab.com/gitlab-org/gitaly/-/issues/1282) is resolved. + `/some/dummy/path` should be set to a local folder that exists, however no data will be stored + in this folder. This will no longer be necessary after + [Gitaly issue #1282](https://gitlab.com/gitlab-org/gitaly/-/issues/1282) is resolved. 1. Save the file and [restart GitLab](../restart_gitlab.md#installations-from-source). 1. On the Gitaly servers, create or edit `/etc/default/gitlab` and add: @@ -625,7 +661,9 @@ calling into itself or other Gitaly servers: sudo chmod 644 key.pem cert.pem ``` -1. Copy all Gitaly server certificates, or their certificate authority, to the system trusted certificates so Gitaly server will trust the certificate when calling into itself or other Gitaly servers. +1. Copy all Gitaly server certificates (or their certificate authority) to the system trusted + certificates folder so Gitaly server will trust the certificate when calling into itself or other Gitaly + servers. ```shell sudo cp cert.pem /usr/local/share/ca-certificates/gitaly.crt @@ -643,15 +681,18 @@ calling into itself or other Gitaly servers: ``` 1. Save the file and [restart GitLab](../restart_gitlab.md#installations-from-source). -1. (Optional) After [verifying that all Gitaly traffic is being served over TLS](#observe-type-of-gitaly-connections), - you can improve security by disabling non-TLS connections by commenting out - or deleting `listen_addr` in `/home/git/gitaly/config.toml`, saving the file, - and [restarting GitLab](../restart_gitlab.md#installations-from-source). +1. Verify Gitaly traffic is being served over TLS by + [observing the types of Gitaly connections](#observe-type-of-gitaly-connections). +1. (Optional) Improve security by: + 1. Disabling non-TLS connections by commenting out or deleting `listen_addr` in + `/home/git/gitaly/config.toml`. + 1. Saving the file. + 1. [Restarting GitLab](../restart_gitlab.md#installations-from-source). ### Observe type of Gitaly connections -To observe what type of connections are actually being used in a -production environment you can use the following Prometheus query: +[Prometheus](../monitoring/prometheus/index.md) can be used observe what type of connections Gitaly +is serving a production environment. Use the following Prometheus query: ```prometheus sum(rate(gitaly_connections_total[5m])) by (type) @@ -660,24 +701,26 @@ sum(rate(gitaly_connections_total[5m])) by (type) ## `gitaly-ruby` Gitaly was developed to replace the Ruby application code in GitLab. -In order to save time and/or avoid the risk of rewriting existing -application logic, in some cases we chose to copy some application code -from GitLab into Gitaly almost as-is. To be able to run that code, -`gitaly-ruby` was created, which is a "sidecar" process for the main Gitaly Go -process. Some examples of things that are implemented in `gitaly-ruby` are -RPCs that deal with wikis, and RPCs that create commits on behalf of -a user, such as merge commits. -### Number of `gitaly-ruby` workers +To save time and avoid the risk of rewriting existing application logic, we chose to copy some +application code from GitLab into Gitaly. + +To be able to run that code, `gitaly-ruby` was created, which is a "sidecar" process for the main +Gitaly Go process. Some examples of things that are implemented in `gitaly-ruby` are: + +- RPCs that deal with wikis. +- RPCs that create commits on behalf of a user, such as merge commits. + +### Configure number of `gitaly-ruby` workers -`gitaly-ruby` has much less capacity than Gitaly itself. If your Gitaly -server has to handle a lot of requests, the default setting of having -just one active `gitaly-ruby` sidecar might not be enough. If you see -`ResourceExhausted` errors from Gitaly, it's very likely that you have not -enough `gitaly-ruby` capacity. +`gitaly-ruby` has much less capacity than Gitaly implemented in Go. If your Gitaly server has to handle lots of +requests, the default setting of having just one active `gitaly-ruby` sidecar might not be enough. -You can increase the number of `gitaly-ruby` processes on your Gitaly -server with the following settings. +If you see `ResourceExhausted` errors from Gitaly, it's very likely that you have not enough +`gitaly-ruby` capacity. + +You can increase the number of `gitaly-ruby` processes on your Gitaly server with the following +settings: **For Omnibus GitLab** @@ -702,13 +745,16 @@ server with the following settings. 1. Save the file and [restart GitLab](../restart_gitlab.md#installations-from-source). -## Limiting RPC concurrency +## Limit RPC concurrency + +Clone traffic can put a large strain on your Gitaly service. The bulk of the work gets done in the +either of the following RPCs: + +- `SSHUploadPack` (for Git SSH). +- `PostUploadPack` (for Git HTTP). -It can happen that CI clone traffic puts a large strain on your Gitaly -service. The bulk of the work gets done in the SSHUploadPack (for Git -SSH) and PostUploadPack (for Git HTTP) RPC's. To prevent such workloads -from overcrowding your Gitaly server you can set concurrency limits in -Gitaly's configuration file. +To prevent such workloads from overwhelming your Gitaly server, you can set concurrency limits in +Gitaly's configuration file. For example: ```ruby # in /etc/gitlab/gitlab.rb @@ -725,226 +771,250 @@ gitaly['concurrency'] = [ ] ``` -This will limit the number of in-flight RPC calls for the given RPC's. -The limit is applied per repository. In the example above, each on the -Gitaly server can have at most 20 simultaneous `PostUploadPack` calls in -flight, and the same for `SSHUploadPack`. If another request comes in for -a repository that has used up its 20 slots, that request will get -queued. +This limits the number of in-flight RPC calls for the given RPCs. The limit is applied per +repository. In the example above: + +- Each repository served by the Gitaly server can have at most 20 simultaneous `PostUploadPack` RPC + calls in flight, and the same for `SSHUploadPack`. +- If another request comes in for a repository that has used up its 20 slots, that request gets + queued. + +You can observe the behavior of this queue using the Gitaly logs and Prometheus: -You can observe the behavior of this queue via the Gitaly logs and via -Prometheus. In the Gitaly logs, you can look for the string (or -structured log field) `acquire_ms`. Messages that have this field are -reporting about the concurrency limiter. In Prometheus, look for the -`gitaly_rate_limiting_in_progress`, `gitaly_rate_limiting_queued` and -`gitaly_rate_limiting_seconds` metrics. +- In the Gitaly logs, look for the string (or structured log field) `acquire_ms`. Messages that have + this field are reporting about the concurrency limiter. +- In Prometheus, look for the following metrics: -The name of the Prometheus metric is not quite right because this is a -concurrency limiter, not a rate limiter. If a Gitaly client makes 1000 requests -in a row in a very short timespan, the concurrency will not exceed 1, -and this mechanism (the concurrency limiter) will do nothing. + - `gitaly_rate_limiting_in_progress`. + - `gitaly_rate_limiting_queued`. + - `gitaly_rate_limiting_seconds`. -## Rotating a Gitaly authentication token +NOTE: **Note:** +Though the name of the Prometheus metric contains `rate_limiting`, it is a concurrency limiter, not +a rate limiter. If a Gitaly client makes 1000 requests in a row very quickly, concurrency will not +exceed 1 and the concurrency limiter has no effect. + +## Rotate Gitaly authentication token + +Rotating credentials in a production environment often requires downtime, causes outages, or both. -Rotating credentials in a production environment often either requires -downtime, or causes outages, or both. If you are careful, though, you -*can* rotate Gitaly credentials without a service interruption. +However, you can rotate Gitaly credentials without a service interruption. Rotating a Gitaly +authentication token involves: -This procedure also works if you are running GitLab on a single server. -In that case, "Gitaly server" and "Gitaly client" refers to the same -machine. +- [Verifying authentication monitoring](#verify-authentication-monitoring). +- [Enabling "auth transitioning" mode](#enable-auth-transitioning-mode). +- [Updating Gitaly authentication tokens](#update-gitaly-authentication-token). +- [Ensuring there are no authentication failures](#ensure-there-are-no-authentication-failures). +- [Disabling "auth transitioning" mode](#disable-auth-transitioning-mode). +- [Verifying authentication is enforced](#verify-authentication-is-enforced). -### 1. Monitor current authentication behavior +This procedure also works if you are running GitLab on a single server. In that case, "Gitaly +server" and "Gitaly client" refers to the same machine. -Use Prometheus to see what the current authentication behavior of your -GitLab installation is. +### Verify authentication monitoring + +Before rotating a Gitaly authentication token, verify that you can monitor the authentication +behavior of your GitLab installation using Prometheus. Use the following Prometheus query: ```prometheus sum(rate(gitaly_authentications_total[5m])) by (enforced, status) ``` -In a system where authentication is configured correctly, and where you -have live traffic, you will see something like this: +In a system where authentication is configured correctly and where you have live traffic, you will +see something like this: ```prometheus {enforced="true",status="ok"} 4424.985419441742 ``` -There may also be other numbers with rate 0. We only care about the -non-zero numbers. +There may also be other numbers with rate 0. We only care about the non-zero numbers. -The only non-zero number should have `enforced="true",status="ok"`. If -you have other non-zero numbers, something is wrong in your -configuration. +The only non-zero number should have `enforced="true",status="ok"`. If you have other non-zero +numbers, something is wrong in your configuration. -The `status="ok"` number reflects your current request rate. In the example -above, Gitaly is handling about 4000 requests per second. +The `status="ok"` number reflects your current request rate. In the example above, Gitaly is +handling about 4000 requests per second. -Now you have established that you can monitor the Gitaly authentication -behavior of your GitLab installation. +Now that you have established that you can monitor the Gitaly authentication behavior of your GitLab +installation, you can begin the rest of the procedure. -### 2. Reconfigure all Gitaly servers to be in "auth transitioning" mode +### Enable "auth transitioning" mode -The second step is to temporarily disable authentication on the Gitaly servers. +Temporarily disable Gitaly authentication on the Gitaly servers by putting them into "auth +transitioning" mode as follows: ```ruby # in /etc/gitlab/gitlab.rb gitaly['auth_transitioning'] = true ``` -After you have applied this, your Prometheus query should return -something like this: +After you have made this change, your [Prometheus query](#verify-authentication-monitoring) +should return something like: ```prometheus {enforced="false",status="would be ok"} 4424.985419441742 ``` -Because `enforced="false"`, it will be safe to start rolling out the new -token. +Because `enforced="false"`, it is safe to start rolling out the new token. -### 3. Update Gitaly token on all clients and servers +### Update Gitaly authentication token -```ruby -# in /etc/gitlab/gitlab.rb +To update to a new Gitaly authentication token, on each Gitaly client **and** Gitaly server: -gitaly['auth_token'] = 'my new secret token' -``` +1. Update the configuration: + + ```ruby + # in /etc/gitlab/gitlab.rb + + gitaly['auth_token'] = '<new secret token>' + ``` -Remember to apply this on both your Gitaly clients *and* servers. If you -check your Prometheus query while this change is being rolled out, you -will see non-zero values for the `enforced="false",status="denied"` counter. +1. Restart Gitaly: -### 4. Use Prometheus to ensure there are no authentication failures + ```shell + gitlab-ctl restart gitaly + ``` + +If you run your [Prometheus query](#verify-authentication-monitoring) while this change is +being rolled out, you will see non-zero values for the `enforced="false",status="denied"` counter. -After you applied the Gitaly token change everywhere, and all services -involved have been restarted, you should will temporarily see a mix of -`status="would be ok"` and `status="denied"`. +### Ensure there are no authentication failures -After the new token has been picked up by all Gitaly clients and -servers, the **only non-zero rate** should be -`enforced="false",status="would be ok"`. +After the new token is set, and all services involved have been restarted, you will +[temporarily see](#verify-authentication-monitoring) a mix of: -### 5. Disable "auth transitioning" Mode +- `status="would be ok"`. +- `status="denied"`. -Now we turn off the 'auth transitioning' mode. These final steps are -important: without them, you have **no authentication**. +After the new token has been picked up by all Gitaly clients and Gitaly servers, the +**only non-zero rate** should be `enforced="false",status="would be ok"`. -Update the configuration on your Gitaly servers: +### Disable "auth transitioning" mode + +To re-enable Gitaly authentication, disable "auth transitioning" mode. Update the configuration on +your Gitaly servers as follows: ```ruby # in /etc/gitlab/gitlab.rb gitaly['auth_transitioning'] = false ``` -### 6. Verify that authentication is enforced again +CAUTION: **Caution:** +Without completing this step, you have **no Gitaly authentication**. + +### Verify authentication is enforced -Refresh your Prometheus query. You should now see the same kind of -result as you did in the beginning: +Refresh your [Prometheus query](#verify-authentication-monitoring). You should now see a similar +result as you did at the start. For example: ```prometheus {enforced="true",status="ok"} 4424.985419441742 ``` -Note that `enforced="true"`, meaning that authentication is being enforced. +Note that `enforced="true"` means that authentication is being enforced. -## Direct Git access in GitLab Rails +## Direct Git access bypassing Gitaly -Also known as "the Rugged patches". +While it is possible to access Gitaly repositories stored on disk directly with a Git client, +it is not advisable because Gitaly is being continuously improved and changed. Theses improvements may invalidate assumptions, resulting in performance degradation, instability, and even data loss. + +Gitaly has optimizations, such as the +[`info/refs` advertisement cache](https://gitlab.com/gitlab-org/gitaly/blob/master/doc/design_diskcache.md), +that rely on Gitaly controlling and monitoring access to repositories via the +official gRPC interface. Likewise, Praefect has optimizations, such as fault +tolerance and distributed reads, that depend on the gRPC interface and +database to determine repository state. + +For these reasons, **accessing repositories directly is done at your own risk +and is not supported**. + +## Direct access to Git in GitLab + +Direct access to Git uses code in GitLab known as the "Rugged patches". ### History -Before Gitaly existed, the things that are now Gitaly clients used to -access Git repositories directly. Either on a local disk in the case of -e.g. a single-machine Omnibus GitLab installation, or via NFS in the -case of a horizontally scaled GitLab installation. +Before Gitaly existed, what are now Gitaly clients used to access Git repositories directly, either: + +- On a local disk in the case of a single-machine Omnibus GitLab installation +- Using NFS in the case of a horizontally-scaled GitLab installation. -Besides running plain `git` commands, in GitLab Rails we also used to -use a Ruby gem (library) called +Besides running plain `git` commands, GitLab used to use a Ruby library called [Rugged](https://github.com/libgit2/rugged). Rugged is a wrapper around -[libgit2](https://libgit2.org/), a stand-alone implementation of Git in -the form of a C library. - -Over time it has become clear to use that Rugged, and particularly -Rugged in combination with the [Unicorn](https://yhbt.net/unicorn/) -web server, is extremely efficient. Because libgit2 is a *library* and -not an external process, there was very little overhead between GitLab -application code that tried to look up data in Git repositories, and the -Git implementation itself. - -Because Rugged+Unicorn was so efficient, GitLab's application code ended -up with lots of duplicate Git object lookups (like looking up the -`master` commit a dozen times in one request). We could write -inefficient code without being punished for it. - -When we migrated these Git lookups to Gitaly calls, we were suddenly -getting a much higher fixed cost per Git lookup. Even when Gitaly is -able to re-use an already-running `git` process to look up e.g. a commit -you still have the cost of a network roundtrip to Gitaly, and within -Gitaly a write/read roundtrip on the Unix pipes that connect Gitaly to -the `git` process. - -Using GitLab.com performance as our yardstick, we pushed down the number -of Gitaly calls per request until the loss of Rugged's efficiency was no -longer felt. It also helped that we run Gitaly itself directly on the -Git file severs, rather than via NFS mounts: this gave us a speed boost -that counteracted the negative effect of not using Rugged anymore. - -Unfortunately, some *other* deployments of GitLab could not ditch NFS -like we did on GitLab.com and they got the worst of both worlds: the -slowness of NFS and the increased inherent overhead of Gitaly. - -As a performance band-aid for these stuck-on-NFS deployments, we -re-introduced some of the old Rugged code that got deleted from -GitLab Rails during the Gitaly migration project. These pieces of -re-introduced code are informally referred to as "the Rugged patches". - -### Activation of direct Git access in GitLab Rails - -The Ruby methods that perform direct Git access are hidden behind [feature -flags](../../development/gitaly.md#legacy-rugged-code). These feature -flags are off by default. It is not good if you need to know about -feature flags to get the best performance so in a second iteration, we -added an automatic mechanism that will enable direct Git access. - -When GitLab Rails calls a function that has a Rugged patch it performs -two checks. The result of both of these checks is cached. - -1. Is the feature flag for this patch set in the database? If so, do - what the feature flag says. -1. If the feature flag is not set (i.e. neither true nor false), try to - see if we can access filesystem underneath the Gitaly server - directly. If so, use the Rugged patch. - -To see if GitLab Rails can access the repository filesystem directly, we use -the following heuristic: - -- Gitaly ensures that the filesystem has a metadata file in its root - with a UUID in it. -- Gitaly reports this UUID to GitLab Rails via the `ServerInfo` RPC. -- GitLab Rails tries to read the metadata file directly. If it exists, - and if the UUID's match, assume we have direct access. - -Because of the way the UUID check works, and because Omnibus GitLab will -fill in the correct repository paths in the GitLab Rails config file -`config/gitlab.yml`, **direct Git access in GitLab Rails is on by default in -Omnibus**. - -### Plans to remove direct Git access in GitLab Rails - -For the sake of removing complexity it is desirable that we get rid of -direct Git access in GitLab Rails. For as long as some GitLab installations are stuck -with Git repositories on slow NFS, however, we cannot just remove them. - -There are two prongs to our efforts to remove direct Git access in GitLab Rails: - -1. Reduce the number of (inefficient) Gitaly queries made by - GitLab Rails. -1. Persuade everybody who runs a Highly Available / horizontally scaled - GitLab installation to move off of NFS. - -The second prong is the only real solution. For this we need [Gitaly -HA](https://gitlab.com/groups/gitlab-org/-/epics?scope=all&utf8=%E2%9C%93&state=opened&label_name[]=Gitaly%20HA), -which is still under development as of December 2019. +[libgit2](https://libgit2.org/), a stand-alone implementation of Git in the form of a C library. + +Over time it became clear that Rugged, particularly in combination with +[Unicorn](https://yhbt.net/unicorn/), is extremely efficient. Because `libgit2` is a library and +not an external process, there was very little overhead between: + +- GitLab application code that tried to look up data in Git repositories. +- The Git implementation itself. + +Because the combination of Rugged and Unicorn was so efficient, GitLab's application code ended up with lots of +duplicate Git object lookups. For example, looking up the `master` commit a dozen times in one +request. We could write inefficient code without poor performance. + +When we migrated these Git lookups to Gitaly calls, we suddenly had a much higher fixed cost per Git +lookup. Even when Gitaly is able to re-use an already-running `git` process (for example, to look up +a commit), you still have: + +- The cost of a network roundtrip to Gitaly. +- Within Gitaly, a write/read roundtrip on the Unix pipes that connect Gitaly to the `git` process. + +Using GitLab.com to measure, we reduced the number of Gitaly calls per request until the loss of +Rugged's efficiency was no longer felt. It also helped that we run Gitaly itself directly on the Git +file severs, rather than via NFS mounts. This gave us a speed boost that counteracted the negative +effect of not using Rugged anymore. + +Unfortunately, other deployments of GitLab could not remove NFS like we did on GitLab.com, and they +got the worst of both worlds: + +- The slowness of NFS. +- The increased inherent overhead of Gitaly. + +The code removed from GitLab during the Gitaly migration project affected these deployments. As a +performance workaround for these NFS-based deployments, we re-introduced some of the old Rugged +code. This re-introduced code is informally referred to as the "Rugged patches". + +### How it works + +The Ruby methods that perform direct Git access are behind +[feature flags](../../development/gitaly.md#legacy-rugged-code), disabled by default. It wasn't +convenient to set feature flags to get the best performance, so we added an automatic mechanism that +enables direct Git access. + +When GitLab calls a function that has a "Rugged patch", it performs two checks: + +- Is the feature flag for this patch set in the database? If so, the feature flag setting controls + GitLab's use of "Rugged patch" code. +- If the feature flag is not set, GitLab tries accessing the filesystem underneath the + Gitaly server directly. If it can, it will use the "Rugged patch". + +The result of both of these checks is cached. + +To see if GitLab can access the repository filesystem directly, we use the following heuristic: + +- Gitaly ensures that the filesystem has a metadata file in its root with a UUID in it. +- Gitaly reports this UUID to GitLab via the `ServerInfo` RPC. +- GitLab Rails tries to read the metadata file directly. If it exists, and if the UUID's match, + assume we have direct access. + +Direct Git access is enable by default in Omnibus GitLab because it fills in the correct repository +paths in the GitLab configuration file `config/gitlab.yml`. This satisfies the UUID check. + +### Transition to Gitaly Cluster + +For the sake of removing complexity, we must remove direct Git access in GitLab. However, we can't +remove it as long some GitLab installations require Git repositories on NFS. + +There are two facets to our efforts to remove direct Git access in GitLab: + +- Reduce the number of inefficient Gitaly queries made by GitLab. +- Persuade administrators of fault-tolerant or horizontally-scaled GitLab instances to migrate off + NFS. + +The second facet presents the only real solution. For this, we developed +[Gitaly Cluster](praefect.md). ## Troubleshooting Gitaly diff --git a/doc/administration/gitaly/praefect.md b/doc/administration/gitaly/praefect.md index 3d4e606205e..1f97cd304f9 100644 --- a/doc/administration/gitaly/praefect.md +++ b/doc/administration/gitaly/praefect.md @@ -35,8 +35,8 @@ The availability objectives for Gitaly clusters are: Writes are replicated asynchronously. Any writes that have not been replicated to the newly promoted primary are lost. - [Strong Consistency](https://gitlab.com/groups/gitlab-org/-/epics/1189) is - planned to improve this to "no loss". + [Strong consistency](#strong-consistency) can be used to avoid loss in some + circumstances. - **Recovery Time Objective (RTO):** Less than 10 seconds. @@ -103,7 +103,7 @@ GitLab](https://about.gitlab.com/install/). You will need the IP/host address for each node. -1. `LOAD_BALANCER_SERVER_ADDRESS`: the IP/hots address of the load balancer +1. `LOAD_BALANCER_SERVER_ADDRESS`: the IP/host address of the load balancer 1. `POSTGRESQL_SERVER_ADDRESS`: the IP/host address of the PostgreSQL server 1. `PRAEFECT_HOST`: the IP/host address of the Praefect server 1. `GITALY_HOST`: the IP/host address of each Gitaly server @@ -131,14 +131,13 @@ with secure tokens as you complete the setup process. Praefect cluster directly; that could lead to data loss. 1. `PRAEFECT_SQL_PASSWORD`: this password is used by Praefect to connect to PostgreSQL. -1. `GRAFANA_PASSWORD`: this password is used to access the `admin` - account in the Grafana dashboards. We will note in the instructions below where these secrets are required. ### PostgreSQL -NOTE: **Note:** do not store the GitLab application database and the Praefect +NOTE: **Note:** +Do not store the GitLab application database and the Praefect database on the same PostgreSQL server if using [Geo](../geo/replication/index.md). The replication state is internal to each instance of GitLab and should not be replicated. @@ -283,9 +282,16 @@ application server, or a Gitaly node. 1. Configure the **Praefect** cluster to connect to each Gitaly node in the cluster by editing `/etc/gitlab/gitlab.rb`. - In the example below we have configured one virtual storage (or shard) named - `storage-1`. This cluster has three Gitaly nodes `gitaly-1`, `gitaly-2`, and - `gitaly-3`, which will be replicas of each other. + The virtual storage's name must match the configured storage name in GitLab + configuration. In a later step, we configure the storage name as `default` + so we use `default` here as well. This cluster has three Gitaly nodes `gitaly-1`, + `gitaly-2`, and `gitaly-3`, which will be replicas of each other. + + CAUTION: **Caution:** + If you have data on an already existing storage called + `default`, you should configure the virtual storage with another name and + [migrate the data to the Praefect storage](#migrating-existing-repositories-to-praefect) + afterwards. Replace `PRAEFECT_INTERNAL_TOKEN` with a strong secret, which will be used by Praefect when communicating with Gitaly nodes in the cluster. This token is @@ -296,7 +302,8 @@ application server, or a Gitaly node. More Gitaly nodes can be added to the cluster to increase the number of replicas. More clusters can also be added for very large GitLab instances. - NOTE: **Note:** The `gitaly-1` node is currently denoted the primary. This + NOTE: **Note:** + The `gitaly-1` node is currently denoted the primary. This can be used to manually fail from one node to another. This will be removed in the [future](https://gitlab.com/gitlab-org/gitaly/-/issues/2634). @@ -304,7 +311,7 @@ application server, or a Gitaly node. # Name of storage hash must match storage name in git_data_dirs on GitLab # server ('praefect') and in git_data_dirs on Gitaly nodes ('gitaly-1') praefect['virtual_storages'] = { - 'storage-1' => { + 'default' => { 'gitaly-1' => { 'address' => 'tcp://GITALY_HOST:8075', 'token' => 'PRAEFECT_INTERNAL_TOKEN', @@ -322,6 +329,8 @@ application server, or a Gitaly node. } ``` +1. [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/2013) in GitLab 13.1 and later, enable [distribution of reads](#distributed-reads). + 1. Save the changes to `/etc/gitlab/gitlab.rb` and [reconfigure Praefect](../restart_gitlab.md#omnibus-gitlab-reconfigure): @@ -349,9 +358,146 @@ application server, or a Gitaly node. **The steps above must be completed for each Praefect node!** +## Enabling TLS support + +> [Introduced](https://gitlab.com/gitlab-org/gitaly/-/issues/1698) in GitLab 13.2. + +Praefect supports TLS encryption. To communicate with a Praefect instance that listens +for secure connections, you must: + +- Use a `tls://` URL scheme in the `gitaly_address` of the corresponding storage entry + in the GitLab configuration. +- Bring your own certificates because this isn't provided automatically. The certificate + corresponding to each Praefect server must be installed on that Praefect server. + +Additionally the certificate, or its certificate authority, must be installed on all Gitaly servers +and on all Praefect clients that communicate with it following the procedure described in +[GitLab custom certificate configuration](https://docs.gitlab.com/omnibus/settings/ssl.html#install-custom-public-certificates) (and repeated below). + +Note the following: + +- The certificate must specify the address you use to access the Praefect server. If + addressing the Praefect server by: + + - Hostname, you can either use the Common Name field for this, or add it as a Subject + Alternative Name. + - IP address, you must add it as a Subject Alternative Name to the certificate. + +- You can configure Praefect servers with both an unencrypted listening address + `listen_addr` and an encrypted listening address `tls_listen_addr` at the same time. + This allows you to do a gradual transition from unencrypted to encrypted traffic, if + necessary. + +To configure Praefect with TLS: + +**For Omnibus GitLab** + +1. Create certificates for Praefect servers. +1. On the Praefect servers, create the `/etc/gitlab/ssl` directory and copy your key + and certificate there: + + ```shell + sudo mkdir -p /etc/gitlab/ssl + sudo chmod 755 /etc/gitlab/ssl + sudo cp key.pem cert.pem /etc/gitlab/ssl/ + sudo chmod 644 key.pem cert.pem + ``` + +1. Edit `/etc/gitlab/gitlab.rb` and add: + + ```ruby + praefect['tls_listen_addr'] = "0.0.0.0:3305" + praefect['certificate_path'] = "/etc/gitlab/ssl/cert.pem" + praefect['key_path'] = "/etc/gitlab/ssl/key.pem" + ``` + +1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). +1. On the Praefect clients (including each Gitaly server), copy the certificates, + or their certificate authority, into `/etc/gitlab/trusted-certs`: + + ```shell + sudo cp cert.pem /etc/gitlab/trusted-certs/ + ``` + +1. On the Praefect clients (except Gitaly servers), edit `git_data_dirs` in + `/etc/gitlab/gitlab.rb` as follows: + + ```ruby + git_data_dirs({ + 'default' => { 'gitaly_address' => 'tls://praefect1.internal:3305' }, + 'storage1' => { 'gitaly_address' => 'tls://praefect2.internal:3305' }, + }) + ``` + +1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). + +**For installations from source** + +1. Create certificates for Praefect servers. +1. On the Praefect servers, create the `/etc/gitlab/ssl` directory and copy your key and certificate + there: + + ```shell + sudo mkdir -p /etc/gitlab/ssl + sudo chmod 755 /etc/gitlab/ssl + sudo cp key.pem cert.pem /etc/gitlab/ssl/ + sudo chmod 644 key.pem cert.pem + ``` + +1. On the Praefect clients (including each Gitaly server), copy the certificates, + or their certificate authority, into the system trusted certificates: + + ```shell + sudo cp cert.pem /usr/local/share/ca-certificates/praefect.crt + sudo update-ca-certificates + ``` + +1. On the Praefect clients (except Gitaly servers), edit `storages` in + `/home/git/gitlab/config/gitlab.yml` as follows: + + ```yaml + gitlab: + repositories: + storages: + default: + gitaly_address: tls://praefect1.internal:3305 + path: /some/dummy/path + storage1: + gitaly_address: tls://praefect2.internal:3305 + path: /some/dummy/path + ``` + + NOTE: **Note:** + `/some/dummy/path` should be set to a local folder that exists, however no + data will be stored in this folder. This will no longer be necessary after + [this issue](https://gitlab.com/gitlab-org/gitaly/-/issues/1282) is resolved. + +1. Save the file and [restart GitLab](../restart_gitlab.md#installations-from-source). +1. Copy all Praefect server certificates, or their certificate authority, to the system + trusted certificates on each Gitaly server so the Praefect server will trust the + certificate when called by Gitaly servers: + + ```shell + sudo cp cert.pem /usr/local/share/ca-certificates/praefect.crt + sudo update-ca-certificates + ``` + +1. Edit `/home/git/praefect/config.toml` and add: + + ```toml + tls_listen_addr = '0.0.0.0:3305' + + [tls] + certificate_path = '/etc/gitlab/ssl/cert.pem' + key_path = '/etc/gitlab/ssl/key.pem' + ``` + +1. Save the file and [restart GitLab](../restart_gitlab.md#installations-from-source). + ### Gitaly -NOTE: **Note:** Complete these steps for **each** Gitaly node. +NOTE: **Note:** +Complete these steps for **each** Gitaly node. To complete this section you will need: @@ -555,6 +701,17 @@ Particular attention should be shown to: external_url 'GITLAB_SERVER_URL' ``` +1. Disable the default Gitaly service running on the GitLab host. It won't be needed + as GitLab will connect to the configured cluster. + + CAUTION: **Caution:** + If you have existing data stored on the default Gitaly storage, + you should [migrate the data your Praefect storage first](#migrating-existing-repositories-to-praefect). + + ```ruby + gitaly['enable'] = false + ``` + 1. Add the Praefect cluster as a storage location by editing `/etc/gitlab/gitlab.rb`. @@ -562,28 +719,17 @@ Particular attention should be shown to: - `LOAD_BALANCER_SERVER_ADDRESS` with the IP address or hostname of the load balancer. - - `GITLAB_HOST` with the IP address or hostname of the GitLab server - `PRAEFECT_EXTERNAL_TOKEN` with the real secret ```ruby git_data_dirs({ "default" => { - "gitaly_address" => "tcp://GITLAB_HOST:8075" - }, - "storage-1" => { "gitaly_address" => "tcp://LOAD_BALANCER_SERVER_ADDRESS:2305", "gitaly_token" => 'PRAEFECT_EXTERNAL_TOKEN' } }) ``` -1. Allow Gitaly to listen on a TCP port by editing - `/etc/gitlab/gitlab.rb` - - ```ruby - gitaly['listen_addr'] = '0.0.0.0:8075' - ``` - 1. Configure the `gitlab_shell['secret_token']` so that callbacks from Gitaly nodes during a `git push` are properly authenticated by editing `/etc/gitlab/gitlab.rb`: @@ -632,14 +778,6 @@ Particular attention should be shown to: gitlab-ctl reconfigure ``` -1. To ensure that Gitaly [has updated its Prometheus listen - address](https://gitlab.com/gitlab-org/gitaly/-/issues/2734), [restart - Gitaly](../restart_gitlab.md#omnibus-gitlab-restart): - - ```shell - gitlab-ctl restart gitaly - ``` - 1. Verify each `gitlab-shell` on each Gitaly instance can reach GitLab. On each Gitaly instance run: ```shell @@ -652,16 +790,11 @@ Particular attention should be shown to: gitlab-rake gitlab:gitaly:check ``` -1. Update the **Repository storage** settings from **Admin Area > Settings > - Repository > Repository storage** to make the newly configured Praefect - cluster the storage location for new Git repositories. - - - The default option is unchecked. - - The Praefect option is checked. +1. Check in **Admin Area > Settings > Repository > Repository storage** that the Praefect storage + is configured to store new repositories. Following this guide, the `default` storage should have + weight 100 to store all new repositories. -  - -1. Verify everything is still working by creating a new project. Check the +1. Verify everything is working by creating a new project. Check the "Initialize repository with a README" box so that there is content in the repository that viewed. If the project is created, and you can see the README file, it works! @@ -712,6 +845,72 @@ To get started quickly: Congratulations! You've configured an observable highly available Praefect cluster. +## Distributed reads + +> Introduced in GitLab 13.1 in [beta](https://about.gitlab.com/handbook/product/#alpha-beta-ga) with feature flag `gitaly_distributed_reads` set to disabled. + +Praefect supports distribution of read operations across Gitaly nodes that are +configured for the virtual node. + +To allow for [performance testing](https://gitlab.com/gitlab-org/quality/performance/-/issues/231), +distributed reads are currently in +[beta](https://about.gitlab.com/handbook/product/#alpha-beta-ga) and disabled by +default. To enable distributed reads, the `gitaly_distributed_reads` +[feature flag](../feature_flags.md) must be enabled in a Ruby console: + +```ruby +Feature.enable(:gitaly_distributed_reads) +``` + +If enabled, all RPCs marked with `ACCESSOR` option like +[GetBlob](https://gitlab.com/gitlab-org/gitaly/-/blob/v12.10.6/proto/blob.proto#L16) +are redirected to an up to date and healthy Gitaly node. + +_Up to date_ in this context means that: + +- There is no replication operations scheduled for this node. +- The last replication operation is in _completed_ state. + +If there is no such nodes, or any other error occurs during node selection, the primary +node will be chosen to serve the request. + +To track distribution of read operations, you can use the `gitaly_praefect_read_distribution` +Prometheus counter metric. It has two labels: + +- `virtual_storage`. +- `storage`. + +They reflect configuration defined for this instance of Praefect. + +## Strong consistency + +> Introduced in GitLab 13.1 in [alpha](https://about.gitlab.com/handbook/product/#alpha-beta-ga), disabled by default. + +Praefect guarantees eventual consistency by replicating all writes to secondary nodes +after the write to the primary Gitaly node has happened. + +Praefect can instead provide strong consistency by creating a transaction and writing +changes to all Gitaly nodes at once. Strong consistency is currently in +[alpha](https://about.gitlab.com/handbook/product/#alpha-beta-ga) and not enabled by +default. If enabled, transactions are only available for a subset of RPCs. For more +information, see the [strong consistency epic](https://gitlab.com/groups/gitlab-org/-/epics/1189). + +To enable strong consistency: + +- In GitLab 13.2 and later, enable the `:gitaly_reference_transactions` feature flag. +- In GitLab 13.1, enable the `:gitaly_reference_transactions` and `:gitaly_hooks_rpc` + feature flags. + +Enabling feature flags requires [access to the Rails console](../feature_flags.md#start-the-gitlab-rails-console). +In the Rails console, enable or disable the flags as required. For example: + +```ruby +Feature.enable(:gitaly_reference_transactions) +``` + +To monitor strong consistency, use the `gitaly_praefect_transactions_total` and +`gitaly_praefect_transactions_delay_seconds` Prometheus counter metrics. + ## Automatic failover and leader election Praefect regularly checks the health of each backend Gitaly node. This @@ -739,38 +938,68 @@ current primary node is found to be unhealthy. It is likely that we will implement support for Consul, and a cloud native strategy in the future. -## Identifying Impact of a Primary Node Failure +## Primary Node Failure -When a primary Gitaly node fails, there is a chance of data loss. Data loss can occur if there were outstanding replication jobs the secondaries did not manage to process before the failure. The `dataloss` Praefect sub-command helps identify these cases by counting the number of dead replication jobs for each repository. This command must be executed on a Praefect node. +Praefect recovers from a failing primary Gitaly node by promoting a healthy secondary as the new primary. To minimize data loss, Praefect elects the secondary with the least unreplicated writes from the primary. There can still be some unreplicated writes, leading to data loss. -A time frame to search can be specified with `-from` and `-to`: +Praefect switches a virtual storage in to read-only mode after a failover event. This eases data recovery efforts by preventing new, possibly conflicting writes to the newly elected primary. This allows the administrator to attempt recovering the lost data before allowing new writes. + +If you prefer write availability over consistency, this behavior can be turned off by setting `praefect['failover_read_only_after_failover'] = false` in `/etc/gitlab/gitlab.rb` and [reconfiguring Praefect](../restart_gitlab.md#omnibus-gitlab-reconfigure). + +### Checking for data loss + +The Praefect `dataloss` sub-command helps identify lost writes by checking for uncompleted replication jobs. This is useful for identifying possible data loss cases after a failover. This command must be executed on a Praefect node. ```shell -sudo /opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml dataloss -from <rfc3339-time> -to <rfc3339-time> +sudo /opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml dataloss [-virtual-storage <virtual-storage>] ``` -If the time frame is not specified, dead replication jobs from the last six hours are counted: +If the virtual storage is not specified, every configured virtual storage is checked for data loss. ```shell sudo /opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml dataloss +``` -Failed replication jobs between [2020-01-02 00:00:00 +0000 UTC, 2020-01-02 06:00:00 +0000 UTC): -@hashed/fa/53/fa539965395b8382145f8370b34eab249cf610d2d6f2943c95b9b9d08a63d4a3.git: 2 jobs +```shell +Virtual storage: default + Current read-only primary: gitaly-2 + Previous write-enabled primary: gitaly-1 + Nodes with data loss from failing over from gitaly-1: + @hashed/2c/62/2c624232cdd221771294dfbb310aca000a0df6ac8b66b696d90ef06fdefb64a3.git: gitaly-0 + @hashed/4b/22/4b227777d4dd1fc61c6f884f48641d02b4d121d3fd328cb08b5531fcacdabf8a.git: gitaly-0, gitaly-2 ``` -To specify a time frame in UTC, run: +Currently `dataloss` only considers a repository up to date if it has been directly replicated to from the previous write-enabled primary. While reconciling from an up to date secondary can recover the data, this is not visible in the data loss report. This is due for improvement via [Gitaly#2866](https://gitlab.com/gitlab-org/gitaly/-/issues/2866). + +NOTE: **Note:** +`dataloss` is still in beta and the output format is subject to change. + +### Checking repository checksums + +To check a project's repository checksums across on all Gitaly nodes, run the +[replicas Rake task](../raketasks/praefect.md#replica-checksums) on the main GitLab node. + +### Recovering lost writes + +The Praefect `reconcile` sub-command can be used to recover lost writes from the +previous primary once it is back online. This is only possible when the virtual storage +is still in read-only mode. ```shell -sudo /opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml dataloss -from 2020-01-02T00:00:00+00:00 -to 2020-01-02T00:02:00+00:00 +sudo /opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml reconcile -virtual <virtual-storage> -reference <previous-primary> -target <current-primary> -f ``` -### Checking repository checksums +Refer to [Backend Node Recovery](#backend-node-recovery) section for more details on +the `reconcile` sub-command. + +### Enabling Writes -To check a project's repository checksums across on all Gitaly nodes, the -replicas Rake task can be run on the main GitLab node: +Any data recovery attempts should have been made before enabling writes to eliminate +any chance of conflicting writes. Virtual storage can be re-enabled for writes by using +the Praefect `enable-writes` sub-command. ```shell -sudo gitlab-rake "gitlab:praefect:replicas[project_id]" +sudo /opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml enable-writes -virtual-storage <virtual-storage> ``` ## Backend Node Recovery diff --git a/doc/administration/gitaly/reference.md b/doc/administration/gitaly/reference.md index 52fd6fa6900..0429149ec2d 100644 --- a/doc/administration/gitaly/reference.md +++ b/doc/administration/gitaly/reference.md @@ -91,7 +91,7 @@ certificate_path = '/home/git/cert.cert' key_path = '/home/git/key.pem' ``` -[Read more](index.md#tls-support) about TLS in Gitaly. +[Read more](index.md#enable-tls-support) about TLS in Gitaly. ### Storage diff --git a/doc/administration/high_availability/consul.md b/doc/administration/high_availability/consul.md index a87c1f1027f..978ba08c4fa 100644 --- a/doc/administration/high_availability/consul.md +++ b/doc/administration/high_availability/consul.md @@ -113,14 +113,14 @@ Nodes running GitLab-bundled Consul should be: - Members of a healthy cluster prior to upgrading the Omnibus GitLab package. - Upgraded one node at a time. -NOTE: **NOTE:** +NOTE: **Note:** Running `curl http://127.0.0.1:8500/v1/health/state/critical` from any Consul node will identify existing health issues in the cluster. The command will return an empty array if the cluster is healthy. Consul clusters communicate using the raft protocol. If the current leader goes offline, there needs to be a leader election. A leader node must exist to facilitate synchronization across the cluster. If too many nodes go offline at the same time, the cluster will lose quorum and not elect a leader due to [broken consensus](https://www.consul.io/docs/internals/consensus.html). Consult the [troubleshooting section](#troubleshooting) if the cluster is not able to recover after the upgrade. The [outage recovery](#outage-recovery) may be of particular interest. -NOTE: **NOTE:** +NOTE: **Note:** GitLab only uses Consul to store transient data that is easily regenerated. If the bundled Consul was not used by any process other than GitLab itself, then [rebuilding the cluster from scratch](#recreate-from-scratch) is fine. ## Troubleshooting diff --git a/doc/administration/high_availability/database.md b/doc/administration/high_availability/database.md index 75183436046..784e496d10e 100644 --- a/doc/administration/high_availability/database.md +++ b/doc/administration/high_availability/database.md @@ -1,20 +1,5 @@ --- -type: reference +redirect_to: '../postgresql/index.md' --- -# Configuring PostgreSQL for Scaling and High Availability - -In this section, you'll be guided through configuring a PostgreSQL database to -be used with GitLab in one of our [Scalable and Highly Available Setups](../reference_architectures/index.md). - -## Provide your own PostgreSQL instance **(CORE ONLY)** - -This content has been moved to a [new location](../postgresql/external.md). - -## Standalone PostgreSQL using Omnibus GitLab **(CORE ONLY)** - -This content has been moved to a [new location](../postgresql/standalone.md). - -## PostgreSQL replication and failover with Omnibus GitLab **(PREMIUM ONLY)** - -This content has been moved to a [new location](../postgresql/replication_and_failover.md). +This document was moved to [another location](../postgresql/index.md). diff --git a/doc/administration/high_availability/gitlab.md b/doc/administration/high_availability/gitlab.md index 67a84f99bea..dc8c997bab5 100644 --- a/doc/administration/high_availability/gitlab.md +++ b/doc/administration/high_availability/gitlab.md @@ -6,11 +6,13 @@ type: reference This section describes how to configure the GitLab application (Rails) component. -NOTE: **Note:** There is some additional configuration near the bottom for +NOTE: **Note:** +There is some additional configuration near the bottom for additional GitLab application servers. It's important to read and understand these additional steps before proceeding with GitLab installation. -NOTE: **Note:** [Cloud Object Storage service](object_storage.md) with [Gitaly](gitaly.md) +NOTE: **Note:** +[Cloud Object Storage service](object_storage.md) with [Gitaly](gitaly.md) is recommended over [NFS](nfs.md) wherever possible for improved performance. 1. If necessary, install the NFS client utility packages using the following @@ -79,19 +81,22 @@ is recommended over [NFS](nfs.md) wherever possible for improved performance. 1. [Enable monitoring](#enable-monitoring) - NOTE: **Note:** To maintain uniformity of links across HA clusters, the `external_url` + NOTE: **Note:** + To maintain uniformity of links across HA clusters, the `external_url` on the first application server as well as the additional application servers should point to the external URL that users will use to access GitLab. In a typical HA setup, this will be the URL of the load balancer which will route traffic to all GitLab application servers in the HA cluster. - NOTE: **Note:** When you specify `https` in the `external_url`, as in the example + NOTE: **Note:** + When you specify `https` in the `external_url`, as in the example above, GitLab assumes you have SSL certificates in `/etc/gitlab/ssl/`. If certificates are not present, NGINX will fail to start. See [NGINX documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https) for more information. - NOTE: **Note:** It is best to set the `uid` and `gid`s prior to the initial reconfigure + NOTE: **Note:** + It is best to set the `uid` and `gid`s prior to the initial reconfigure of GitLab. Omnibus will not recursively `chown` directories if set after the initial reconfigure. ## First GitLab application server @@ -126,14 +131,15 @@ need some extra configuration. from running on upgrade. Only the primary GitLab application server should handle migrations. -1. **Recommended** Configure host keys. Copy the contents (primary and public keys) of `/etc/ssh/` on +1. **Recommended** Configure host keys. Copy the contents (private and public keys) of `/etc/ssh/` on the primary application server to `/etc/ssh` on all secondary servers. This prevents false man-in-the-middle-attack alerts when accessing servers in your High Availability cluster behind a load balancer. 1. Run `sudo gitlab-ctl reconfigure` to compile the configuration. -NOTE: **Note:** You will need to restart the GitLab applications nodes after an update has occurred and database +NOTE: **Note:** +You will need to restart the GitLab applications nodes after an update has occurred and database migrations performed. ## Enable Monitoring diff --git a/doc/administration/high_availability/monitoring_node.md b/doc/administration/high_availability/monitoring_node.md index 653a0b32ad7..6b6f0ae9ea3 100644 --- a/doc/administration/high_availability/monitoring_node.md +++ b/doc/administration/high_availability/monitoring_node.md @@ -71,6 +71,19 @@ Omnibus: 1. Run `sudo gitlab-ctl reconfigure` to compile the configuration. +The next step is to tell all the other nodes where the monitoring node is: + +1. Edit `/etc/gitlab/gitlab.rb`, and add, or find and uncomment the following line: + + ```ruby + gitlab_rails['prometheus_address'] = '10.0.0.1:9090' + ``` + + Where `10.0.0.1:9090` is the IP address and port of the Prometheus node. + +1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to + take effect. + ## Migrating to Service Discovery Once monitoring using Service Discovery is enabled with `consul['monitoring_service_discovery'] = true`, diff --git a/doc/administration/high_availability/nfs.md b/doc/administration/high_availability/nfs.md index 6511f9bd85d..6e8dc2c6c57 100644 --- a/doc/administration/high_availability/nfs.md +++ b/doc/administration/high_availability/nfs.md @@ -12,11 +12,29 @@ From GitLab 13.0, using NFS for Git repositories is deprecated. In GitLab 14.0, support for NFS for Git repositories is scheduled to be removed. Upgrade to [Gitaly Cluster](../gitaly/praefect.md) as soon as possible. -NOTE: **Note:** Filesystem performance has a big impact on overall GitLab +NOTE: **Note:** +Filesystem performance has a big impact on overall GitLab performance, especially for actions that read or write to Git repositories. See [Filesystem Performance Benchmarking](../operations/filesystem_benchmarking.md) for steps to test filesystem performance. +## Known kernel version incompatibilities + +RedHat Enterprise Linux (RHEL) and CentOS v7.7 and v7.8 ship with kernel +version `3.10.0-1127`, which [contains a +bug](https://bugzilla.redhat.com/show_bug.cgi?id=1783554) that causes +[uploads to fail to copy over NFS](https://gitlab.com/gitlab-org/gitlab/-/issues/218999). The +following GitLab versions include a fix to work properly with that +kernel version: + +1. [12.10.12](https://about.gitlab.com/releases/2020/06/25/gitlab-12-10-12-released/) +1. [13.0.7](https://about.gitlab.com/releases/2020/06/25/gitlab-13-0-7-released/) +1. [13.1.1](https://about.gitlab.com/releases/2020/06/24/gitlab-13-1-1-released/) +1. 13.2 and up + +If you are using that kernel version, be sure to upgrade GitLab to avoid +errors. + ## NFS Server features ### Required features @@ -88,7 +106,8 @@ administrators to keep NFS server delegation disabled. #### Improving NFS performance with Unicorn -NOTE: **Note:** From GitLab 12.1, it will automatically be detected if Rugged can and should be used per storage. +NOTE: **Note:** +From GitLab 12.1, it will automatically be detected if Rugged can and should be used per storage. If you previously enabled Rugged using the feature flag, you will need to unset the feature flag by using: @@ -100,7 +119,8 @@ If the Rugged feature flag is explicitly set to either true or false, GitLab wil #### Improving NFS performance with Puma -NOTE: **Note:** From GitLab 12.7, Rugged auto-detection is disabled if Puma thread count is greater than 1. +NOTE: **Note:** +From GitLab 12.7, Rugged auto-detection is disabled if Puma thread count is greater than 1. If you want to use Rugged with Puma, it is recommended to [set Puma thread count to 1](https://docs.gitlab.com/omnibus/settings/puma.html#puma-settings). diff --git a/doc/administration/high_availability/nfs_host_client_setup.md b/doc/administration/high_availability/nfs_host_client_setup.md index 7519ebf028d..213680e2f64 100644 --- a/doc/administration/high_availability/nfs_host_client_setup.md +++ b/doc/administration/high_availability/nfs_host_client_setup.md @@ -38,10 +38,10 @@ In this setup we will share the home directory on the host with the client. Edit ```plaintext #/etc/exports for one client -/home <client-ip-address>(rw,sync,no_root_squash,no_subtree_check) +/home <client_ip_address>(rw,sync,no_root_squash,no_subtree_check) #/etc/exports for three clients -/home <client-ip-address>(rw,sync,no_root_squash,no_subtree_check) <client-2-ip-address>(rw,sync,no_root_squash,no_subtree_check) <client-3-ip-address>(rw,sync,no_root_squash,no_subtree_check) +/home <client_ip_address>(rw,sync,no_root_squash,no_subtree_check) <client_2_ip_address>(rw,sync,no_root_squash,no_subtree_check) <client_3_ip_address>(rw,sync,no_root_squash,no_subtree_check) ``` Restart the NFS server after making changes to the `exports` file for the changes @@ -54,7 +54,7 @@ systemctl restart nfs-kernel-server NOTE: **Note:** You may need to update your server's firewall. See the [firewall section](#nfs-in-a-firewalled-environment) at the end of this guide. -## Client/ GitLab application node Setup +## Client / GitLab application node Setup > Follow the instructions below to connect any GitLab Rails application node running inside your HA environment to the NFS server configured above. @@ -90,7 +90,7 @@ df -h ### Step 3 - Set up Automatic Mounts on Boot -Edit `/etc/fstab` on client as below to mount the remote shares automatically at boot. +Edit `/etc/fstab` on the client as below to mount the remote shares automatically at boot. Note that GitLab requires advisory file locking, which is only supported natively in NFS version 4. NFSv3 also supports locking as long as Linux Kernel 2.6.5+ is used. We recommend using version 4 and do not specifically test NFSv3. @@ -98,14 +98,19 @@ See [NFS documentation](nfs.md#nfs-client-mount-options) for guidance on mount o ```plaintext #/etc/fstab -10.0.0.1:/nfs/home /nfs/home nfs4 defaults,hard,vers=4.1,rsize=1048576,wsize=1048576,noatime,nofail,lookupcache=positive 0 2 +<host_ip_address>:/home /nfs/home nfs4 defaults,hard,vers=4.1,rsize=1048576,wsize=1048576,noatime,nofail,lookupcache=positive 0 2 ``` Reboot the client and confirm that the mount point is mounted automatically. +NOTE: **Note:** +If you followed our guide to [GitLab Pages on a separate server](../pages/index.md#running-gitlab-pages-on-a-separate-server) +here, please continue there with the pages-specific NFS mounts. +The step below is for broader use-cases than only sharing pages data. + ### Step 4 - Set up GitLab to Use NFS mounts -When using the default Omnibus configuration you will need to share 5 data locations +When using the default Omnibus configuration you will need to share 4 data locations between all GitLab cluster nodes. No other locations should be shared. Changing the default file locations in `gitlab.rb` on the client allows you to have one main mount point and have all the required locations as subdirectories to use the NFS mount for @@ -136,7 +141,7 @@ the command: `sudo ufw status`. If it's being blocked, then you can allow traffi client with the command below. ```shell -sudo ufw allow from <client-ip-address> to any port nfs +sudo ufw allow from <client_ip_address> to any port nfs ``` <!-- ## Troubleshooting diff --git a/doc/administration/high_availability/redis.md b/doc/administration/high_availability/redis.md index bad50f7ca74..2b5771f49f2 100644 --- a/doc/administration/high_availability/redis.md +++ b/doc/administration/high_availability/redis.md @@ -1,1008 +1,5 @@ --- -type: reference +redirect_to: ../redis/index.md --- -# Configuring Redis for Scaling and High Availability - -## Provide your own Redis instance **(CORE ONLY)** - -The following are the requirements for providing your own Redis instance: - -- Redis version 5.0 or higher is recommended, as this is what ships with - Omnibus GitLab packages starting with GitLab 12.7. -- Support for Redis 3.2 is deprecated with GitLab 12.10 and will be completely - removed in GitLab 13.0. -- GitLab 12.0 and later requires Redis version 3.2 or higher. Older Redis - versions do not support an optional count argument to SPOP which is now - required for [Merge Trains](../../ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md). -- In addition, if Redis 4 or later is available, GitLab makes use of certain - commands like `UNLINK` and `USAGE` which were introduced only in Redis 4. -- Standalone Redis or Redis high availability with Sentinel are supported. Redis - Cluster is not supported. -- Managed Redis from cloud providers such as AWS ElastiCache will work. If these - services support high availability, be sure it is not the Redis Cluster type. - -Note the Redis node's IP address or hostname, port, and password (if required). -These will be necessary when configuring the GitLab application servers later. - -## Redis in a Scaled and Highly Available Environment - -This section is relevant for [scalable and highly available setups](../reference_architectures/index.md). - -### Provide your own Redis instance **(CORE ONLY)** - -If you want to use your own deployed Redis instance(s), -see [Provide your own Redis instance](#provide-your-own-redis-instance-core-only) -for more details. However, you can use the Omnibus GitLab package to easily -deploy the bundled Redis. - -### Standalone Redis using Omnibus GitLab **(CORE ONLY)** - -The Omnibus GitLab package can be used to configure a standalone Redis server. -In this configuration Redis is not highly available, and represents a single -point of failure. However, in a scaled environment the objective is to allow -the environment to handle more users or to increase throughput. Redis itself -is generally stable and can handle many requests so it is an acceptable -trade off to have only a single instance. See the [reference architectures](../reference_architectures/index.md) -page for an overview of GitLab scaling and high availability options. - -The steps below are the minimum necessary to configure a Redis server with -Omnibus: - -1. SSH into the Redis server. -1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab - package you want using **steps 1 and 2** from the GitLab downloads page. - - Do not complete any other steps on the download page. - -1. Edit `/etc/gitlab/gitlab.rb` and add the contents: - - ```ruby - ## Enable Redis - redis['enable'] = true - - ## Disable all other services - sidekiq['enable'] = false - gitlab_workhorse['enable'] = false - puma['enable'] = false - postgresql['enable'] = false - nginx['enable'] = false - prometheus['enable'] = false - alertmanager['enable'] = false - pgbouncer_exporter['enable'] = false - gitlab_exporter['enable'] = false - gitaly['enable'] = false - - redis['bind'] = '0.0.0.0' - redis['port'] = 6379 - redis['password'] = 'SECRET_PASSWORD_HERE' - - gitlab_rails['enable'] = false - ``` - -1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. -1. Note the Redis node's IP address or hostname, port, and - Redis password. These will be necessary when configuring the GitLab - application servers later. -1. [Enable Monitoring](#enable-monitoring) - -Advanced configuration options are supported and can be added if -needed. - -Continue configuration of other components by going back to the -[reference architectures](../reference_architectures/index.md#configure-gitlab-to-scale) page. - -### High Availability with Omnibus GitLab **(PREMIUM ONLY)** - -> Experimental Redis Sentinel support was [introduced in GitLab 8.11](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/1877). -Starting with 8.14, Redis Sentinel is no longer experimental. -If you've used it with versions `< 8.14` before, please check the updated -documentation here. - -High Availability with [Redis](https://redis.io/) is possible using a **Master** x **Replica** -topology with a [Redis Sentinel](https://redis.io/topics/sentinel) service to watch and automatically -start the failover procedure. - -You can choose to install and manage Redis and Sentinel yourself, use -a hosted cloud solution or you can use the one that comes bundled with -Omnibus GitLab packages. - -> **Notes:** -> -> - Redis requires authentication for High Availability. See -> [Redis Security](https://redis.io/topics/security) documentation for more -> information. We recommend using a combination of a Redis password and tight -> firewall rules to secure your Redis service. -> - You are highly encouraged to read the [Redis Sentinel](https://redis.io/topics/sentinel) documentation -> before configuring Redis HA with GitLab to fully understand the topology and -> architecture. -> - This is the documentation for the Omnibus GitLab packages. For installations -> from source, follow the [Redis HA source installation](redis_source.md) guide. -> - Redis Sentinel daemon is bundled with Omnibus GitLab Enterprise Edition only. -> For configuring Sentinel with the Omnibus GitLab Community Edition and -> installations from source, read the -> [Available configuration setups](#available-configuration-setups) section -> below. - -## Overview - -Before diving into the details of setting up Redis and Redis Sentinel for HA, -make sure you read this Overview section to better understand how the components -are tied together. - -You need at least `3` independent machines: physical, or VMs running into -distinct physical machines. It is essential that all master and replica Redis -instances run in different machines. If you fail to provision the machines in -that specific way, any issue with the shared environment can bring your entire -setup down. - -It is OK to run a Sentinel alongside of a master or replica Redis instance. -There should be no more than one Sentinel on the same machine though. - -You also need to take into consideration the underlying network topology, -making sure you have redundant connectivity between Redis / Sentinel and -GitLab instances, otherwise the networks will become a single point of -failure. - -Make sure that you read this document once as a whole before configuring the -components below. - -> **Notes:** -> -> - Starting with GitLab `8.11`, you can configure a list of Redis Sentinel -> servers that will monitor a group of Redis servers to provide failover support. -> - Starting with GitLab `8.14`, the Omnibus GitLab Enterprise Edition package -> comes with Redis Sentinel daemon built-in. - -High Availability with Redis requires a few things: - -- Multiple Redis instances -- Run Redis in a **Master** x **Replica** topology -- Multiple Sentinel instances -- Application support and visibility to all Sentinel and Redis instances - -Redis Sentinel can handle the most important tasks in an HA environment and that's -to help keep servers online with minimal to no downtime. Redis Sentinel: - -- Monitors **Master** and **Replicas** instances to see if they are available -- Promotes a **Replica** to **Master** when the **Master** fails -- Demotes a **Master** to **Replica** when the failed **Master** comes back online - (to prevent data-partitioning) -- Can be queried by the application to always connect to the current **Master** - server - -When a **Master** fails to respond, it's the application's responsibility -(in our case GitLab) to handle timeout and reconnect (querying a **Sentinel** -for a new **Master**). - -To get a better understanding on how to correctly set up Sentinel, please read -the [Redis Sentinel documentation](https://redis.io/topics/sentinel) first, as -failing to configure it correctly can lead to data loss or can bring your -whole cluster down, invalidating the failover effort. - -### Recommended setup - -For a minimal setup, you will install the Omnibus GitLab package in `3` -**independent** machines, both with **Redis** and **Sentinel**: - -- Redis Master + Sentinel -- Redis Replica + Sentinel -- Redis Replica + Sentinel - -If you are not sure or don't understand why and where the amount of nodes come -from, read [Redis setup overview](#redis-setup-overview) and -[Sentinel setup overview](#sentinel-setup-overview). - -For a recommended setup that can resist more failures, you will install -the Omnibus GitLab package in `5` **independent** machines, both with -**Redis** and **Sentinel**: - -- Redis Master + Sentinel -- Redis Replica + Sentinel -- Redis Replica + Sentinel -- Redis Replica + Sentinel -- Redis Replica + Sentinel - -### Redis setup overview - -You must have at least `3` Redis servers: `1` Master, `2` Replicas, and they -need to each be on independent machines (see explanation above). - -You can have additional Redis nodes, that will help survive a situation -where more nodes goes down. Whenever there is only `2` nodes online, a failover -will not be initiated. - -As an example, if you have `6` Redis nodes, a maximum of `3` can be -simultaneously down. - -Please note that there are different requirements for Sentinel nodes. -If you host them in the same Redis machines, you may need to take -that restrictions into consideration when calculating the amount of -nodes to be provisioned. See [Sentinel setup overview](#sentinel-setup-overview) -documentation for more information. - -All Redis nodes should be configured the same way and with similar server specs, as -in a failover situation, any **Replica** can be promoted as the new **Master** by -the Sentinel servers. - -The replication requires authentication, so you need to define a password to -protect all Redis nodes and the Sentinels. They will all share the same -password, and all instances must be able to talk to -each other over the network. - -### Sentinel setup overview - -Sentinels watch both other Sentinels and Redis nodes. Whenever a Sentinel -detects that a Redis node is not responding, it will announce that to the -other Sentinels. They have to reach the **quorum**, that is the minimum amount -of Sentinels that agrees a node is down, in order to be able to start a failover. - -Whenever the **quorum** is met, the **majority** of all known Sentinel nodes -need to be available and reachable, so that they can elect the Sentinel **leader** -who will take all the decisions to restore the service availability by: - -- Promoting a new **Master** -- Reconfiguring the other **Replicas** and make them point to the new **Master** -- Announce the new **Master** to every other Sentinel peer -- Reconfigure the old **Master** and demote to **Replica** when it comes back online - -You must have at least `3` Redis Sentinel servers, and they need to -be each in an independent machine (that are believed to fail independently), -ideally in different geographical areas. - -You can configure them in the same machines where you've configured the other -Redis servers, but understand that if a whole node goes down, you loose both -a Sentinel and a Redis instance. - -The number of sentinels should ideally always be an **odd** number, for the -consensus algorithm to be effective in the case of a failure. - -In a `3` nodes topology, you can only afford `1` Sentinel node going down. -Whenever the **majority** of the Sentinels goes down, the network partition -protection prevents destructive actions and a failover **will not be started**. - -Here are some examples: - -- With `5` or `6` sentinels, a maximum of `2` can go down for a failover begin. -- With `7` sentinels, a maximum of `3` nodes can go down. - -The **Leader** election can sometimes fail the voting round when **consensus** -is not achieved (see the odd number of nodes requirement above). In that case, -a new attempt will be made after the amount of time defined in -`sentinel['failover_timeout']` (in milliseconds). - ->**Note:** -We will see where `sentinel['failover_timeout']` is defined later. - -The `failover_timeout` variable has a lot of different use cases. According to -the official documentation: - -- The time needed to re-start a failover after a previous failover was - already tried against the same master by a given Sentinel, is two - times the failover timeout. - -- The time needed for a replica replicating to a wrong master according - to a Sentinel current configuration, to be forced to replicate - with the right master, is exactly the failover timeout (counting since - the moment a Sentinel detected the misconfiguration). - -- The time needed to cancel a failover that is already in progress but - did not produced any configuration change (REPLICAOF NO ONE yet not - acknowledged by the promoted replica). - -- The maximum time a failover in progress waits for all the replicas to be - reconfigured as replicas of the new master. However even after this time - the replicas will be reconfigured by the Sentinels anyway, but not with - the exact parallel-syncs progression as specified. - -### Available configuration setups - -Based on your infrastructure setup and how you have installed GitLab, there are -multiple ways to configure Redis HA. Omnibus GitLab packages have Redis and/or -Redis Sentinel bundled with them so you only need to focus on configuration. -Pick the one that suits your needs. - -- [Installations from source](../../install/installation.md): You need to install Redis and Sentinel - yourself. Use the [Redis HA installation from source](redis_source.md) - documentation. -- [Omnibus GitLab **Community Edition** (CE) package](https://about.gitlab.com/install/?version=ce): Redis is bundled, so you - can use the package with only the Redis service enabled as described in steps - 1 and 2 of this document (works for both master and replica setups). To install - and configure Sentinel, jump directly to the Sentinel section in the - [Redis HA installation from source](redis_source.md#step-3-configuring-the-redis-sentinel-instances) documentation. -- [Omnibus GitLab **Enterprise Edition** (EE) package](https://about.gitlab.com/install/?version=ee): Both Redis and Sentinel - are bundled in the package, so you can use the EE package to set up the whole - Redis HA infrastructure (master, replica and Sentinel) which is described in - this document. -- If you have installed GitLab using the Omnibus GitLab packages (CE or EE), - but you want to use your own external Redis server, follow steps 1-3 in the - [Redis HA installation from source](redis_source.md) documentation, then go - straight to step 4 in this guide to - [set up the GitLab application](#step-4-configuring-the-gitlab-application). - -## Configuring Redis HA - -This is the section where we install and set up the new Redis instances. - -> **Notes:** -> -> - We assume that you have installed GitLab and all HA components from scratch. If you -> already have it installed and running, read how to -> [switch from a single-machine installation to Redis HA](#switching-from-an-existing-single-machine-installation-to-redis-ha). -> - Redis nodes (both master and replica) will need the same password defined in -> `redis['password']`. At any time during a failover the Sentinels can -> reconfigure a node and change its status from master to replica and vice versa. - -### Prerequisites - -The prerequisites for a HA Redis setup are the following: - -1. Provision the minimum required number of instances as specified in the - [recommended setup](#recommended-setup) section. -1. We **Do not** recommend installing Redis or Redis Sentinel in the same machines your - GitLab application is running on as this weakens your HA configuration. You can however opt in to install Redis - and Sentinel in the same machine. -1. All Redis nodes must be able to talk to each other and accept incoming - connections over Redis (`6379`) and Sentinel (`26379`) ports (unless you - change the default ones). -1. The server that hosts the GitLab application must be able to access the - Redis nodes. -1. Protect the nodes from access from external networks ([Internet](https://gitlab.com/gitlab-org/gitlab-foss/uploads/c4cc8cd353604bd80315f9384035ff9e/The_Internet_IT_Crowd.png)), using - firewall. - -### Step 1. Configuring the master Redis instance - -1. SSH into the **master** Redis server. -1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab - package you want using **steps 1 and 2** from the GitLab downloads page. - - Make sure you select the correct Omnibus package, with the same version - and type (Community, Enterprise editions) of your current install. - - Do not complete any other steps on the download page. - -1. Edit `/etc/gitlab/gitlab.rb` and add the contents: - - ```ruby - # Specify server role as 'redis_master_role' - roles ['redis_master_role'] - - # IP address pointing to a local IP that the other machines can reach to. - # You can also set bind to '0.0.0.0' which listen in all interfaces. - # If you really need to bind to an external accessible IP, make - # sure you add extra firewall rules to prevent unauthorized access. - redis['bind'] = '10.0.0.1' - - # Define a port so Redis can listen for TCP requests which will allow other - # machines to connect to it. - redis['port'] = 6379 - - # Set up password authentication for Redis (use the same password in all nodes). - redis['password'] = 'redis-password-goes-here' - ``` - -1. Only the primary GitLab application server should handle migrations. To - prevent database migrations from running on upgrade, add the following - configuration to your `/etc/gitlab/gitlab.rb` file: - - ```ruby - gitlab_rails['auto_migrate'] = false - ``` - -1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. - -> Note: You can specify multiple roles like sentinel and Redis as: -> `roles ['redis_sentinel_role', 'redis_master_role']`. Read more about high -> availability roles at <https://docs.gitlab.com/omnibus/roles/>. - -### Step 2. Configuring the replica Redis instances - -1. SSH into the **replica** Redis server. -1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab - package you want using **steps 1 and 2** from the GitLab downloads page. - - Make sure you select the correct Omnibus package, with the same version - and type (Community, Enterprise editions) of your current install. - - Do not complete any other steps on the download page. - -1. Edit `/etc/gitlab/gitlab.rb` and add the contents: - - ```ruby - # Specify server role as 'redis_replica_role' - roles ['redis_replica_role'] - - # IP address pointing to a local IP that the other machines can reach to. - # You can also set bind to '0.0.0.0' which listen in all interfaces. - # If you really need to bind to an external accessible IP, make - # sure you add extra firewall rules to prevent unauthorized access. - redis['bind'] = '10.0.0.2' - - # Define a port so Redis can listen for TCP requests which will allow other - # machines to connect to it. - redis['port'] = 6379 - - # The same password for Redis authentication you set up for the master node. - redis['password'] = 'redis-password-goes-here' - - # The IP of the master Redis node. - redis['master_ip'] = '10.0.0.1' - - # Port of master Redis server, uncomment to change to non default. Defaults - # to `6379`. - #redis['master_port'] = 6379 - ``` - -1. To prevent reconfigure from running automatically on upgrade, run: - - ```shell - sudo touch /etc/gitlab/skip-auto-reconfigure - ``` - -1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. -1. Go through the steps again for all the other replica nodes. - -> Note: You can specify multiple roles like sentinel and Redis as: -> `roles ['redis_sentinel_role', 'redis_replica_role']`. Read more about high -> availability roles at <https://docs.gitlab.com/omnibus/roles/>. - ---- - -These values don't have to be changed again in `/etc/gitlab/gitlab.rb` after -a failover, as the nodes will be managed by the Sentinels, and even after a -`gitlab-ctl reconfigure`, they will get their configuration restored by -the same Sentinels. - -### Step 3. Configuring the Redis Sentinel instances - ->**Note:** -Redis Sentinel is bundled with Omnibus GitLab Enterprise Edition only. The -following section assumes you are using Omnibus GitLab Enterprise Edition. -For the Omnibus Community Edition and installations from source, follow the -[Redis HA source install](redis_source.md) guide. - -NOTE: **Note:** If you are using an external Redis Sentinel instance, be sure -to exclude the `requirepass` parameter from the Sentinel -configuration. This parameter will cause clients to report `NOAUTH -Authentication required.`. [Redis Sentinel 3.2.x does not support -password authentication](https://github.com/antirez/redis/issues/3279). - -Now that the Redis servers are all set up, let's configure the Sentinel -servers. - -If you are not sure if your Redis servers are working and replicating -correctly, please read the [Troubleshooting Replication](#troubleshooting-redis-replication) -and fix it before proceeding with Sentinel setup. - -You must have at least `3` Redis Sentinel servers, and they need to -be each in an independent machine. You can configure them in the same -machines where you've configured the other Redis servers. - -With GitLab Enterprise Edition, you can use the Omnibus package to set up -multiple machines with the Sentinel daemon. - ---- - -1. SSH into the server that will host Redis Sentinel. -1. **You can omit this step if the Sentinels will be hosted in the same node as - the other Redis instances.** - - [Download/install](https://about.gitlab.com/install/) the - Omnibus GitLab Enterprise Edition package using **steps 1 and 2** from the - GitLab downloads page. - - Make sure you select the correct Omnibus package, with the same version - the GitLab application is running. - - Do not complete any other steps on the download page. - -1. Edit `/etc/gitlab/gitlab.rb` and add the contents (if you are installing the - Sentinels in the same node as the other Redis instances, some values might - be duplicate below): - - ```ruby - roles ['redis_sentinel_role'] - - # Must be the same in every sentinel node - redis['master_name'] = 'gitlab-redis' - - # The same password for Redis authentication you set up for the master node. - redis['master_password'] = 'redis-password-goes-here' - - # The IP of the master Redis node. - redis['master_ip'] = '10.0.0.1' - - # Define a port so Redis can listen for TCP requests which will allow other - # machines to connect to it. - redis['port'] = 6379 - - # Port of master Redis server, uncomment to change to non default. Defaults - # to `6379`. - #redis['master_port'] = 6379 - - ## Configure Sentinel - sentinel['bind'] = '10.0.0.1' - - # Port that Sentinel listens on, uncomment to change to non default. Defaults - # to `26379`. - # sentinel['port'] = 26379 - - ## Quorum must reflect the amount of voting sentinels it take to start a failover. - ## Value must NOT be greater then the amount of sentinels. - ## - ## The quorum can be used to tune Sentinel in two ways: - ## 1. If a the quorum is set to a value smaller than the majority of Sentinels - ## we deploy, we are basically making Sentinel more sensible to master failures, - ## triggering a failover as soon as even just a minority of Sentinels is no longer - ## able to talk with the master. - ## 1. If a quorum is set to a value greater than the majority of Sentinels, we are - ## making Sentinel able to failover only when there are a very large number (larger - ## than majority) of well connected Sentinels which agree about the master being down.s - sentinel['quorum'] = 2 - - ## Consider unresponsive server down after x amount of ms. - # sentinel['down_after_milliseconds'] = 10000 - - ## Specifies the failover timeout in milliseconds. It is used in many ways: - ## - ## - The time needed to re-start a failover after a previous failover was - ## already tried against the same master by a given Sentinel, is two - ## times the failover timeout. - ## - ## - The time needed for a replica replicating to a wrong master according - ## to a Sentinel current configuration, to be forced to replicate - ## with the right master, is exactly the failover timeout (counting since - ## the moment a Sentinel detected the misconfiguration). - ## - ## - The time needed to cancel a failover that is already in progress but - ## did not produced any configuration change (REPLICAOF NO ONE yet not - ## acknowledged by the promoted replica). - ## - ## - The maximum time a failover in progress waits for all the replica to be - ## reconfigured as replicas of the new master. However even after this time - ## the replicas will be reconfigured by the Sentinels anyway, but not with - ## the exact parallel-syncs progression as specified. - # sentinel['failover_timeout'] = 60000 - ``` - -1. To prevent database migrations from running on upgrade, run: - - ```shell - sudo touch /etc/gitlab/skip-auto-reconfigure - ``` - - Only the primary GitLab application server should handle migrations. - -1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. -1. Go through the steps again for all the other Sentinel nodes. - -### Step 4. Configuring the GitLab application - -The final part is to inform the main GitLab application server of the Redis -Sentinels servers and authentication credentials. - -You can enable or disable Sentinel support at any time in new or existing -installations. From the GitLab application perspective, all it requires is -the correct credentials for the Sentinel nodes. - -While it doesn't require a list of all Sentinel nodes, in case of a failure, -it needs to access at least one of the listed. - ->**Note:** -The following steps should be performed in the [GitLab application server](gitlab.md) -which ideally should not have Redis or Sentinels on it for a HA setup. - -1. SSH into the server where the GitLab application is installed. -1. Edit `/etc/gitlab/gitlab.rb` and add/change the following lines: - - ```ruby - ## Must be the same in every sentinel node - redis['master_name'] = 'gitlab-redis' - - ## The same password for Redis authentication you set up for the master node. - redis['master_password'] = 'redis-password-goes-here' - - ## A list of sentinels with `host` and `port` - gitlab_rails['redis_sentinels'] = [ - {'host' => '10.0.0.1', 'port' => 26379}, - {'host' => '10.0.0.2', 'port' => 26379}, - {'host' => '10.0.0.3', 'port' => 26379} - ] - ``` - -1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. - -## Switching from an existing single-machine installation to Redis HA - -If you already have a single-machine GitLab install running, you will need to -replicate from this machine first, before de-activating the Redis instance -inside it. - -Your single-machine install will be the initial **Master**, and the `3` others -should be configured as **Replica** pointing to this machine. - -After replication catches up, you will need to stop services in the -single-machine install, to rotate the **Master** to one of the new nodes. - -Make the required changes in configuration and restart the new nodes again. - -To disable Redis in the single install, edit `/etc/gitlab/gitlab.rb`: - -```ruby -redis['enable'] = false -``` - -If you fail to replicate first, you may loose data (unprocessed background jobs). - -## Example of a minimal configuration with 1 master, 2 replicas and 3 Sentinels - ->**Note:** -Redis Sentinel is bundled with Omnibus GitLab Enterprise Edition only. For -different setups, read the -[available configuration setups](#available-configuration-setups) section. - -In this example we consider that all servers have an internal network -interface with IPs in the `10.0.0.x` range, and that they can connect -to each other using these IPs. - -In a real world usage, you would also set up firewall rules to prevent -unauthorized access from other machines and block traffic from the -outside (Internet). - -We will use the same `3` nodes with **Redis** + **Sentinel** topology -discussed in [Redis setup overview](#redis-setup-overview) and -[Sentinel setup overview](#sentinel-setup-overview) documentation. - -Here is a list and description of each **machine** and the assigned **IP**: - -- `10.0.0.1`: Redis Master + Sentinel 1 -- `10.0.0.2`: Redis Replica 1 + Sentinel 2 -- `10.0.0.3`: Redis Replica 2 + Sentinel 3 -- `10.0.0.4`: GitLab application - -Please note that after the initial configuration, if a failover is initiated -by the Sentinel nodes, the Redis nodes will be reconfigured and the **Master** -will change permanently (including in `redis.conf`) from one node to the other, -until a new failover is initiated again. - -The same thing will happen with `sentinel.conf` that will be overridden after the -initial execution, after any new sentinel node starts watching the **Master**, -or a failover promotes a different **Master** node. - -### Example configuration for Redis master and Sentinel 1 - -In `/etc/gitlab/gitlab.rb`: - -```ruby -roles ['redis_sentinel_role', 'redis_master_role'] -redis['bind'] = '10.0.0.1' -redis['port'] = 6379 -redis['password'] = 'redis-password-goes-here' -redis['master_name'] = 'gitlab-redis' # must be the same in every sentinel node -redis['master_password'] = 'redis-password-goes-here' # the same value defined in redis['password'] in the master instance -redis['master_ip'] = '10.0.0.1' # ip of the initial master redis instance -#redis['master_port'] = 6379 # port of the initial master redis instance, uncomment to change to non default -sentinel['bind'] = '10.0.0.1' -# sentinel['port'] = 26379 # uncomment to change default port -sentinel['quorum'] = 2 -# sentinel['down_after_milliseconds'] = 10000 -# sentinel['failover_timeout'] = 60000 -``` - -[Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. - -### Example configuration for Redis replica 1 and Sentinel 2 - -In `/etc/gitlab/gitlab.rb`: - -```ruby -roles ['redis_sentinel_role', 'redis_replica_role'] -redis['bind'] = '10.0.0.2' -redis['port'] = 6379 -redis['password'] = 'redis-password-goes-here' -redis['master_password'] = 'redis-password-goes-here' -redis['master_ip'] = '10.0.0.1' # IP of master Redis server -#redis['master_port'] = 6379 # Port of master Redis server, uncomment to change to non default -redis['master_name'] = 'gitlab-redis' # must be the same in every sentinel node -sentinel['bind'] = '10.0.0.2' -# sentinel['port'] = 26379 # uncomment to change default port -sentinel['quorum'] = 2 -# sentinel['down_after_milliseconds'] = 10000 -# sentinel['failover_timeout'] = 60000 -``` - -[Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. - -### Example configuration for Redis replica 2 and Sentinel 3 - -In `/etc/gitlab/gitlab.rb`: - -```ruby -roles ['redis_sentinel_role', 'redis_replica_role'] -redis['bind'] = '10.0.0.3' -redis['port'] = 6379 -redis['password'] = 'redis-password-goes-here' -redis['master_password'] = 'redis-password-goes-here' -redis['master_ip'] = '10.0.0.1' # IP of master Redis server -#redis['master_port'] = 6379 # Port of master Redis server, uncomment to change to non default -redis['master_name'] = 'gitlab-redis' # must be the same in every sentinel node -sentinel['bind'] = '10.0.0.3' -# sentinel['port'] = 26379 # uncomment to change default port -sentinel['quorum'] = 2 -# sentinel['down_after_milliseconds'] = 10000 -# sentinel['failover_timeout'] = 60000 -``` - -[Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. - -### Example configuration for the GitLab application - -In `/etc/gitlab/gitlab.rb`: - -```ruby -redis['master_name'] = 'gitlab-redis' -redis['master_password'] = 'redis-password-goes-here' -gitlab_rails['redis_sentinels'] = [ - {'host' => '10.0.0.1', 'port' => 26379}, - {'host' => '10.0.0.2', 'port' => 26379}, - {'host' => '10.0.0.3', 'port' => 26379} -] -``` - -[Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. - -## Enable Monitoring - -> [Introduced](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/3786) in GitLab 12.0. - -If you enable Monitoring, it must be enabled on **all** Redis servers. - -1. Make sure to collect [`CONSUL_SERVER_NODES`](../postgresql/replication_and_failover.md#consul-information), which are the IP addresses or DNS records of the Consul server nodes, for the next step. Note they are presented as `Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z` - -1. Create/edit `/etc/gitlab/gitlab.rb` and add the following configuration: - - ```ruby - # Enable service discovery for Prometheus - consul['enable'] = true - consul['monitoring_service_discovery'] = true - - # Replace placeholders - # Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z - # with the addresses of the Consul server nodes - consul['configuration'] = { - retry_join: %w(Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z), - } - - # Set the network addresses that the exporters will listen on - node_exporter['listen_address'] = '0.0.0.0:9100' - redis_exporter['listen_address'] = '0.0.0.0:9121' - ``` - -1. Run `sudo gitlab-ctl reconfigure` to compile the configuration. - -## Advanced configuration - -Omnibus GitLab configures some things behind the curtains to make the sysadmins' -lives easier. If you want to know what happens underneath keep reading. - -### Running multiple Redis clusters - -GitLab supports running [separate Redis clusters for different persistent -classes](https://docs.gitlab.com/omnibus/settings/redis.html#running-with-multiple-redis-instances): -cache, queues, and shared_state. To make this work with Sentinel: - -1. Set the appropriate variable in `/etc/gitlab/gitlab.rb` for each instance you are using: - - ```ruby - gitlab_rails['redis_cache_instance'] = REDIS_CACHE_URL - gitlab_rails['redis_queues_instance'] = REDIS_QUEUES_URL - gitlab_rails['redis_shared_state_instance'] = REDIS_SHARED_STATE_URL - ``` - - **Note**: Redis URLs should be in the format: `redis://:PASSWORD@SENTINEL_MASTER_NAME` - - 1. PASSWORD is the plaintext password for the Redis instance - 1. SENTINEL_MASTER_NAME is the Sentinel master name (e.g. `gitlab-redis-cache`) - -1. Include an array of hashes with host/port combinations, such as the following: - - ```ruby - gitlab_rails['redis_cache_sentinels'] = [ - { host: REDIS_CACHE_SENTINEL_HOST, port: PORT1 }, - { host: REDIS_CACHE_SENTINEL_HOST2, port: PORT2 } - ] - gitlab_rails['redis_queues_sentinels'] = [ - { host: REDIS_QUEUES_SENTINEL_HOST, port: PORT1 }, - { host: REDIS_QUEUES_SENTINEL_HOST2, port: PORT2 } - ] - gitlab_rails['redis_shared_state_sentinels'] = [ - { host: SHARED_STATE_SENTINEL_HOST, port: PORT1 }, - { host: SHARED_STATE_SENTINEL_HOST2, port: PORT2 } - ] - ``` - -1. Note that for each persistence class, GitLab will default to using the - configuration specified in `gitlab_rails['redis_sentinels']` unless - overridden by the settings above. -1. Be sure to include BOTH configuration options for each persistent classes. For example, - if you choose to configure a cache instance, you must specify both `gitlab_rails['redis_cache_instance']` - and `gitlab_rails['redis_cache_sentinels']` for GitLab to generate the proper configuration files. -1. Run `gitlab-ctl reconfigure` - -### Control running services - -In the previous example, we've used `redis_sentinel_role` and -`redis_master_role` which simplifies the amount of configuration changes. - -If you want more control, here is what each one sets for you automatically -when enabled: - -```ruby -## Redis Sentinel Role -redis_sentinel_role['enable'] = true - -# When Sentinel Role is enabled, the following services are also enabled -sentinel['enable'] = true - -# The following services are disabled -redis['enable'] = false -bootstrap['enable'] = false -nginx['enable'] = false -postgresql['enable'] = false -gitlab_rails['enable'] = false -mailroom['enable'] = false - -------- - -## Redis master/replica Role -redis_master_role['enable'] = true # enable only one of them -redis_replica_role['enable'] = true # enable only one of them - -# When Redis Master or Replica role are enabled, the following services are -# enabled/disabled. Note that if Redis and Sentinel roles are combined, both -# services will be enabled. - -# The following services are disabled -sentinel['enable'] = false -bootstrap['enable'] = false -nginx['enable'] = false -postgresql['enable'] = false -gitlab_rails['enable'] = false -mailroom['enable'] = false - -# For Redis Replica role, also change this setting from default 'true' to 'false': -redis['master'] = false -``` - -You can find the relevant attributes defined in [`gitlab_rails.rb`](https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/libraries/gitlab_rails.rb). - -## Troubleshooting - -There are a lot of moving parts that needs to be taken care carefully -in order for the HA setup to work as expected. - -Before proceeding with the troubleshooting below, check your firewall rules: - -- Redis machines - - Accept TCP connection in `6379` - - Connect to the other Redis machines via TCP in `6379` -- Sentinel machines - - Accept TCP connection in `26379` - - Connect to other Sentinel machines via TCP in `26379` - - Connect to the Redis machines via TCP in `6379` - -### Troubleshooting Redis replication - -You can check if everything is correct by connecting to each server using -`redis-cli` application, and sending the `info replication` command as below. - -```shell -/opt/gitlab/embedded/bin/redis-cli -h <redis-host-or-ip> -a '<redis-password>' info replication -``` - -When connected to a `master` Redis, you will see the number of connected -`replicas`, and a list of each with connection details: - -```plaintext -# Replication -role:master -connected_replicas:1 -replica0:ip=10.133.5.21,port=6379,state=online,offset=208037514,lag=1 -master_repl_offset:208037658 -repl_backlog_active:1 -repl_backlog_size:1048576 -repl_backlog_first_byte_offset:206989083 -repl_backlog_histlen:1048576 -``` - -When it's a `replica`, you will see details of the master connection and if -its `up` or `down`: - -```plaintext -# Replication -role:replica -master_host:10.133.1.58 -master_port:6379 -master_link_status:up -master_last_io_seconds_ago:1 -master_sync_in_progress:0 -replica_repl_offset:208096498 -replica_priority:100 -replica_read_only:1 -connected_replicas:0 -master_repl_offset:0 -repl_backlog_active:0 -repl_backlog_size:1048576 -repl_backlog_first_byte_offset:0 -repl_backlog_histlen:0 -``` - -### Troubleshooting Sentinel - -If you get an error like: `Redis::CannotConnectError: No sentinels available.`, -there may be something wrong with your configuration files or it can be related -to [this issue](https://github.com/redis/redis-rb/issues/531). - -You must make sure you are defining the same value in `redis['master_name']` -and `redis['master_pasword']` as you defined for your sentinel node. - -The way the Redis connector `redis-rb` works with sentinel is a bit -non-intuitive. We try to hide the complexity in omnibus, but it still requires -a few extra configurations. - ---- - -To make sure your configuration is correct: - -1. SSH into your GitLab application server -1. Enter the Rails console: - - ```shell - # For Omnibus installations - sudo gitlab-rails console - - # For source installations - sudo -u git rails console -e production - ``` - -1. Run in the console: - - ```ruby - redis = Redis.new(Gitlab::Redis::SharedState.params) - redis.info - ``` - - Keep this screen open and try to simulate a failover below. - -1. To simulate a failover on master Redis, SSH into the Redis server and run: - - ```shell - # port must match your master redis port, and the sleep time must be a few seconds bigger than defined one - redis-cli -h localhost -p 6379 DEBUG sleep 20 - ``` - -1. Then back in the Rails console from the first step, run: - - ```ruby - redis.info - ``` - - You should see a different port after a few seconds delay - (the failover/reconnect time). - -## Changelog - -Changes to Redis HA over time. - -**8.14** - -- Redis Sentinel support is production-ready and bundled in the Omnibus GitLab - Enterprise Edition package -- Documentation restructure for better readability - -**8.11** - -- Experimental Redis Sentinel support was added - -## Further reading - -Read more on High Availability: - -1. [High Availability Overview](README.md) -1. [Configure the database](../postgresql/replication_and_failover.md) -1. [Configure NFS](nfs.md) -1. [Configure the GitLab application servers](gitlab.md) -1. [Configure the load balancers](load_balancer.md) +This document was moved to [another location](../redis/index.md). diff --git a/doc/administration/high_availability/redis_source.md b/doc/administration/high_availability/redis_source.md index 97be480bc3b..75496638979 100644 --- a/doc/administration/high_availability/redis_source.md +++ b/doc/administration/high_availability/redis_source.md @@ -1,371 +1,5 @@ --- -type: reference +redirect_to: ../redis/replication_and_failover_external.md --- -# Configuring non-Omnibus Redis for GitLab HA - -This is the documentation for configuring a Highly Available Redis setup when -you have installed Redis all by yourself and not using the bundled one that -comes with the Omnibus packages. - -Note also that you may elect to override all references to -`/home/git/gitlab/config/resque.yml` in accordance with the advanced Redis -settings outlined in -[Configuration Files Documentation](https://gitlab.com/gitlab-org/gitlab/blob/master/config/README.md). - -We cannot stress enough the importance of reading the -[Overview section](redis.md#overview) of the Omnibus Redis HA as it provides -some invaluable information to the configuration of Redis. Please proceed to -read it before going forward with this guide. - -We also highly recommend that you use the Omnibus GitLab packages, as we -optimize them specifically for GitLab, and we will take care of upgrading Redis -to the latest supported version. - -If you're not sure whether this guide is for you, please refer to -[Available configuration setups](redis.md#available-configuration-setups) in -the Omnibus Redis HA documentation. - -## Configuring your own Redis server - -This is the section where we install and set up the new Redis instances. - -### Prerequisites - -- All Redis servers in this guide must be configured to use a TCP connection - instead of a socket. To configure Redis to use TCP connections you need to - define both `bind` and `port` in the Redis config file. You can bind to all - interfaces (`0.0.0.0`) or specify the IP of the desired interface - (e.g., one from an internal network). -- Since Redis 3.2, you must define a password to receive external connections - (`requirepass`). -- If you are using Redis with Sentinel, you will also need to define the same - password for the replica password definition (`masterauth`) in the same instance. - -In addition, read the prerequisites as described in the -[Omnibus Redis HA document](redis.md#prerequisites) since they provide some -valuable information for the general setup. - -### Step 1. Configuring the master Redis instance - -Assuming that the Redis master instance IP is `10.0.0.1`: - -1. [Install Redis](../../install/installation.md#7-redis). -1. Edit `/etc/redis/redis.conf`: - - ```conf - ## Define a `bind` address pointing to a local IP that your other machines - ## can reach you. If you really need to bind to an external accessible IP, make - ## sure you add extra firewall rules to prevent unauthorized access: - bind 10.0.0.1 - - ## Define a `port` to force redis to listen on TCP so other machines can - ## connect to it (default port is `6379`). - port 6379 - - ## Set up password authentication (use the same password in all nodes). - ## The password should be defined equal for both `requirepass` and `masterauth` - ## when setting up Redis to use with Sentinel. - requirepass redis-password-goes-here - masterauth redis-password-goes-here - ``` - -1. Restart the Redis service for the changes to take effect. - -### Step 2. Configuring the replica Redis instances - -Assuming that the Redis replica instance IP is `10.0.0.2`: - -1. [Install Redis](../../install/installation.md#7-redis). -1. Edit `/etc/redis/redis.conf`: - - ```conf - ## Define a `bind` address pointing to a local IP that your other machines - ## can reach you. If you really need to bind to an external accessible IP, make - ## sure you add extra firewall rules to prevent unauthorized access: - bind 10.0.0.2 - - ## Define a `port` to force redis to listen on TCP so other machines can - ## connect to it (default port is `6379`). - port 6379 - - ## Set up password authentication (use the same password in all nodes). - ## The password should be defined equal for both `requirepass` and `masterauth` - ## when setting up Redis to use with Sentinel. - requirepass redis-password-goes-here - masterauth redis-password-goes-here - - ## Define `replicaof` pointing to the Redis master instance with IP and port. - replicaof 10.0.0.1 6379 - ``` - -1. Restart the Redis service for the changes to take effect. -1. Go through the steps again for all the other replica nodes. - -### Step 3. Configuring the Redis Sentinel instances - -Sentinel is a special type of Redis server. It inherits most of the basic -configuration options you can define in `redis.conf`, with specific ones -starting with `sentinel` prefix. - -Assuming that the Redis Sentinel is installed on the same instance as Redis -master with IP `10.0.0.1` (some settings might overlap with the master): - -1. [Install Redis Sentinel](https://redis.io/topics/sentinel) -1. Edit `/etc/redis/sentinel.conf`: - - ```conf - ## Define a `bind` address pointing to a local IP that your other machines - ## can reach you. If you really need to bind to an external accessible IP, make - ## sure you add extra firewall rules to prevent unauthorized access: - bind 10.0.0.1 - - ## Define a `port` to force Sentinel to listen on TCP so other machines can - ## connect to it (default port is `6379`). - port 26379 - - ## Set up password authentication (use the same password in all nodes). - ## The password should be defined equal for both `requirepass` and `masterauth` - ## when setting up Redis to use with Sentinel. - requirepass redis-password-goes-here - masterauth redis-password-goes-here - - ## Define with `sentinel auth-pass` the same shared password you have - ## defined for both Redis master and replicas instances. - sentinel auth-pass gitlab-redis redis-password-goes-here - - ## Define with `sentinel monitor` the IP and port of the Redis - ## master node, and the quorum required to start a failover. - sentinel monitor gitlab-redis 10.0.0.1 6379 2 - - ## Define with `sentinel down-after-milliseconds` the time in `ms` - ## that an unresponsive server will be considered down. - sentinel down-after-milliseconds gitlab-redis 10000 - - ## Define a value for `sentinel failover_timeout` in `ms`. This has multiple - ## meanings: - ## - ## * The time needed to re-start a failover after a previous failover was - ## already tried against the same master by a given Sentinel, is two - ## times the failover timeout. - ## - ## * The time needed for a replica replicating to a wrong master according - ## to a Sentinel current configuration, to be forced to replicate - ## with the right master, is exactly the failover timeout (counting since - ## the moment a Sentinel detected the misconfiguration). - ## - ## * The time needed to cancel a failover that is already in progress but - ## did not produced any configuration change (REPLICAOF NO ONE yet not - ## acknowledged by the promoted replica). - ## - ## * The maximum time a failover in progress waits for all the replicas to be - ## reconfigured as replicas of the new master. However even after this time - ## the replicas will be reconfigured by the Sentinels anyway, but not with - ## the exact parallel-syncs progression as specified. - sentinel failover_timeout 30000 - ``` - -1. Restart the Redis service for the changes to take effect. -1. Go through the steps again for all the other Sentinel nodes. - -### Step 4. Configuring the GitLab application - -You can enable or disable Sentinel support at any time in new or existing -installations. From the GitLab application perspective, all it requires is -the correct credentials for the Sentinel nodes. - -While it doesn't require a list of all Sentinel nodes, in case of a failure, -it needs to access at least one of listed ones. - -The following steps should be performed in the [GitLab application server](gitlab.md) -which ideally should not have Redis or Sentinels in the same machine for a HA -setup: - -1. Edit `/home/git/gitlab/config/resque.yml` following the example in - [resque.yml.example](https://gitlab.com/gitlab-org/gitlab/blob/master/config/resque.yml.example), and uncomment the Sentinel lines, pointing to - the correct server credentials: - - ```yaml - # resque.yaml - production: - url: redis://:redi-password-goes-here@gitlab-redis/ - sentinels: - - - host: 10.0.0.1 - port: 26379 # point to sentinel, not to redis port - - - host: 10.0.0.2 - port: 26379 # point to sentinel, not to redis port - - - host: 10.0.0.3 - port: 26379 # point to sentinel, not to redis port - ``` - -1. [Restart GitLab](../restart_gitlab.md#installations-from-source) for the changes to take effect. - -## Example of minimal configuration with 1 master, 2 replicas and 3 Sentinels - -In this example we consider that all servers have an internal network -interface with IPs in the `10.0.0.x` range, and that they can connect -to each other using these IPs. - -In a real world usage, you would also set up firewall rules to prevent -unauthorized access from other machines, and block traffic from the -outside ([Internet](https://gitlab.com/gitlab-org/gitlab-foss/uploads/c4cc8cd353604bd80315f9384035ff9e/The_Internet_IT_Crowd.png)). - -For this example, **Sentinel 1** will be configured in the same machine as the -**Redis Master**, **Sentinel 2** and **Sentinel 3** in the same machines as the -**Replica 1** and **Replica 2** respectively. - -Here is a list and description of each **machine** and the assigned **IP**: - -- `10.0.0.1`: Redis Master + Sentinel 1 -- `10.0.0.2`: Redis Replica 1 + Sentinel 2 -- `10.0.0.3`: Redis Replica 2 + Sentinel 3 -- `10.0.0.4`: GitLab application - -Please note that after the initial configuration, if a failover is initiated -by the Sentinel nodes, the Redis nodes will be reconfigured and the **Master** -will change permanently (including in `redis.conf`) from one node to the other, -until a new failover is initiated again. - -The same thing will happen with `sentinel.conf` that will be overridden after the -initial execution, after any new sentinel node starts watching the **Master**, -or a failover promotes a different **Master** node. - -### Example configuration for Redis master and Sentinel 1 - -1. In `/etc/redis/redis.conf`: - - ```conf - bind 10.0.0.1 - port 6379 - requirepass redis-password-goes-here - masterauth redis-password-goes-here - ``` - -1. In `/etc/redis/sentinel.conf`: - - ```conf - bind 10.0.0.1 - port 26379 - sentinel auth-pass gitlab-redis redis-password-goes-here - sentinel monitor gitlab-redis 10.0.0.1 6379 2 - sentinel down-after-milliseconds gitlab-redis 10000 - sentinel failover_timeout 30000 - ``` - -1. Restart the Redis service for the changes to take effect. - -### Example configuration for Redis replica 1 and Sentinel 2 - -1. In `/etc/redis/redis.conf`: - - ```conf - bind 10.0.0.2 - port 6379 - requirepass redis-password-goes-here - masterauth redis-password-goes-here - replicaof 10.0.0.1 6379 - ``` - -1. In `/etc/redis/sentinel.conf`: - - ```conf - bind 10.0.0.2 - port 26379 - sentinel auth-pass gitlab-redis redis-password-goes-here - sentinel monitor gitlab-redis 10.0.0.1 6379 2 - sentinel down-after-milliseconds gitlab-redis 10000 - sentinel failover_timeout 30000 - ``` - -1. Restart the Redis service for the changes to take effect. - -### Example configuration for Redis replica 2 and Sentinel 3 - -1. In `/etc/redis/redis.conf`: - - ```conf - bind 10.0.0.3 - port 6379 - requirepass redis-password-goes-here - masterauth redis-password-goes-here - replicaof 10.0.0.1 6379 - ``` - -1. In `/etc/redis/sentinel.conf`: - - ```conf - bind 10.0.0.3 - port 26379 - sentinel auth-pass gitlab-redis redis-password-goes-here - sentinel monitor gitlab-redis 10.0.0.1 6379 2 - sentinel down-after-milliseconds gitlab-redis 10000 - sentinel failover_timeout 30000 - ``` - -1. Restart the Redis service for the changes to take effect. - -### Example configuration of the GitLab application - -1. Edit `/home/git/gitlab/config/resque.yml`: - - ```yaml - production: - url: redis://:redi-password-goes-here@gitlab-redis/ - sentinels: - - - host: 10.0.0.1 - port: 26379 # point to sentinel, not to redis port - - - host: 10.0.0.2 - port: 26379 # point to sentinel, not to redis port - - - host: 10.0.0.3 - port: 26379 # point to sentinel, not to redis port - ``` - -1. [Restart GitLab](../restart_gitlab.md#installations-from-source) for the changes to take effect. - -## Troubleshooting - -We have a more detailed [Troubleshooting](redis.md#troubleshooting) explained -in the documentation for Omnibus GitLab installations. Here we will list only -the things that are specific to a source installation. - -If you get an error in GitLab like `Redis::CannotConnectError: No sentinels available.`, -there may be something wrong with your configuration files or it can be related -to [this upstream issue](https://github.com/redis/redis-rb/issues/531). - -You must make sure that `resque.yml` and `sentinel.conf` are configured correctly, -otherwise `redis-rb` will not work properly. - -The `master-group-name` (`gitlab-redis`) defined in (`sentinel.conf`) -**must** be used as the hostname in GitLab (`resque.yml`): - -```conf -# sentinel.conf: -sentinel monitor gitlab-redis 10.0.0.1 6379 2 -sentinel down-after-milliseconds gitlab-redis 10000 -sentinel config-epoch gitlab-redis 0 -sentinel leader-epoch gitlab-redis 0 -``` - -```yaml -# resque.yaml -production: - url: redis://:myredispassword@gitlab-redis/ - sentinels: - - - host: 10.0.0.1 - port: 26379 # point to sentinel, not to redis port - - - host: 10.0.0.2 - port: 26379 # point to sentinel, not to redis port - - - host: 10.0.0.3 - port: 26379 # point to sentinel, not to redis port -``` - -When in doubt, please read [Redis Sentinel documentation](https://redis.io/topics/sentinel). +This document was moved to [another location](../redis/replication_and_failover_external.md). diff --git a/doc/administration/high_availability/sidekiq.md b/doc/administration/high_availability/sidekiq.md index 493929dcf3b..98a9af64e5e 100644 --- a/doc/administration/high_availability/sidekiq.md +++ b/doc/administration/high_availability/sidekiq.md @@ -94,7 +94,8 @@ you want using steps 1 and 2 from the GitLab downloads page. 1. Run `gitlab-ctl reconfigure`. -NOTE: **Note:** You will need to restart the Sidekiq nodes after an update has occurred and database +NOTE: **Note:** +You will need to restart the Sidekiq nodes after an update has occurred and database migrations performed. ## Example configuration diff --git a/doc/administration/img/repository_storages_admin_ui_v12_10.png b/doc/administration/img/repository_storages_admin_ui_v12_10.png Binary files differdeleted file mode 100644 index e5ac09524b8..00000000000 --- a/doc/administration/img/repository_storages_admin_ui_v12_10.png +++ /dev/null diff --git a/doc/administration/img/repository_storages_admin_ui_v13_1.png b/doc/administration/img/repository_storages_admin_ui_v13_1.png Binary files differnew file mode 100644 index 00000000000..f8c13a35369 --- /dev/null +++ b/doc/administration/img/repository_storages_admin_ui_v13_1.png diff --git a/doc/administration/incoming_email.md b/doc/administration/incoming_email.md index e078a7c1098..36156c4a580 100644 --- a/doc/administration/incoming_email.md +++ b/doc/administration/incoming_email.md @@ -17,10 +17,15 @@ GitLab has several features based on receiving incoming emails: allow GitLab users to create a new merge request by sending an email to a user-specific email address. - [Service Desk](../user/project/service_desk.md): provide e-mail support to - your customers through GitLab. **(PREMIUM)** + your customers through GitLab. ## Requirements +NOTE: **Note:** +It is **not** recommended to use an email address that receives or will receive any +messages not intended for the GitLab instance. Any incoming emails not intended +for GitLab will receive a reject notice. + Handling incoming emails requires an [IMAP](https://en.wikipedia.org/wiki/Internet_Message_Access_Protocol)-enabled email account. GitLab requires one of the following three strategies: @@ -69,6 +74,11 @@ and [allowed less secure apps to access the account](https://support.google.com/ or [turn-on 2-step validation](https://support.google.com/accounts/answer/185839) and use [an application password](https://support.google.com/mail/answer/185833). +If you want to use Office 365, and two-factor authentication is enabled, make sure +you're using an +[app password](https://docs.microsoft.com/en-us/azure/active-directory/user-help/multi-factor-authentication-end-user-app-passwords) +instead of the regular password for the mailbox. + To set up a basic Postfix mail server with IMAP access on Ubuntu, follow the [Postfix setup documentation](reply_by_email_postfix_setup.md). @@ -101,6 +111,16 @@ Alternatively, use a dedicated domain for GitLab email communications such as See GitLab issue [#30366](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/30366) for a real-world example of this exploit. +CAUTION:**Caution:** +Be sure to use a mail server that has been configured to reduce +spam. +A Postfix mail server that is running on a default configuration, for example, +can result in abuse. All messages received on the configured mailbox will be processed +and messages that are not intended for the GitLab instance will receive a reject notice. +If the sender's address is spoofed, the reject notice will be delivered to the spoofed +`FROM` address, which can cause the mail server's IP or domain to appear on a block +list. + ### Omnibus package installations 1. Find the `incoming_email` section in `/etc/gitlab/gitlab.rb`, enable the feature diff --git a/doc/administration/instance_limits.md b/doc/administration/instance_limits.md index 3d2f7380494..6d2fbd95d5e 100644 --- a/doc/administration/instance_limits.md +++ b/doc/administration/instance_limits.md @@ -8,6 +8,99 @@ GitLab, like most large applications, enforces limits within certain features to minimum quality of performance. Allowing some features to be limitless could affect security, performance, data, or could even exhaust the allocated resources for the application. +## Rate limits + +Rate limits can be used to improve the security and durability of GitLab. + +For example, a simple script can make thousands of web requests per second. Whether malicious, apathetic, or just a bug, your application and infrastructure may not be able to cope with the load. Rate limits can help mitigate these types of attacks. + +Read more about [configuring rate limits](../security/rate_limits.md) in the Security documentation. + +### Issue creation + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/28129) in GitLab 12.10. + +This setting limits the request rate to the issue creation endpoint. + +Read more on [issue creation rate limits](../user/admin_area/settings/rate_limit_on_issues_creation.md). + +- **Default rate limit** - Disabled by default + +### By User or IP + +This setting limits the request rate per user or IP. + +Read more on [User and IP rate limits](../user/admin_area/settings/user_and_ip_rate_limits.md). + +- **Default rate limit** - Disabled by default + +### By raw endpoint + +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/30829) in GitLab 12.2. + +This setting limits the request rate per endpoint. + +Read more on [raw endpoint rate limits](../user/admin_area/settings/rate_limits_on_raw_endpoints.md). + +- **Default rate limit** - 300 requests per project, per commit and per file path + +### By protected path + +This setting limits the request rate on specific paths. + +GitLab rate limits the following paths by default: + +```plaintext +'/users/password', +'/users/sign_in', +'/api/#{API::API.version}/session.json', +'/api/#{API::API.version}/session', +'/users', +'/users/confirmation', +'/unsubscribes/', +'/import/github/personal_access_token', +'/admin/session' +``` + +Read more on [protected path rate limits](../user/admin_area/settings/protected_paths.md). + +- **Default rate limit** - After 10 requests, the client must wait 60 seconds before trying again + +### Import/Export + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/35728) in GitLab 13.2. + +This setting limits the import/export actions for groups and projects. + +| Limit | Default (per minute per user) | +| ----- | ----------------------------- | +| Project Import | 6 | +| Project Export | 6 | +| Project Export Download | 1 | +| Group Import | 6 | +| Group Export | 6 | +| Group Export | Download | 1 | + +Read more on [import/export rate limits](../user/admin_area/settings/import_export_rate_limits.md). + +### Rack attack + +This method of rate limiting is cumbersome, but has some advantages. It allows +throttling of specific paths, and is also integrated into Git and container +registry requests. + +Read more on the [Rack Attack initializer](../security/rack_attack.md) method of setting rate limits. + +- **Default rate limit** - Disabled + +## Gitaly concurrency limit + +Clone traffic can put a large strain on your Gitaly service. To prevent such workloads from overwhelming your Gitaly server, you can set concurrency limits in Gitaly’s configuration file. + +Read more on [Gitaly concurrency limits](gitaly/index.md#limit-rpc-concurrency). + +- **Default rate limit** - Disabled + ## Number of comments per issue, merge request or commit > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/22388) in GitLab 12.4. @@ -77,13 +170,14 @@ To set this limit on a self-managed installation, run the following in the # Plan.default.create_limits! # For project webhooks -Plan.default.limits.update!(project_hooks: 100) +Plan.default.actual_limits.update!(project_hooks: 100) # For group webhooks -Plan.default.limits.update!(group_hooks: 100) +Plan.default.actual_limits.update!(group_hooks: 100) ``` -NOTE: **Note:** Set the limit to `0` to disable it. +NOTE: **Note:** +Set the limit to `0` to disable it. ## Incoming emails from auto-responders @@ -115,12 +209,13 @@ To set this limit on a self-managed installation, run the following in the # If limits don't exist for the default plan, you can create one with: # Plan.default.create_limits! -Plan.default.limits.update!(offset_pagination_limit: 10000) +Plan.default.actual_limits.update!(offset_pagination_limit: 10000) ``` - **Default offset pagination limit:** 50000 -NOTE: **Note:** Set the limit to `0` to disable it. +NOTE: **Note:** +Set the limit to `0` to disable it. ## CI/CD limits @@ -149,10 +244,11 @@ To set this limit on a self-managed installation, run the following in the # If limits don't exist for the default plan, you can create one with: # Plan.default.create_limits! -Plan.default.limits.update!(ci_active_jobs: 500) +Plan.default.actual_limits.update!(ci_active_jobs: 500) ``` -NOTE: **Note:** Set the limit to `0` to disable it. +NOTE: **Note:** +Set the limit to `0` to disable it. ### Number of CI/CD subscriptions to a project @@ -171,10 +267,11 @@ To set this limit on a self-managed installation, run the following in the [GitLab Rails console](troubleshooting/debug.md#starting-a-rails-console-session): ```ruby -Plan.default.limits.update!(ci_project_subscriptions: 500) +Plan.default.actual_limits.update!(ci_project_subscriptions: 500) ``` -NOTE: **Note:** Set the limit to `0` to disable it. +NOTE: **Note:** +Set the limit to `0` to disable it. ### Number of pipeline schedules @@ -196,7 +293,7 @@ To set this limit on a self-managed installation, run the following in the [GitLab Rails console](troubleshooting/debug.md#starting-a-rails-console-session): ```ruby -Plan.default.limits.update!(ci_pipeline_schedules: 100) +Plan.default.actual_limits.update!(ci_pipeline_schedules: 100) ``` ### Number of instance level variables @@ -214,7 +311,7 @@ To update this limit to a new value on a self-managed installation, run the foll [GitLab Rails console](troubleshooting/debug.md#starting-a-rails-console-session): ```ruby -Plan.default.limits.update!(ci_instance_level_variables: 30) +Plan.default.actual_limits.update!(ci_instance_level_variables: 30) ``` ## Instance monitoring and metrics @@ -243,6 +340,38 @@ Prometheus alert payloads sent to the `notify.json` endpoint are limited to 1 MB Alert payloads sent to the `notify.json` endpoint are limited to 1 MB in size. +### Metrics dashboard YAML files + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/34834) in GitLab 13.2. + +The memory occupied by a parsed metrics dashboard YAML file cannot exceed 1 MB. + +The maximum depth of each YAML file is limited to 100. The maximum depth of a YAML +file is the amount of nesting of its most nested key. Each hash and array on the +path of the most nested key counts towards its depth. For example, the depth of the +most nested key in the following YAML is 7: + +```yaml +dashboard: 'Test dashboard' +links: +- title: Link 1 + url: https://gitlab.com +panel_groups: +- group: Group A + priority: 1 + panels: + - title: "Super Chart A1" + type: "area-chart" + y_label: "y_label" + weight: 1 + max_value: 1 + metrics: + - id: metric_a1 + query_range: 'query' + unit: unit + label: Legend Label +``` + ## Environment data on Deploy Boards [Deploy Boards](../user/project/deploy_boards.md) load information from Kubernetes about @@ -274,7 +403,8 @@ characters and the rest will not be indexed and hence will not be searchable. This limit can be configured for self-managed installations when [enabling Elasticsearch](../integration/elasticsearch.md#enabling-elasticsearch). -NOTE: **Note:** Set the limit to `0` to disable it. +NOTE: **Note:** +Set the limit to `0` to disable it. ## Wiki limits @@ -284,6 +414,10 @@ NOTE: **Note:** Set the limit to `0` to disable it. See the [documentation on Snippets settings](snippets/index.md). +## Design Management limits + +See the [Design Management Limitations](../user/project/issues/design_management.md#limitations) section. + ## Push Event Limits ### Webhooks and Project Services diff --git a/doc/administration/integration/plantuml.md b/doc/administration/integration/plantuml.md index cd25fbf351b..2a30eced7c4 100644 --- a/doc/administration/integration/plantuml.md +++ b/doc/administration/integration/plantuml.md @@ -130,7 +130,8 @@ that, login with an Admin account and do following: - Check **Enable PlantUML** checkbox. - Set the PlantUML instance as `https://gitlab.example.com/-/plantuml/`. -NOTE: **Note:** If you are using a PlantUML server running v1.2020.9 and +NOTE: **Note:** +If you are using a PlantUML server running v1.2020.9 and above (for example, [plantuml.com](https://plantuml.com)), set the `PLANTUML_ENCODING` environment variable to enable the `deflate` compression. On Omnibus, this can be done set in `/etc/gitlab.rb`: diff --git a/doc/administration/integration/terminal.md b/doc/administration/integration/terminal.md index bc5816ce120..fd9e09dc17a 100644 --- a/doc/administration/integration/terminal.md +++ b/doc/administration/integration/terminal.md @@ -45,7 +45,8 @@ detail below. ## Enabling and disabling terminal support -NOTE: **Note:** AWS Elastic Load Balancers (ELBs) do not support web sockets. +NOTE: **Note:** +AWS Elastic Load Balancers (ELBs) do not support web sockets. AWS Application Load Balancers (ALBs) must be used if you want web terminals to work. See [AWS Elastic Load Balancing Product Comparison](https://aws.amazon.com/elasticloadbalancing/features/#compare) for more information. diff --git a/doc/administration/job_artifacts.md b/doc/administration/job_artifacts.md index 0dbda67d39b..cb31a8934b1 100644 --- a/doc/administration/job_artifacts.md +++ b/doc/administration/job_artifacts.md @@ -106,6 +106,11 @@ If you configure GitLab to store CI logs and artifacts on object storage, you mu #### Object Storage Settings +NOTE: **Note:** +In GitLab 13.2 and later, we recommend using the +[consolidated object storage settings](object_storage.md#consolidated-object-storage-configuration). +This section describes the earlier configuration format. + For source installations the following settings are nested under `artifacts:` and then `object_store:`. On Omnibus GitLab installs they are prefixed by `artifacts_object_store_`. | Setting | Description | Default | @@ -117,22 +122,9 @@ For source installations the following settings are nested under `artifacts:` an | `proxy_download` | Set to true to enable proxying all files served. Option allows to reduce egress traffic as this allows clients to download directly from remote storage instead of proxying all data | `false` | | `connection` | Various connection options described below | | -##### S3 compatible connection settings - -The connection settings match those provided by [Fog](https://github.com/fog), and are as follows: +#### Connection settings -| Setting | Description | Default | -|---------|-------------|---------| -| `provider` | Always `AWS` for compatible hosts | AWS | -| `aws_access_key_id` | AWS credentials, or compatible | | -| `aws_secret_access_key` | AWS credentials, or compatible | | -| `aws_signature_version` | AWS signature version to use. 2 or 4 are valid options. Digital Ocean Spaces and other providers may need 2. | 4 | -| `enable_signature_v4_streaming` | Set to true to enable HTTP chunked transfers with [AWS v4 signatures](https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html). Oracle Cloud S3 needs this to be false | true | -| `region` | AWS region | us-east-1 | -| `host` | S3 compatible host for when not using AWS, for example `localhost` or `storage.example.com` | s3.amazonaws.com | -| `endpoint` | Can be used when configuring an S3 compatible service such as [MinIO](https://min.io), by entering a URL such as `http://127.0.0.1:9000` | (optional) | -| `path_style` | Set to true to use `host/bucket_name/object` style paths instead of `bucket_name.host/object`. Leave as false for AWS S3 | false | -| `use_iam_profile` | Set to true to use IAM profile instead of access keys | false +See [the available connection settings for different providers](object_storage.md#connection-settings). **In Omnibus installations:** @@ -172,7 +164,7 @@ _The artifacts are stored by default in gitlab-rake gitlab:artifacts:migrate ``` -CAUTION: **CAUTION:** +CAUTION: **Caution:** JUnit test report artifact (`junit.xml.gz`) migration [is not supported](https://gitlab.com/gitlab-org/gitlab/-/issues/27698) by the `gitlab:artifacts:migrate` script. @@ -205,24 +197,14 @@ _The artifacts are stored by default in sudo -u git -H bundle exec rake gitlab:artifacts:migrate RAILS_ENV=production ``` -CAUTION: **CAUTION:** +CAUTION: **Caution:** JUnit test report artifact (`junit.xml.gz`) migration [is not supported](https://gitlab.com/gitlab-org/gitlab/-/issues/27698) by the `gitlab:artifacts:migrate` script. -### OpenStack compatible connection settings +### OpenStack example -The connection settings match those provided by [Fog](https://github.com/fog), and are as follows: - -| Setting | Description | Default | -|---------|-------------|---------| -| `provider` | Always `OpenStack` for compatible hosts | OpenStack | -| `openstack_username` | OpenStack username | | -| `openstack_api_key` | OpenStack API key | | -| `openstack_temp_url_key` | OpenStack key for generating temporary URLs | | -| `openstack_auth_url` | OpenStack authentication endpoint | | -| `openstack_region` | OpenStack region | | -| `openstack_tenant_id` | OpenStack tenant ID | +See [the available connection settings for OpenStack](object_storage.md#openstack-compatible-connection-settings). **In Omnibus installations:** @@ -453,7 +435,7 @@ the number you want. #### Delete job artifacts from jobs completed before a specific date -CAUTION: **CAUTION:** +CAUTION: **Caution:** These commands remove data permanently from the database and from disk. We highly recommend running them only under the guidance of a Support Engineer, or running them in a test environment with a backup of the instance ready to be @@ -479,7 +461,7 @@ If you need to manually remove job artifacts associated with multiple jobs while 1. Delete job artifacts older than a specific date: - NOTE: **NOTE:** + NOTE: **Note:** This step will also erase artifacts that users have chosen to ["keep"](../ci/pipelines/job_artifacts.md#browsing-artifacts). @@ -500,7 +482,7 @@ If you need to manually remove job artifacts associated with multiple jobs while #### Delete job artifacts and logs from jobs completed before a specific date -CAUTION: **CAUTION:** +CAUTION: **Caution:** These commands remove data permanently from the database and from disk. We highly recommend running them only under the guidance of a Support Engineer, or running them in a test environment with a backup of the instance ready to be diff --git a/doc/administration/job_logs.md b/doc/administration/job_logs.md index d3484536a76..4dba33b796a 100644 --- a/doc/administration/job_logs.md +++ b/doc/administration/job_logs.md @@ -73,7 +73,7 @@ job output in the UI will be empty. For example, to delete all job logs older than 60 days, run the following from a shell in your GitLab instance: -DANGER: **Warning:** +DANGER: **Danger:** This command will permanently delete the log files and is irreversible. ```shell diff --git a/doc/administration/lfs/index.md b/doc/administration/lfs/index.md index dd0e25b05f1..4a8151bd091 100644 --- a/doc/administration/lfs/index.md +++ b/doc/administration/lfs/index.md @@ -63,6 +63,11 @@ GitLab provides two different options for the uploading mechanism: "Direct uploa [Read more about using object storage with GitLab](../object_storage.md). +NOTE: **Note:** +In GitLab 13.2 and later, we recommend using the +[consolidated object storage settings](../object_storage.md#consolidated-object-storage-configuration). +This section describes the earlier configuration format. + **Option 1. Direct upload** 1. User pushes an `lfs` file to the GitLab instance @@ -86,54 +91,10 @@ The following general settings are supported. | `proxy_download` | Set to true to enable proxying all files served. Option allows to reduce egress traffic as this allows clients to download directly from remote storage instead of proxying all data | `false` | | `connection` | Various connection options described below | | -The `connection` settings match those provided by [Fog](https://github.com/fog). +See [the available connection settings for different providers](../object_storage.md#connection-settings). Here is a configuration example with S3. -| Setting | Description | example | -|---------|-------------|---------| -| `provider` | The provider name | AWS | -| `aws_access_key_id` | AWS credentials, or compatible | `ABC123DEF456` | -| `aws_secret_access_key` | AWS credentials, or compatible | `ABC123DEF456ABC123DEF456ABC123DEF456` | -| `aws_signature_version` | AWS signature version to use. 2 or 4 are valid options. Digital Ocean Spaces and other providers may need 2. | 4 | -| `enable_signature_v4_streaming` | Set to true to enable HTTP chunked transfers with [AWS v4 signatures](https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html). Oracle Cloud S3 needs this to be false | true | -| `region` | AWS region | us-east-1 | -| `host` | S3 compatible host for when not using AWS, e.g. `localhost` or `storage.example.com` | s3.amazonaws.com | -| `endpoint` | Can be used when configuring an S3 compatible service such as [MinIO](https://min.io), by entering a URL such as `http://127.0.0.1:9000` | (optional) | -| `path_style` | Set to true to use `host/bucket_name/object` style paths instead of `bucket_name.host/object`. Leave as false for AWS S3 | false | -| `use_iam_profile` | Set to true to use IAM profile instead of access keys | false - -Here is a configuration example with GCS. - -| Setting | Description | example | -|---------|-------------|---------| -| `provider` | The provider name | `Google` | -| `google_project` | GCP project name | `gcp-project-12345` | -| `google_client_email` | The email address of the service account | `foo@gcp-project-12345.iam.gserviceaccount.com` | -| `google_json_key_location` | The JSON key path | `/path/to/gcp-project-12345-abcde.json` | - -NOTE: **Note:** -The service account must have permission to access the bucket. -[See more](https://cloud.google.com/storage/docs/authentication) - -Here is a configuration example with Rackspace Cloud Files. - -| Setting | Description | example | -|---------|-------------|---------| -| `provider` | The provider name | `Rackspace` | -| `rackspace_username` | The username of the Rackspace account with access to the container | `joe.smith` | -| `rackspace_api_key` | The API key of the Rackspace account with access to the container | `ABC123DEF456ABC123DEF456ABC123DE` | -| `rackspace_region` | The Rackspace storage region to use, a three letter code from the [list of service access endpoints](https://developer.rackspace.com/docs/cloud-files/v1/general-api-info/service-access/) | `iad` | -| `rackspace_temp_url_key` | The private key you have set in the Rackspace API for temporary URLs. Read more [here](https://developer.rackspace.com/docs/cloud-files/v1/use-cases/public-access-to-your-cloud-files-account/#tempurl) | `ABC123DEF456ABC123DEF456ABC123DE` | - -NOTE: **Note:** -Regardless of whether the container has public access enabled or disabled, Fog will -use the TempURL method to grant access to LFS objects. If you see errors in logs referencing -instantiating storage with a `temp-url-key`, ensure that you have set the key properly -on the Rackspace API and in `gitlab.rb`. You can verify the value of the key Rackspace -has set by sending a GET request with token header to the service access endpoint URL -and comparing the output of the returned headers. - ### Manual uploading to an object storage There are two ways to manually do the same thing as automatic uploading (described above). diff --git a/doc/administration/logs.md b/doc/administration/logs.md index 7d7053a26db..3db9d32563e 100644 --- a/doc/administration/logs.md +++ b/doc/administration/logs.md @@ -26,7 +26,7 @@ GitLab, thanks to [Lograge](https://github.com/roidrage/lograge/). Note that requests from the API are logged to a separate file in `api_json.log`. Each line contains a JSON line that can be ingested by services like Elasticsearch and Splunk. -Line breaks have been added to this example for legibility: +Line breaks were added to examples for legibility: ```json { @@ -79,7 +79,7 @@ seconds: User clone and fetch activity using HTTP transport appears in this log as `action: git_upload_pack`. In addition, the log contains the originating IP address, -(`remote_ip`),the user's ID (`user_id`), and username (`username`). +(`remote_ip`), the user's ID (`user_id`), and username (`username`). Some endpoints such as `/search` may make requests to Elasticsearch if using [Advanced Global Search](../user/search/advanced_global_search.md). These will @@ -112,7 +112,8 @@ The ActionCable connection or channel class is used as the `controller`. } ``` -NOTE: **Note:** Starting with GitLab 12.5, if an error occurs, an +NOTE: **Note:** +Starting with GitLab 12.5, if an error occurs, an `exception` field is included with `class`, `message`, and `backtrace`. Previous versions included an `error` field instead of `exception.class` and `exception.message`. For example: @@ -227,7 +228,7 @@ It helps you see requests made directly to the API. For example: } ``` -This entry shows an access to an internal endpoint to check whether an +This entry shows an internal endpoint accessed to check whether an associated SSH key can download the project in question via a `git fetch` or `git clone`. In this example, we see: @@ -320,7 +321,7 @@ packages or in `/home/git/gitlab/log/kubernetes.log` for installations from source. It logs information related to the Kubernetes Integration including errors -during installing cluster applications on your GitLab managed Kubernetes +during installing cluster applications on your managed Kubernetes clusters. Each line contains a JSON line that can be ingested by services like Elasticsearch and Splunk. @@ -362,7 +363,7 @@ After 12.2, this file was renamed from `githost.log` to `git_json.log` and stored in JSON format. GitLab has to interact with Git repositories, but in some rare cases -something can go wrong, and in this case you will know what exactly +something can go wrong, and in this case you may need know what exactly happened. This log file contains all failed requests from GitLab to Git repositories. In the majority of cases this file will be useful for developers only. For example: @@ -473,8 +474,8 @@ This file lives in `/var/log/gitlab/gitlab-rails/sidekiq_client.log` for Omnibus GitLab packages or in `/home/git/gitlab/log/sidekiq_client.log` for installations from source. -This file contains logging information about jobs before they are start -being processed by Sidekiq, for example before being enqueued. +This file contains logging information about jobs before Sidekiq starts +processing them, such as before being enqueued. This log file follows the same structure as [`sidekiq.log`](#sidekiqlog), so it will be structured as JSON if @@ -571,32 +572,45 @@ User clone/fetch activity using SSH transport appears in this log as `executing ## Gitaly Logs -This file lives in `/var/log/gitlab/gitaly/current` and is produced by [runit](http://smarden.org/runit/). `runit` is packaged with Omnibus and a brief explanation of its purpose is available [in the omnibus documentation](https://docs.gitlab.com/omnibus/architecture/#runit). [Log files are rotated](http://smarden.org/runit/svlogd.8.html), renamed in Unix timestamp format and `gzip`-compressed (e.g. `@1584057562.s`). +This file lives in `/var/log/gitlab/gitaly/current` and is produced by [runit](http://smarden.org/runit/). `runit` is packaged with Omnibus GitLab and a brief explanation of its purpose is available [in the Omnibus GitLab documentation](https://docs.gitlab.com/omnibus/architecture/#runit). [Log files are rotated](http://smarden.org/runit/svlogd.8.html), renamed in Unix timestamp format, and `gzip`-compressed (like `@1584057562.s`). ### `grpc.log` This file lives in `/var/log/gitlab/gitlab-rails/grpc.log` for Omnibus GitLab packages. Native [gRPC](https://grpc.io/) logging used by Gitaly. -## `puma_stderr.log` & `puma_stdout.log` +## Puma Logs -This file lives in `/var/log/gitlab/puma/puma_stderr.log` and `/var/log/gitlab/puma/puma_stdout.log` for -Omnibus GitLab packages or in `/home/git/gitlab/log/puma_stderr.log` and `/home/git/gitlab/log/puma_stdout.log` -for installations from source. +### `puma_stdout.log` + +This file lives in `/var/log/gitlab/puma/puma_stdout.log` for +Omnibus GitLab packages, and `/home/git/gitlab/log/puma_stdout.log` for +installations from source. + +### `puma_stderr.log` + +This file lives in `/var/log/gitlab/puma/puma_stderr.log` for +Omnibus GitLab packages, or in `/home/git/gitlab/log/puma_stderr.log` for +installations from source. -## `unicorn_stderr.log` & `unicorn_stdout.log` +## Unicorn Logs NOTE: **Note:** Starting with GitLab 13.0, Puma is the default web server used in GitLab all-in-one package based installations as well as GitLab Helm chart deployments. -This file lives in `/var/log/gitlab/unicorn/unicorn_stderr.log` and `/var/log/gitlab/unicorn/unicorn_stdout.log` for -Omnibus GitLab packages or in `/home/git/gitlab/log/unicorn_stderr.log` and `/home/git/gitlab/log/unicorn_stdout.log` +### `unicorn_stdout.log` + +This file lives in `/var/log/gitlab/unicorn/unicorn_stdout.log` for +Omnibus GitLab packages or in `/home/git/gitlab/log/unicorn_stdout.log` for +for installations from source. + +### `unicorn_stderr.log` + +This file lives in `/var/log/gitlab/unicorn/unicorn_stderr.log` for +Omnibus GitLab packages or in `/home/git/gitlab/log/unicorn_stderr.log` for for installations from source. -Unicorn is a high-performance forking Web server which is used for -serving the GitLab application. You can look at this log if, for -example, your application does not respond. This log contains all -information about the state of Unicorn processes at any given time. +These logs contain all information about the state of Unicorn processes at any given time. ```plaintext I, [2015-02-13T06:14:46.680381 #9047] INFO -- : Refreshing Gem list @@ -657,7 +671,7 @@ This log records: - [Protected paths](../user/admin_area/settings/protected_paths.md) abusive requests. NOTE: **Note:** -From [%12.3](https://gitlab.com/gitlab-org/gitlab/-/issues/29239), user ID and username are also +In GitLab versions [12.3](https://gitlab.com/gitlab-org/gitlab/-/issues/29239) and greater, user ID and username are also recorded on this log, if available. ## `graphql_json.log` @@ -686,7 +700,7 @@ installations from source. > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/19186) in GitLab 12.6. -This file lives in `/var/log/gitlab/mailroom/mail_room_json.log` for +This file lives in `/var/log/gitlab/mailroom/current` for Omnibus GitLab packages or in `/home/git/gitlab/log/mail_room_json.log` for installations from source. @@ -793,8 +807,8 @@ This file lives in `/var/log/gitlab/gitlab-rails/service_measurement.log` for Omnibus GitLab packages or in `/home/git/gitlab/log/service_measurement.log` for installations from source. -It contain only a single structured log with measurements for each service execution. -It will contain measurement such as: number of SQL calls, `execution_time`, `gc_stats`, memory usage, etc... +It contains only a single structured log with measurements for each service execution. +It will contain measurements such as the number of SQL calls, `execution_time`, `gc_stats`, and `memory usage`. For example: @@ -870,22 +884,46 @@ For example: } ``` +## Mattermost Logs + +For Omnibus GitLab installations, Mattermost logs reside in `/var/log/gitlab/mattermost/mattermost.log`. + ## Workhorse Logs -For Omnibus installations, Workhorse logs reside in `/var/log/gitlab/gitlab-workhorse/current`. +For Omnibus GitLab installations, Workhorse logs reside in `/var/log/gitlab/gitlab-workhorse/`. ## PostgreSQL Logs -For Omnibus installations, PostgreSQL logs reside in `/var/log/gitlab/postgresql/current`. +For Omnibus GitLab installations, PostgreSQL logs reside in `/var/log/gitlab/postgresql/`. ## Prometheus Logs -For Omnibus installations, Prometheus logs reside in `/var/log/gitlab/prometheus/current`. +For Omnibus GitLab installations, Prometheus logs reside in `/var/log/gitlab/prometheus/`. ## Redis Logs -For Omnibus installations, Redis logs reside in `/var/log/gitlab/redis/current`. +For Omnibus GitLab installations, Redis logs reside in `/var/log/gitlab/redis/`. -## Mattermost Logs +## Alertmanager Logs + +For Omnibus GitLab installations, Alertmanager logs reside in `/var/log/gitlab/alertmanager/`. + +## Crond Logs + +For Omnibus GitLab installations, crond logs reside in `/var/log/gitlab/crond/`. + +## Grafana Logs + +For Omnibus GitLab installations, Grafana logs reside in `/var/log/gitlab/grafana/`. + +## LogRotate Logs + +For Omnibus GitLab installations, logrotate logs reside in `/var/log/gitlab/logrotate/`. + +## GitLab Monitor Logs + +For Omnibus GitLab installations, GitLab Monitor logs reside in `/var/log/gitlab/gitlab-monitor/`. + +## GitLab Exporter -For Omnibus installations, Mattermost logs reside in `/var/log/gitlab/mattermost/mattermost.log`. +For Omnibus GitLab installations, GitLab Exporter logs reside in `/var/log/gitlab/gitlab-exporter/`. diff --git a/doc/administration/merge_request_diffs.md b/doc/administration/merge_request_diffs.md index da9ca42ed9e..93cdbff6621 100644 --- a/doc/administration/merge_request_diffs.md +++ b/doc/administration/merge_request_diffs.md @@ -72,6 +72,11 @@ be configured already. ## Object Storage Settings +NOTE: **Note:** +In GitLab 13.2 and later, we recommend using the +[consolidated object storage settings](object_storage.md#consolidated-object-storage-configuration). +This section describes the earlier configuration format. + For source installations, these settings are nested under `external_diffs:` and then `object_store:`. On Omnibus installations, they are prefixed by `external_diffs_object_store_`. @@ -87,20 +92,7 @@ then `object_store:`. On Omnibus installations, they are prefixed by ### S3 compatible connection settings -The connection settings match those provided by [Fog](https://github.com/fog), and are as follows: - -| Setting | Description | Default | -|---------|-------------|---------| -| `provider` | Always `AWS` for compatible hosts | AWS | -| `aws_access_key_id` | AWS credentials, or compatible | | -| `aws_secret_access_key` | AWS credentials, or compatible | | -| `aws_signature_version` | AWS signature version to use. 2 or 4 are valid options. Digital Ocean Spaces and other providers may need 2. | 4 | -| `enable_signature_v4_streaming` | Set to true to enable HTTP chunked transfers with [AWS v4 signatures](https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html). Oracle Cloud S3 needs this to be false | true | -| `region` | AWS region | us-east-1 | -| `host` | S3 compatible host for when not using AWS, e.g. `localhost` or `storage.example.com` | s3.amazonaws.com | -| `endpoint` | Can be used when configuring an S3 compatible service such as [MinIO](https://min.io), by entering a URL such as `http://127.0.0.1:9000` | (optional) | -| `path_style` | Set to true to use `host/bucket_name/object` style paths instead of `bucket_name.host/object`. Leave as false for AWS S3 | false | -| `use_iam_profile` | Set to true to use IAM profile instead of access keys | false +See [the available connection settings for different providers](object_storage.md#connection-settings). **In Omnibus installations:** @@ -195,3 +187,51 @@ conditions become true: These rules strike a balance between space and performance by only storing frequently-accessed diffs in the database. Diffs that are less likely to be accessed are moved to external storage instead. + +## Correcting incorrectly-migrated diffs + +Versions of GitLab earlier than `v13.0.0` would incorrectly record the location +of some merge request diffs when [external diffs in object storage](#object-storage-settings) +were enabled. This mainly affected imported merge requests, and was resolved +with [this merge request](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/31005). + +If you are using object storage, have never used on-disk storage for external +diffs, the "changes" tab for some merge requests fails to load with a 500 error, +and the exception for that error is of this form: + +```plain +Errno::ENOENT (No such file or directory @ rb_sysopen - /var/opt/gitlab/gitlab-rails/shared/external-diffs/merge_request_diffs/mr-6167082/diff-8199789) +``` + +Then you are affected by this issue. Since it's not possible to safely determine +all these conditions automatically, we've provided a Rake task in GitLab v13.2.0 +that you can run manually to correct the data: + +**In Omnibus installations:** + +```shell +sudo gitlab-rake gitlab:external_diffs:force_object_storage +``` + +**In installations from source:** + +```shell +sudo -u git -H bundle exec rake gitlab:external_diffs:force_object_storage RAILS_ENV=production +``` + +Environment variables can be provided to modify the behavior of the task. The +available variables are: + +| Name | Default value | Purpose | +| ---- | ------------- | ------- | +| `ANSI` | `true` | Use ANSI escape codes to make output more understandable | +| `BATCH_SIZE` | `1000` | Iterate through the table in batches of this size | +| `START_ID` | `nil` | If set, begin scanning at this ID | +| `END_ID` | `nil` | If set, stop scanning at this ID | +| `UPDATE_DELAY` | `1` | Number of seconds to sleep between updates | + +The `START_ID` and `END_ID` variables may be used to run the update in parallel, +by assigning different processes to different parts of the table. The `BATCH` +and `UPDATE_DELAY` parameters allow the speed of the migration to be traded off +against concurrent access to the table. The `ANSI` parameter should be set to +false if your terminal does not support ANSI escape codes. diff --git a/doc/administration/monitoring/gitlab_self_monitoring_project/img/self_monitoring_default_dashboard.png b/doc/administration/monitoring/gitlab_self_monitoring_project/img/self_monitoring_default_dashboard.png Binary files differnew file mode 100644 index 00000000000..1d61823ce41 --- /dev/null +++ b/doc/administration/monitoring/gitlab_self_monitoring_project/img/self_monitoring_default_dashboard.png diff --git a/doc/administration/monitoring/gitlab_self_monitoring_project/index.md b/doc/administration/monitoring/gitlab_self_monitoring_project/index.md index d05fb803c5c..44ba26296b9 100644 --- a/doc/administration/monitoring/gitlab_self_monitoring_project/index.md +++ b/doc/administration/monitoring/gitlab_self_monitoring_project/index.md @@ -26,7 +26,7 @@ of the project shows some basic resource usage charts, such as CPU and memory us of each server in [Omnibus GitLab](https://docs.gitlab.com/omnibus/) installations. You can also use the project to configure your own -[custom metrics](../../../user/project/integrations/prometheus.md#adding-custom-metrics) using +[custom metrics](../../../operations/metrics/index.md#adding-custom-metrics) using metrics exposed by the [GitLab exporter](../prometheus/gitlab_metrics.md#metrics-available). ## Creating the self monitoring project @@ -47,6 +47,19 @@ project. If you create the project again, it will be created in its default stat It can take a few seconds for it to be deleted. 1. After the project is deleted, GitLab displays a message confirming your action. +## Dashboards available in Omnibus GitLab + +Omnibus GitLab provides a dashboard that displays CPU and memory usage +of each GitLab server. To select the servers to be displayed in the +panels, provide a regular expression in the **Instance label regex** field. +The dashboard uses metrics available in +[Omnibus GitLab](https://docs.gitlab.com/omnibus/) installations. + + + +You can also +[create your own dashboards](../../../operations/metrics/dashboards/index.md#defining-custom-dashboards-per-project). + ## Connection to Prometheus The project will be automatically configured to connect to the @@ -60,18 +73,18 @@ you should ## Taking action on Prometheus alerts **(ULTIMATE)** -You can [add a webhook](../../../user/project/integrations/prometheus.md#external-prometheus-instances) +You can [add a webhook](../../../operations/metrics/alerts.md#external-prometheus-instances) to the Prometheus configuration in order for GitLab to receive notifications of any alerts. Once the webhook is setup, you can -[take action on incoming alerts](../../../user/project/integrations/prometheus.md#taking-action-on-incidents-ultimate). +[take action on incoming alerts](../../../operations/metrics/alerts.md#trigger-actions-from-alerts-ultimate). ## Adding custom metrics to the self monitoring project You can add custom metrics in the self monitoring project by: -1. [Duplicating](../../../user/project/integrations/prometheus.md#duplicating-a-gitlab-defined-dashboard) the default dashboard. -1. [Editing](../../../user/project/integrations/prometheus.md#view-and-edit-the-source-file-of-a-custom-dashboard) the newly created dashboard file and configuring it with [dashboard YAML properties](../../../user/project/integrations/prometheus.md#dashboard-yaml-properties). +1. [Duplicating](../../../operations/metrics/dashboards/index.md#duplicating-a-gitlab-defined-dashboard) the default dashboard. +1. [Editing](../../../operations/metrics/dashboards/index.md#view-and-edit-the-source-file-of-a-custom-dashboard) the newly created dashboard file and configuring it with [dashboard YAML properties](../../../operations/metrics/dashboards/yaml.md). ## Troubleshooting diff --git a/doc/administration/monitoring/performance/grafana_configuration.md b/doc/administration/monitoring/performance/grafana_configuration.md index 4dbe3aed84e..96f1377fb73 100644 --- a/doc/administration/monitoring/performance/grafana_configuration.md +++ b/doc/administration/monitoring/performance/grafana_configuration.md @@ -6,85 +6,90 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Grafana Configuration -[Grafana](https://grafana.com/) is a tool that allows you to visualize time -series metrics through graphs and dashboards. GitLab writes performance data to Prometheus -and Grafana will allow you to query to display useful graphs. +[Grafana](https://grafana.com/) is a tool that enables you to visualize time +series metrics through graphs and dashboards. GitLab writes performance data to Prometheus, +and Grafana allows you to query the data to display useful graphs. ## Installation -[Omnibus GitLab can help you install Grafana (recommended)](https://docs.gitlab.com/omnibus/settings/grafana.html) +Omnibus GitLab can [help you install Grafana (recommended)](https://docs.gitlab.com/omnibus/settings/grafana.html) or Grafana supplies package repositories (Yum/Apt) for easy installation. See [Grafana installation documentation](https://grafana.com/docs/grafana/latest/installation/) for detailed steps. NOTE: **Note:** Before starting Grafana for the first time, set the admin user -and password in `/etc/grafana/grafana.ini`. Otherwise, the default password -will be `admin`. +and password in `/etc/grafana/grafana.ini`. If you don't, the default password +is `admin`. ## Configuration -Login as the admin user. Expand the menu by clicking the Grafana logo in the -top left corner. Choose 'Data Sources' from the menu. Then, click 'Add new' -in the top bar. - - - - +1. Log in to Grafana as the admin user. +1. Expand the menu by clicking the Grafana logo in the top left corner. +1. Choose **Data Sources** from the menu. +1. Click **Add new** in the top bar: +  +1. Edit the data source to fit your needs: +  +1. Click **Save**. ## Import Dashboards -You can now import a set of default dashboards that will give you a good -start on displaying useful information. GitLab has published a set of default -[Grafana dashboards](https://gitlab.com/gitlab-org/grafana-dashboards) to get you started. Clone the -repository or download a zip/tarball, then follow these steps to import each -JSON file. - -Open the dashboard dropdown menu and click 'Import' - - - -Click 'Choose file' and browse to the location where you downloaded or cloned -the dashboard repository. Pick one of the JSON files to import. - - - -Once the dashboard is imported, be sure to click save icon in the top bar. If -you do not save the dashboard after importing it will be removed when you -navigate away. - - +You can now import a set of default dashboards to start displaying useful information. +GitLab has published a set of default +[Grafana dashboards](https://gitlab.com/gitlab-org/grafana-dashboards) to get you started. +Clone the repository, or download a ZIP file or tarball, then follow these steps to import each +JSON file individually: + +1. Log in to Grafana as the admin user. +1. Open the dashboard dropdown menu and click **Import**: +  +1. Click **Choose file**, and browse to the location where you downloaded or + cloned the dashboard repository. Select a JSON file to import: +  +1. After the dashboard is imported, click the **Save dashboard** icon in the top bar: +  + + NOTE: **Note:** + If you don't save the dashboard after importing it, the dashboard is removed + when you navigate away from the page. Repeat this process for each dashboard you wish to import. -Alternatively you can automatically import all the dashboards into your Grafana -instance. See the README of the [Grafana dashboards](https://gitlab.com/gitlab-org/grafana-dashboards) -repository for more information on this process. +Alternatively, you can import all the dashboards into your Grafana +instance. For more information about this process, see the +[README of the Grafana dashboards](https://gitlab.com/gitlab-org/grafana-dashboards) +repository. ## Integration with GitLab UI > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/61005) in GitLab 12.1. -If you have set up Grafana, you can enable a link to access it easily from the sidebar: +After setting up Grafana, you can enable a link to access it easily from the +GitLab sidebar: -1. Go to the **Admin Area > Settings > Metrics and profiling**. +1. Navigate to the **{admin}** **Admin Area > Settings > Metrics and profiling**. 1. Expand **Metrics - Grafana**. -1. Check the "Enable access to Grafana" checkbox. -1. If Grafana is enabled through Omnibus GitLab and on the same server, - leave **Grafana URL** unchanged. It should be `/-/grafana`. - - In any other case, enter the full URL of the Grafana instance. +1. Check the **Enable access to Grafana** checkbox. +1. Configure the **Grafana URL**: + - *If Grafana is enabled through Omnibus GitLab and on the same server,* + leave **Grafana URL** unchanged. It should be `/-/grafana`. + - *Otherwise,* enter the full URL of the Grafana instance. 1. Click **Save changes**. -1. The new link will be available in the **Admin Area > Monitoring > Metrics Dashboard**. + +GitLab displays your link in the **{admin}** **Admin Area > Monitoring > Metrics Dashboard**. ## Security Update -Users running GitLab version 12.0 or later should immediately upgrade to one of the following security releases due to a known vulnerability with the embedded Grafana dashboard: +Users running GitLab version 12.0 or later should immediately upgrade to one of the +following security releases due to a known vulnerability with the embedded Grafana dashboard: - 12.0.6 - 12.1.6 -After upgrading, the Grafana dashboard will be disabled and the location of your existing Grafana data will be changed from `/var/opt/gitlab/grafana/data/` to `/var/opt/gitlab/grafana/data.bak.#{Date.today}/`. +After upgrading, the Grafana dashboard is disabled, and the location of your +existing Grafana data is changed from `/var/opt/gitlab/grafana/data/` to +`/var/opt/gitlab/grafana/data.bak.#{Date.today}/`. To prevent the data from being relocated, you can run the following command prior to upgrading: @@ -100,19 +105,23 @@ sudo mv /var/opt/gitlab/grafana/data.bak.xxxx/ /var/opt/gitlab/grafana/data/ However, you should **not** reinstate your old data _except_ under one of the following conditions: -1. If you are certain that you changed your default admin password when you enabled Grafana -1. If you run GitLab in a private network, accessed only by trusted users, and your Grafana login page has not been exposed to the internet +1. If you're certain that you changed your default admin password when you enabled Grafana. +1. If you run GitLab in a private network, accessed only by trusted users, and your + Grafana login page has not been exposed to the internet. -If you require access to your old Grafana data but do not meet one of these criteria, you may consider: +If you require access to your old Grafana data but don't meet one of these criteria, you may consider: 1. Reinstating it temporarily. 1. [Exporting the dashboards](https://grafana.com/docs/grafana/latest/reference/export_import/#exporting-a-dashboard) you need. 1. Refreshing the data and [re-importing your dashboards](https://grafana.com/docs/grafana/latest/reference/export_import/#importing-a-dashboard). DANGER: **Danger:** -This poses a temporary vulnerability while your old Grafana data is in use and the decision to do so should be weighed carefully with your need to access existing data and dashboards. +These actions pose a temporary vulnerability while your old Grafana data is in use. +Deciding to take any of these actions should be weighed carefully with your need to access +existing data and dashboards. -For more information and further mitigation details, please refer to our [blog post on the security release](https://about.gitlab.com/releases/2019/08/12/critical-security-release-gitlab-12-dot-1-dot-6-released/). +For more information and further mitigation details, please refer to our +[blog post on the security release](https://about.gitlab.com/releases/2019/08/12/critical-security-release-gitlab-12-dot-1-dot-6-released/). --- diff --git a/doc/administration/monitoring/performance/img/performance_bar_configuration_settings.png b/doc/administration/monitoring/performance/img/performance_bar_configuration_settings.png Binary files differdeleted file mode 100644 index fafc50cd000..00000000000 --- a/doc/administration/monitoring/performance/img/performance_bar_configuration_settings.png +++ /dev/null diff --git a/doc/administration/monitoring/performance/img/performance_bar_frontend.png b/doc/administration/monitoring/performance/img/performance_bar_frontend.png Binary files differindex 32a241e032b..9cc874c883a 100644 --- a/doc/administration/monitoring/performance/img/performance_bar_frontend.png +++ b/doc/administration/monitoring/performance/img/performance_bar_frontend.png diff --git a/doc/administration/monitoring/performance/img/performance_bar_gitaly_calls.png b/doc/administration/monitoring/performance/img/performance_bar_gitaly_calls.png Binary files differindex 2c43201cbd0..6caa2869d9b 100644 --- a/doc/administration/monitoring/performance/img/performance_bar_gitaly_calls.png +++ b/doc/administration/monitoring/performance/img/performance_bar_gitaly_calls.png diff --git a/doc/administration/monitoring/performance/img/performance_bar_gitaly_threshold.png b/doc/administration/monitoring/performance/img/performance_bar_gitaly_threshold.png Binary files differindex 4e42d904cdf..386558da813 100644 --- a/doc/administration/monitoring/performance/img/performance_bar_gitaly_threshold.png +++ b/doc/administration/monitoring/performance/img/performance_bar_gitaly_threshold.png diff --git a/doc/administration/monitoring/performance/img/performance_bar_redis_calls.png b/doc/administration/monitoring/performance/img/performance_bar_redis_calls.png Binary files differindex ecb2dffbf8d..f2df8c794db 100644 --- a/doc/administration/monitoring/performance/img/performance_bar_redis_calls.png +++ b/doc/administration/monitoring/performance/img/performance_bar_redis_calls.png diff --git a/doc/administration/monitoring/performance/img/performance_bar_request_selector_warning.png b/doc/administration/monitoring/performance/img/performance_bar_request_selector_warning.png Binary files differindex 74711387ffe..3872c8b41b2 100644 --- a/doc/administration/monitoring/performance/img/performance_bar_request_selector_warning.png +++ b/doc/administration/monitoring/performance/img/performance_bar_request_selector_warning.png diff --git a/doc/administration/monitoring/performance/img/performance_bar_rugged_calls.png b/doc/administration/monitoring/performance/img/performance_bar_rugged_calls.png Binary files differindex 210f80a713d..0340ca1b2f7 100644 --- a/doc/administration/monitoring/performance/img/performance_bar_rugged_calls.png +++ b/doc/administration/monitoring/performance/img/performance_bar_rugged_calls.png diff --git a/doc/administration/monitoring/performance/performance_bar.md b/doc/administration/monitoring/performance/performance_bar.md index c681dedbca8..8002a9ab296 100644 --- a/doc/administration/monitoring/performance/performance_bar.md +++ b/doc/administration/monitoring/performance/performance_bar.md @@ -6,71 +6,83 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Performance Bar -A Performance Bar can be displayed, to dig into the performance of a page. When -activated, it looks as follows: +You can display the GitLab Performance Bar to see statistics for the performance +of a page. When activated, it looks as follows:  -It allows you to see (from left to right): +From left to right, it displays: -- the current host serving the page -- time taken and number of DB queries; click through for details of these queries +- **Current Host**: the current host serving the page. +- **Database queries**: the time taken (in milliseconds) and the total number + of database queries, displayed in the format `00ms / 00pg`. Click to display + a modal window with more details:  -- time taken and number of [Gitaly](../../gitaly/index.md) calls; click through for details of these calls +- **Gitaly calls**: the time taken (in milliseconds) and the total number of + [Gitaly](../../gitaly/index.md) calls. Click to display a modal window with more + details:  -- time taken and number of [Rugged](../../high_availability/nfs.md#improving-nfs-performance-with-gitlab) calls; click through for details of these calls +- **Rugged calls**: the time taken (in milliseconds) and the total number of + [Rugged](../../high_availability/nfs.md#improving-nfs-performance-with-gitlab) calls. + Click to display a modal window with more details:  -- time taken and number of Redis calls; click through for details of these calls +- **Redis calls**: the time taken (in milliseconds) and the total number of + Redis calls. Click to display a modal window with more details:  -- total load timings of the page; click through for details of these calls. Values in the following order: - - Backend - Time that the actual base page took to load - - [First Contentful Paint](hhttps://web.dev/first-contentful-paint/) - Time until something was visible to the user - - [DomContentLoaded](https://developers.google.com/web/fundamentals/performance/critical-rendering-path/measure-crp) Event - - Number of Requests that the page loaded -  -- a link to add a request's details to the performance bar; the request can be - added by its full URL (authenticated as the current user), or by the value of - its `X-Request-Id` header -- a link to download the raw JSON used to generate the Performance Bar reports - -On the far right is a request selector that allows you to view the same metrics -(excluding the page timing and line profiler) for any requests made while the -page was open. Only the first two requests per unique URL are captured. +- **Elasticsearch calls**: the time taken (in milliseconds) and the total number of + Elasticsearch calls. Click to display a modal window with more details. +- **Load timings** of the page: if your browser supports load timings (Chromium + and Chrome) several values in milliseconds, separated by slashes. + Click to display a modal window with more details. The values, from left to right: + - **Backend**: time needed for the base page to load. + - [**First Contentful Paint**](https://web.dev/first-contentful-paint/): + Time until something was visible to the user. + - [**DomContentLoaded**](https://developers.google.com/web/fundamentals/performance/critical-rendering-path/measure-crp) Event. + - **Total number of requests** the page loaded: +  +- **Trace**: If Jaeger is integrated, **Trace** links to a Jaeger tracing page + with the current request's `correlation_id` included. +- **+**: A link to add a request's details to the performance bar. The request + can be added by its full URL (authenticated as the current user), or by the value of + its `X-Request-Id` header. +- **Download**: a link to download the raw JSON used to generate the Performance Bar reports. +- **Request Selector**: a select box displayed on the right-hand side of the + Performance Bar which enables you to view these metrics for any requests made while + the current page was open. Only the first two requests per unique URL are captured. ## Request warnings -For requests exceeding predefined limits, a warning icon will be shown -next to the failing metric, along with an explanation. In this example, -the Gitaly call duration exceeded the threshold: +Requests exceeding predefined limits display a warning **{warning}** icon and +explanation next to the failing metric. In this example, the Gitaly call duration +exceeded the threshold:  -If any requests on the current page generated warnings, the icon will -appear next to the request selector: +If any requests on the current page generated warnings, the warning icon displays +next to the **Request selector**:  -And requests with warnings are indicated in the request selector with a -`(!)` after their path: +Requests with warnings display `(!)` after their path in the **Request selector**:  ## Enable the Performance Bar via the Admin panel -GitLab Performance Bar is disabled by default. To enable it for a given group, -navigate to **Admin Area > Settings > Metrics and profiling** -(`admin/application_settings/metrics_and_profiling`), and expand the section -**Profiling - Performance bar**. +The GitLab Performance Bar is disabled by default. To enable it for a given group: -The only required setting you need to set is the full path of the group that -will be allowed to display the Performance Bar. -Make sure _Enable the Performance Bar_ is checked and hit -**Save** to save the changes. +1. Sign in as a user with Administrator [permissions](../../../user/permissions.md). +1. In the menu bar, click the **{admin}** **Admin Area** icon. +1. Navigate to **{settings}** **Settings > Metrics and profiling** + (`admin/application_settings/metrics_and_profiling`), and expand the section + **Profiling - Performance bar**. +1. Click **Enable access to the Performance Bar**. +1. In the **Allowed group** field, provide the full path of the group allowed + to access the GitLab Performance Bar. +1. Click **Save changes**. -Once the Performance Bar is enabled, you will need to press the [<kbd>p</kbd> + -<kbd>b</kbd> keyboard shortcut](../../../user/shortcuts.md) to actually -display it. +## Keyboard shortcut for the Performance Bar -You can toggle the Bar using the same shortcut. - - +After enabling the GitLab Performance Bar, press the [<kbd>p</kbd> + +<kbd>b</kbd> keyboard shortcut](../../../user/shortcuts.md) to display it, and +again to hide it. diff --git a/doc/administration/monitoring/prometheus/gitlab_exporter.md b/doc/administration/monitoring/prometheus/gitlab_exporter.md index 3effca4a2bf..686ed14ba42 100644 --- a/doc/administration/monitoring/prometheus/gitlab_exporter.md +++ b/doc/administration/monitoring/prometheus/gitlab_exporter.md @@ -9,27 +9,25 @@ info: To determine the technical writer assigned to the Stage/Group associated w >- Available since [Omnibus GitLab 8.17](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/1132). >- Renamed from `GitLab monitor exporter` to `GitLab exporter` in [GitLab 12.3](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/16511). -The [GitLab exporter](https://gitlab.com/gitlab-org/gitlab-exporter) allows you to -measure various GitLab metrics, pulled from Redis and the database, in Omnibus GitLab +The [GitLab exporter](https://gitlab.com/gitlab-org/gitlab-exporter) enables you to +measure various GitLab metrics pulled from Redis and the database in Omnibus GitLab instances. NOTE: **Note:** -For installations from source you'll have to install and configure it yourself. +For installations from source you must install and configure it yourself. To enable the GitLab exporter in an Omnibus GitLab instance: -1. [Enable Prometheus](index.md#configuring-prometheus) -1. Edit `/etc/gitlab/gitlab.rb` -1. Add or find and uncomment the following line, making sure it's set to `true`: +1. [Enable Prometheus](index.md#configuring-prometheus). +1. Edit `/etc/gitlab/gitlab.rb`. +1. Add, or find and uncomment, the following line, making sure it's set to `true`: ```ruby gitlab_exporter['enable'] = true ``` 1. Save the file and [reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure) - for the changes to take effect + for the changes to take effect. -Prometheus will now automatically begin collecting performance data from -the GitLab exporter exposed under `localhost:9168`. - -[← Back to the main Prometheus page](index.md) +Prometheus automatically begins collecting performance data from +the GitLab exporter exposed at `localhost:9168`. diff --git a/doc/administration/monitoring/prometheus/gitlab_metrics.md b/doc/administration/monitoring/prometheus/gitlab_metrics.md index f3084b1732e..fa5e5da80c3 100644 --- a/doc/administration/monitoring/prometheus/gitlab_metrics.md +++ b/doc/administration/monitoring/prometheus/gitlab_metrics.md @@ -14,18 +14,18 @@ To enable the GitLab Prometheus metrics: 1. [Restart GitLab](../../restart_gitlab.md#omnibus-gitlab-restart) for the changes to take effect. NOTE: **Note:** -For installations from source you'll have to configure it yourself. +For installations from source you must configure it yourself. ## Collecting the metrics GitLab monitors its own internal service metrics, and makes them available at the `/-/metrics` endpoint. Unlike other [Prometheus](https://prometheus.io) exporters, to access -it, the client IP address needs to be [explicitly allowed](../ip_whitelist.md). +the metrics, the client IP address must be [explicitly allowed](../ip_whitelist.md). For [Omnibus GitLab](https://docs.gitlab.com/omnibus/) and Chart installations, these metrics are enabled and collected as of [GitLab 9.4](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/1702). -For source installations or earlier versions, these metrics must be enabled +For source installations, these metrics must be enabled manually and collected by a Prometheus server. For enabling and viewing metrics from Sidekiq nodes, see [Sidekiq metrics](#sidekiq-metrics). @@ -40,7 +40,7 @@ The following metrics are available: | `gitlab_banzai_cacheless_render_real_duration_seconds` | Histogram | 9.4 | Duration of rendering Markdown into HTML when cached output does not exist | `controller`, `action` | | `gitlab_cache_misses_total` | Counter | 10.2 | Cache read miss | `controller`, `action` | | `gitlab_cache_operation_duration_seconds` | Histogram | 10.2 | Cache access time | | -| `gitlab_cache_operations_total` | Counter | 12.2 | Cache operations by controller/action | `controller`, `action`, `operation` | +| `gitlab_cache_operations_total` | Counter | 12.2 | Cache operations by controller or action | `controller`, `action`, `operation` | | `gitlab_ci_pipeline_creation_duration_seconds` | Histogram | 13.0 | Time in seconds it takes to create a CI/CD pipeline | | | `gitlab_ci_pipeline_size_builds` | Histogram | 13.1 | Total number of builds within a pipeline grouped by a pipeline source | `source` | | `job_waiter_started_total` | Counter | 12.9 | Number of batches of jobs started where a web request is waiting for the jobs to complete | `worker` | @@ -92,16 +92,16 @@ The following metrics are available: | `gitlab_view_rendering_duration_seconds` | Histogram | 10.2 | Duration for views (histogram) | `controller`, `action`, `view` | | `http_requests_total` | Counter | 9.4 | Rack request count | `method` | | `http_request_duration_seconds` | Histogram | 9.4 | HTTP response time from rack middleware | `method`, `status` | -| `gitlab_transaction_db_count_total` | Counter | 13.1 | Counter for total number of sql calls | `controller`, `action` | -| `gitlab_transaction_db_write_count_total` | Counter | 13.1 | Counter for total number of write sql calls | `controller`, `action` | -| `gitlab_transaction_db_cached_count_total` | Counter | 13.1 | Counter for total number of cached sql calls | `controller`, `action` | +| `gitlab_transaction_db_count_total` | Counter | 13.1 | Counter for total number of SQL calls | `controller`, `action` | +| `gitlab_transaction_db_write_count_total` | Counter | 13.1 | Counter for total number of write SQL calls | `controller`, `action` | +| `gitlab_transaction_db_cached_count_total` | Counter | 13.1 | Counter for total number of cached SQL calls | `controller`, `action` | | `http_redis_requests_duration_seconds` | Histogram | 13.1 | Redis requests duration during web transactions | `controller`, `action` | | `http_redis_requests_total` | Counter | 13.1 | Redis requests count during web transactions | `controller`, `action` | | `http_elasticsearch_requests_duration_seconds` **(STARTER)** | Histogram | 13.1 | Elasticsearch requests duration during web transactions | `controller`, `action` | | `http_elasticsearch_requests_total` **(STARTER)** | Counter | 13.1 | Elasticsearch requests count during web transactions | `controller`, `action` | | `pipelines_created_total` | Counter | 9.4 | Counter of pipelines created | | | `rack_uncaught_errors_total` | Counter | 9.4 | Rack connections handling uncaught errors count | | -| `user_session_logins_total` | Counter | 9.4 | Counter of how many users have logged in | | +| `user_session_logins_total` | Counter | 9.4 | Counter of how many users have logged in since GitLab was started or restarted | | | `upload_file_does_not_exist` | Counter | 10.7 in EE, 11.5 in CE | Number of times an upload record could not find its file | | | `failed_login_captcha_total` | Gauge | 11.0 | Counter of failed CAPTCHA attempts during login | | | `successful_login_captcha_total` | Gauge | 11.0 | Counter of successful CAPTCHA attempts during login | | @@ -120,7 +120,7 @@ The following metrics can be controlled by feature flags: ## Sidekiq metrics Sidekiq jobs may also gather metrics, and these metrics can be accessed if the -Sidekiq exporter is enabled (for example, using the `monitoring.sidekiq_exporter` +Sidekiq exporter is enabled: for example, using the `monitoring.sidekiq_exporter` configuration option in `gitlab.yml`. These metrics are served from the `/metrics` path on the configured port. @@ -173,6 +173,7 @@ configuration option in `gitlab.yml`. These metrics are served from the | `geo_repositories_retrying_verification_count` | Gauge | 11.2 | Number of repositories verification failures that Geo is actively trying to correct on secondary | `url` | | `geo_wikis_retrying_verification_count` | Gauge | 11.2 | Number of wikis verification failures that Geo is actively trying to correct on secondary | `url` | | `global_search_bulk_cron_queue_size` | Gauge | 12.10 | Number of database records waiting to be synchronized to Elasticsearch | | +| `global_search_awaiting_indexing_queue_size` | Gauge | 13.2 | Number of database updates waiting to be synchronized to Elasticsearch while indexing is paused | | | `package_files_count` | Gauge | 13.0 | Number of package files on primary | `url` | | `package_files_checksummed_count` | Gauge | 13.0 | Number of package files checksummed on primary | `url` | | `package_files_checksum_failed_count` | Gauge | 13.0 | Number of package files failed to calculate the checksum on primary @@ -187,16 +188,16 @@ The following metrics are available: ## Connection pool metrics -These metrics record the status of the database [connection pools](https://api.rubyonrails.org/classes/ActiveRecord/ConnectionAdapters/ConnectionPool.html). +These metrics record the status of the database +[connection pools](https://api.rubyonrails.org/classes/ActiveRecord/ConnectionAdapters/ConnectionPool.html), +and the metrics all have these labels: -They all have these labels: - -1. `class` - the Ruby class being recorded. - 1. `ActiveRecord::Base` is the main database connection. - 1. `Geo::TrackingBase` is the connection to the Geo tracking database, if - enabled. -1. `host` - the host name used to connect to the database. -1. `port` - the port used to connect to the database. +- `class` - the Ruby class being recorded. + - `ActiveRecord::Base` is the main database connection. + - `Geo::TrackingBase` is the connection to the Geo tracking database, if + enabled. +- `host` - the host name used to connect to the database. +- `port` - the port used to connect to the database. | Metric | Type | Since | Description | |:----------------------------------------------|:------|:------|:--------------------------------------------------| @@ -251,15 +252,29 @@ When Puma is used instead of Unicorn, the following metrics are available: | `puma_idle_threads` | Gauge | 12.0 | Number of spawned threads which are not processing a request | | `puma_killer_terminations_total` | Gauge | 12.0 | Number of workers terminated by PumaWorkerKiller | +## Redis metrics + +These client metrics are meant to complement Redis server metrics. +These metrics are broken down per [Redis +instance](https://docs.gitlab.com/omnibus/settings/redis.html#running-with-multiple-redis-instances). +These metrics all have a `storage` label which indicates the Redis +instance (`cache`, `shared_state` etc.). + +| Metric | Type | Since | Description | +|:--------------------------------- |:------- |:----- |:----------- | +| `gitlab_redis_client_exceptions_total` | Counter | 13.2 | Number of Redis client exceptions, broken down by exception class | +| `gitlab_redis_client_requests_total` | Counter | 13.2 | Number of Redis client requests | +| `gitlab_redis_client_requests_duration_seconds` | Histogram | 13.2 | Redis request latency, excluding blocking commands | + ## Metrics shared directory GitLab's Prometheus client requires a directory to store metrics data shared between multi-process services. Those files are shared among all instances running under Unicorn server. The directory must be accessible to all running Unicorn's processes, or -metrics won't function correctly. +metrics can't function correctly. This directory's location is configured using environment variable `prometheus_multiproc_dir`. For best performance, create this directory in `tmpfs`. If GitLab is installed using [Omnibus GitLab](https://docs.gitlab.com/omnibus/) -and `tmpfs` is available, then the metrics directory will be configured for you. +and `tmpfs` is available, then GitLab configures the metrics directory for you. diff --git a/doc/administration/monitoring/prometheus/index.md b/doc/administration/monitoring/prometheus/index.md index 1e233b890a2..f0ad0a1a2e6 100644 --- a/doc/administration/monitoring/prometheus/index.md +++ b/doc/administration/monitoring/prometheus/index.md @@ -145,6 +145,12 @@ To use an external Prometheus server: gitlab_rails['monitoring_whitelist'] = ['127.0.0.0/8', '192.168.0.1'] ``` +1. On **all** GitLab Rails(Puma/Unicorn, Sidekiq) servers, set the Prometheus server IP address and listen port. For example: + + ```ruby + gitlab_rails['prometheus_address'] = '192.168.0.1:9090' + ``` + 1. To scrape NGINX metrics, you'll also need to configure NGINX to allow the Prometheus server IP. For example: @@ -165,54 +171,54 @@ To use an external Prometheus server: ```yaml scrape_configs: - - job_name: nginx - static_configs: - - targets: - - 1.1.1.1:8060 - - job_name: redis - static_configs: - - targets: - - 1.1.1.1:9121 - - job_name: postgres - static_configs: - - targets: - - 1.1.1.1:9187 - - job_name: node - static_configs: - - targets: - - 1.1.1.1:9100 - - job_name: gitlab-workhorse - static_configs: - - targets: - - 1.1.1.1:9229 - - job_name: gitlab-rails - metrics_path: "/-/metrics" - static_configs: - - targets: - - 1.1.1.1:8080 - - job_name: gitlab-sidekiq - static_configs: - - targets: - - 1.1.1.1:8082 - - job_name: gitlab_exporter_database - metrics_path: "/database" - static_configs: - - targets: - - 1.1.1.1:9168 - - job_name: gitlab_exporter_sidekiq - metrics_path: "/sidekiq" - static_configs: - - targets: - - 1.1.1.1:9168 - - job_name: gitlab_exporter_process - metrics_path: "/process" - static_configs: - - targets: - - 1.1.1.1:9168 - - job_name: gitaly - static_configs: - - targets: - - 1.1.1.1:9236 + - job_name: nginx + static_configs: + - targets: + - 1.1.1.1:8060 + - job_name: redis + static_configs: + - targets: + - 1.1.1.1:9121 + - job_name: postgres + static_configs: + - targets: + - 1.1.1.1:9187 + - job_name: node + static_configs: + - targets: + - 1.1.1.1:9100 + - job_name: gitlab-workhorse + static_configs: + - targets: + - 1.1.1.1:9229 + - job_name: gitlab-rails + metrics_path: "/-/metrics" + static_configs: + - targets: + - 1.1.1.1:8080 + - job_name: gitlab-sidekiq + static_configs: + - targets: + - 1.1.1.1:8082 + - job_name: gitlab_exporter_database + metrics_path: "/database" + static_configs: + - targets: + - 1.1.1.1:9168 + - job_name: gitlab_exporter_sidekiq + metrics_path: "/sidekiq" + static_configs: + - targets: + - 1.1.1.1:9168 + - job_name: gitlab_exporter_process + metrics_path: "/process" + static_configs: + - targets: + - 1.1.1.1:9168 + - job_name: gitaly + static_configs: + - targets: + - 1.1.1.1:9236 ``` 1. Reload the Prometheus server. diff --git a/doc/administration/object_storage.md b/doc/administration/object_storage.md index 1dea2de73f6..b51b722fbd7 100644 --- a/doc/administration/object_storage.md +++ b/doc/administration/object_storage.md @@ -11,29 +11,430 @@ typically much more performant, reliable, and scalable. ## Options -Object storage options that GitLab has tested, or is aware of customers using include: +GitLab has been tested on a number of object storage providers: -- SaaS/Cloud solutions such as [Amazon S3](https://aws.amazon.com/s3/), [Google cloud storage](https://cloud.google.com/storage). +- [Amazon S3](https://aws.amazon.com/s3/) +- [Google Cloud Storage](https://cloud.google.com/storage) +- [Digital Ocean Spaces](https://www.digitalocean.com/products/spaces) +- [Oracle Cloud Infrastructure](https://docs.cloud.oracle.com/en-us/iaas/Content/Object/Tasks/s3compatibleapi.htm) +- [Openstack Swift](https://docs.openstack.org/swift/latest/s3_compat.html) - On-premises hardware and appliances from various storage vendors. - MinIO. We have [a guide to deploying this](https://docs.gitlab.com/charts/advanced/external-object-storage/minio.html) within our Helm Chart documentation. ## Configuration guides -For configuring GitLab to use Object Storage refer to the following guides: - -1. Configure [object storage for backups](../raketasks/backup_restore.md#uploading-backups-to-a-remote-cloud-storage). -1. Configure [object storage for job artifacts](job_artifacts.md#using-object-storage) - including [incremental logging](job_logs.md#new-incremental-logging-architecture). -1. Configure [object storage for LFS objects](lfs/index.md#storing-lfs-objects-in-remote-object-storage). -1. Configure [object storage for uploads](uploads.md#using-object-storage-core-only). -1. Configure [object storage for merge request diffs](merge_request_diffs.md#using-object-storage). -1. Configure [object storage for Container Registry](packages/container_registry.md#container-registry-storage-driver) (optional feature). -1. Configure [object storage for Mattermost](https://docs.mattermost.com/administration/config-settings.html#file-storage) (optional feature). -1. Configure [object storage for packages](packages/index.md#using-object-storage) (optional feature). **(PREMIUM ONLY)** -1. Configure [object storage for Dependency Proxy](packages/dependency_proxy.md#using-object-storage) (optional feature). **(PREMIUM ONLY)** -1. Configure [object storage for Pseudonymizer](pseudonymizer.md#configuration) (optional feature). **(ULTIMATE ONLY)** -1. Configure [object storage for autoscale Runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional - for improved performance). -1. Configure [object storage for Terraform state files](terraform_state.md#using-object-storage-core-only) +There are two ways of specifying object storage configuration in GitLab: + +- [Consolidated form](#consolidated-object-storage-configuration): A single credential is + shared by all supported object types. +- [Storage-specific form](#storage-specific-configuration): Every object defines its + own object storage [connection and configuration](#connection-settings). + +For more information on the differences and to transition from one form to another, see +[Transition to consolidated form](#transition-to-consolidated-form). + +### Consolidated object storage configuration + +> Introduced in [GitLab 13.2](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/4368). + +Using the consolidated object storage configuration has a number of advantages: + +- It can simplify your GitLab configuration since the connection details are shared + across object types. +- It enables the use of [encrypted S3 buckets](#encrypted-s3-buckets). +- It [uploads files to S3 with proper `Content-MD5` headers](https://gitlab.com/gitlab-org/gitlab-workhorse/-/issues/222). + +NOTE: **Note:** +Only AWS S3-compatible providers and Google are +supported at the moment since [direct upload +mode](../development/uploads.md#direct-upload) must be used. Background +upload is not supported in this mode. We recommend direct upload mode because +it does not require a shared folder, and [this setting may become the default](https://gitlab.com/gitlab-org/gitlab/-/issues/27331). + +NOTE: **Note:** +Consolidated object storage configuration cannot be used for +backups or Mattermost. See [the full table for a complete list](#storage-specific-configuration). + +Most types of objects, such as CI artifacts, LFS files, upload +attachments, and so on can be saved in object storage by specifying a single +credential for object storage with multiple buckets. A [different bucket +for each type must be used](#use-separate-buckets). + +When the consolidated form is: + +- Used with an S3-compatible object storage, Workhorse uses its internal S3 client to + upload files. +- Not used with an S3-compatible object storage, Workhorse falls back to using + pre-signed URLs. + +See the section on [ETag mismatch errors](#etag-mismatch) for more details. + +**In Omnibus installations:** + +1. Edit `/etc/gitlab/gitlab.rb` and add the following lines, substituting + the values you want: + + ```ruby + # Consolidated object storage configuration + gitlab_rails['object_store']['enabled'] = true + gitlab_rails['object_store']['proxy_download'] = true + gitlab_rails['object_store']['connection'] = { + 'provider' => 'AWS', + 'region' => '<eu-central-1>', + 'aws_access_key_id' => '<AWS_ACCESS_KEY_ID>', + 'aws_secret_access_key' => '<AWS_SECRET_ACCESS_KEY>' + } + gitlab_rails['object_store']['objects']['artifacts']['bucket'] = '<artifacts>' + gitlab_rails['object_store']['objects']['external_diffs']['bucket'] = '<external-diffs>' + gitlab_rails['object_store']['objects']['lfs']['bucket'] = '<lfs-objects>' + gitlab_rails['object_store']['objects']['uploads']['bucket'] = '<uploads>' + gitlab_rails['object_store']['objects']['packages']['bucket'] = '<packages>' + gitlab_rails['object_store']['objects']['dependency_proxy']['bucket'] = '<dependency-proxy>' + gitlab_rails['object_store']['objects']['terraform_state']['bucket'] = '<terraform-state>' + ``` + + NOTE: For GitLab 9.4 or later, if you're using AWS IAM profiles, be sure to omit the + AWS access key and secret access key/value pairs. For example: + + ```ruby + gitlab_rails['object_store_connection'] = { + 'provider' => 'AWS', + 'region' => '<eu-central-1>', + 'use_iam_profile' => true + } + ``` + +1. Save the file and [reconfigure GitLab](restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + +**In installations from source:** + +1. Edit `/home/git/gitlab/config/gitlab.yml` and add or amend the following lines: + + ```yaml + object_store: + enabled: true + proxy_download: true + connection: + provider: AWS + aws_access_key_id: <AWS_ACCESS_KEY_ID> + aws_secret_access_key: <AWS_SECRET_ACCESS_KEY> + region: <eu-central-1> + objects: + artifacts: + bucket: <artifacts> + external_diffs: + bucket: <external-diffs> + lfs: + bucket: <lfs-objects> + uploads: + bucket: <uploads> + packages: + bucket: <packages> + dependency_proxy: + bucket: <dependency_proxy> + terraform_state: + bucket: <terraform> + ``` + +1. Edit `/home/git/gitlab-workhorse/config.toml` and add or amend the following lines: + + ```toml + [object_storage] + enabled = true + provider = "AWS" + + [object_storage.s3] + aws_access_key_id = "<AWS_ACCESS_KEY_ID>" + aws_secret_access_key = "<AWS_SECRET_ACCESS_KEY>" + ``` + +1. Save the file and [restart GitLab](restart_gitlab.md#installations-from-source) for the changes to take effect. + +#### Common parameters + +In the consolidated configuration, the `object_store` section defines a +common set of parameters. Here we use the YAML from the source +installation because it's easier to see the inheritance: + +```yaml + object_store: + enabled: true + proxy_download: true + connection: + provider: AWS + aws_access_key_id: <AWS_ACCESS_KEY_ID> + aws_secret_access_key: <AWS_SECRET_ACCESS_KEY> + objects: + ... +``` + +The Omnibus configuration maps directly to this: + +```ruby +gitlab_rails['object_store']['enabled'] = true +gitlab_rails['object_store']['proxy_download'] = true +gitlab_rails['object_store']['connection'] = { + 'provider' => 'AWS', + 'aws_access_key_id' => '<AWS_ACCESS_KEY_ID', + 'aws_secret_access_key' => '<AWS_SECRET_ACCESS_KEY>' +} +``` + +| Setting | Description | +|---------|-------------| +| `enabled` | Enable/disable object storage | +| `proxy_download` | Set to `true` to [enable proxying all files served](#proxy-download). Option allows to reduce egress traffic as this allows clients to download directly from remote storage instead of proxying all data | +| `connection` | Various connection options described below | +| `objects` | [Object-specific configuration](#object-specific-configuration) + +### Connection settings + +Both consolidated configuration form and storage-specific configuration form must configure a connection. The following sections describe parameters that can be used +in the `connection` setting. + +#### S3-compatible connection settings + +The connection settings match those provided by [fog-aws](https://github.com/fog/fog-aws): + +| Setting | Description | Default | +|---------|-------------|---------| +| `provider` | Always `AWS` for compatible hosts | `AWS` | +| `aws_access_key_id` | AWS credentials, or compatible | | +| `aws_secret_access_key` | AWS credentials, or compatible | | +| `aws_signature_version` | AWS signature version to use. `2` or `4` are valid options. Digital Ocean Spaces and other providers may need `2`. | `4` | +| `enable_signature_v4_streaming` | Set to `true` to enable HTTP chunked transfers with [AWS v4 signatures](https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html). Oracle Cloud S3 needs this to be `false`. | `true` | +| `region` | AWS region | us-east-1 | +| `host` | S3 compatible host for when not using AWS, e.g. `localhost` or `storage.example.com`. HTTPS and port 443 is assumed. | `s3.amazonaws.com` | +| `endpoint` | Can be used when configuring an S3 compatible service such as [MinIO](https://min.io), by entering a URL such as `http://127.0.0.1:9000`. This takes precedence over `host`. | (optional) | +| `path_style` | Set to `true` to use `host/bucket_name/object` style paths instead of `bucket_name.host/object`. Leave as `false` for AWS S3. | `false` | +| `use_iam_profile` | Set to `true` to use IAM profile instead of access keys | `false` + +#### Oracle Cloud S3 connection settings + +Note that Oracle Cloud S3 must be sure to use the following settings: + +| Setting | Value | +|---------|-------| +| `enable_signature_v4_streaming` | `false` | +| `path_style` | `true` | + +If `enable_signature_v4_streaming` is set to `true`, you may see the +following error in `production.log`: + +```plaintext +STREAMING-AWS4-HMAC-SHA256-PAYLOAD is not supported +``` + +#### Google Cloud Storage (GCS) + +Here are the valid connection parameters for GCS: + +| Setting | Description | example | +|---------|-------------|---------| +| `provider` | The provider name | `Google` | +| `google_project` | GCP project name | `gcp-project-12345` | +| `google_client_email` | The email address of the service account | `foo@gcp-project-12345.iam.gserviceaccount.com` | +| `google_json_key_location` | The JSON key path | `/path/to/gcp-project-12345-abcde.json` | + +NOTE: **Note:** +The service account must have permission to access the bucket. +[See more](https://cloud.google.com/storage/docs/authentication) + +##### Google example (consolidated form) + +For Omnibus installations, this is an example of the `connection` setting: + +```ruby +gitlab_rails['object_store']['connection'] = { + 'provider' => 'Google', + 'google_project' => '<GOOGLE PROJECT>', + 'google_client_email' => '<GOOGLE CLIENT EMAIL>', + 'google_json_key_location' => '<FILENAME>' +} +``` + +#### OpenStack-compatible connection settings + +NOTE: **Note:** +This is not compatible with the consolidated object storage form. +OpenStack Swift is only supported with the storage-specific form. See the +[S3 settings](#s3-compatible-connection-settings) if you want to use the consolidated form. + +While OpenStack Swift provides S3 compatibliity, some users may want to use the +[Swift API](https://docs.openstack.org/swift/latest/api/object_api_v1_overview.html). +Here are the valid connection settings below for the Swift API, provided by +[fog-openstack](https://github.com/fog/fog-openstack): + +| Setting | Description | Default | +|---------|-------------|---------| +| `provider` | Always `OpenStack` for compatible hosts | `OpenStack` | +| `openstack_username` | OpenStack username | | +| `openstack_api_key` | OpenStack API key | | +| `openstack_temp_url_key` | OpenStack key for generating temporary URLs | | +| `openstack_auth_url` | OpenStack authentication endpoint | | +| `openstack_region` | OpenStack region | | +| `openstack_tenant` | OpenStack tenant ID | + +#### Rackspace Cloud Files + +NOTE: **Note:** +This is not compatible with the consolidated object +storage form. Rackspace Cloud is only supported with the storage-specific form. + +Here are the valid connection parameters for Rackspace Cloud, provided by +[fog-rackspace](https://github.com/fog/fog-rackspace/): + +| Setting | Description | example | +|---------|-------------|---------| +| `provider` | The provider name | `Rackspace` | +| `rackspace_username` | The username of the Rackspace account with access to the container | `joe.smith` | +| `rackspace_api_key` | The API key of the Rackspace account with access to the container | `ABC123DEF456ABC123DEF456ABC123DE` | +| `rackspace_region` | The Rackspace storage region to use, a three letter code from the [list of service access endpoints](https://developer.rackspace.com/docs/cloud-files/v1/general-api-info/service-access/) | `iad` | +| `rackspace_temp_url_key` | The private key you have set in the Rackspace API for temporary URLs. Read more [here](https://developer.rackspace.com/docs/cloud-files/v1/use-cases/public-access-to-your-cloud-files-account/#tempurl) | `ABC123DEF456ABC123DEF456ABC123DE` | + +NOTE: **Note:** +Regardless of whether the container has public access enabled or disabled, Fog will +use the TempURL method to grant access to LFS objects. If you see errors in logs referencing +instantiating storage with a `temp-url-key`, ensure that you have set the key properly +on the Rackspace API and in `gitlab.rb`. You can verify the value of the key Rackspace +has set by sending a GET request with token header to the service access endpoint URL +and comparing the output of the returned headers. + +### Object-specific configuration + +The following YAML shows how the `object_store` section defines +object-specific configuration block and how the `enabled` and +`proxy_download` flags can be overriden. The `bucket` is the only +required parameter within each type: + +```yaml + object_store: + connection: + ... + objects: + artifacts: + bucket: artifacts + proxy_download: false + external_diffs: + bucket: external-diffs + lfs: + bucket: lfs-objects + uploads: + bucket: uploads + packages: + bucket: packages + dependency_proxy: + enabled: false + bucket: dependency_proxy + terraform_state: + bucket: terraform +``` + +This maps to this Omnibus GitLab configuration: + +```ruby +gitlab_rails['object_store']['objects']['artifacts']['bucket'] = 'artifacts' +gitlab_rails['object_store']['objects']['artifacts']['proxy_download'] = false +gitlab_rails['object_store']['objects']['external_diffs']['bucket'] = 'external-diffs' +gitlab_rails['object_store']['objects']['lfs']['bucket'] = 'lfs-objects' +gitlab_rails['object_store']['objects']['uploads']['bucket'] = 'uploads' +gitlab_rails['object_store']['objects']['packages']['bucket'] = 'packages' +gitlab_rails['object_store']['objects']['dependency_proxy']['enabled'] = false +gitlab_rails['object_store']['objects']['dependency_proxy']['bucket'] = 'dependency-proxy' +gitlab_rails['object_store']['objects']['terraform_state']['bucket'] = 'terraform-state' +``` + +This is the list of valid `objects` that can be used: + +| Type | Description | +|--------------------|---------------| +| `artifacts` | [CI artifacts](job_artifacts.md) | +| `external_diffs` | [Merge request diffs](merge_request_diffs.md) | +| `uploads` | [User uploads](uploads.md) | +| `lfs` | [Git Large File Storage objects](lfs/index.md) | +| `packages` | [Project packages (e.g. PyPI, Maven, NuGet, etc.)](packages/index.md) | +| `dependency_proxy` | [GitLab Dependency Proxy](packages/dependency_proxy.md) | +| `terraform_state` | [Terraform state files](terraform_state.md) | + +Within each object type, three parameters can be defined: + +| Setting | Required? | Description | +|------------------|-----------|-------------| +| `bucket` | Yes | The bucket name for the object storage. | +| `enabled` | No | Overrides the common parameter | +| `proxy_download` | No | Overrides the common parameter | + +#### Selectively disabling object storage + +As seen above, object storage can be disabled for specific types by +setting the `enabled` flag to `false`. For example, to disable object +storage for CI artifacts: + +```ruby +gitlab_rails['object_store']['objects']['artifacts']['enabled'] = false +``` + +A bucket is not needed if the feature is disabled entirely. For example, +no bucket is needed if CI artifacts are disabled with this setting: + +```ruby +gitlab_rails['artifacts_enabled'] = false +``` + +### Transition to consolidated form + +Prior to GitLab 13.2: + +- Object storage configuration for all types of objects such as CI/CD artifacts, LFS + files, upload attachments, and so on had to be configured independently. +- Object store connection parameters such as passwords and endpoint URLs had to be + duplicated for each type. + +For example, an Omnibus GitLab install might have the following configuration: + +```ruby +# Original object storage configuration +gitlab_rails['artifacts_object_store_enabled'] = true +gitlab_rails['artifacts_object_store_direct_upload'] = true +gitlab_rails['artifacts_object_store_proxy_download'] = true +gitlab_rails['artifacts_object_store_remote_directory'] = 'artifacts' +gitlab_rails['artifacts_object_store_connection'] = { 'provider' => 'AWS', 'aws_access_key_id' => 'access_key', 'aws_secret_access_key' => 'secret' } +gitlab_rails['uploads_object_store_enabled'] = true +gitlab_rails['uploads_object_store_direct_upload'] = true +gitlab_rails['uploads_object_store_proxy_download'] = true +gitlab_rails['uploads_object_store_remote_directory'] = 'uploads' +gitlab_rails['uploads_object_store_connection'] = { 'provider' => 'AWS', 'aws_access_key_id' => 'access_key', 'aws_secret_access_key' => 'secret' } +``` + +While this provides flexibility in that it makes it possible for GitLab +to store objects across different cloud providers, it also creates +additional complexity and unnecessary redundancy. Since both GitLab +Rails and Workhorse components need access to object storage, the +consolidated form avoids excessive duplication of credentials. + +NOTE: **Note:** +The consolidated object storage configuration is **only** used if all +lines from the original form is omitted. To move to the consolidated form, remove the original configuration (for example, `artifacts_object_store_enabled`, `uploads_object_store_connection`, and so on.) + +## Storage-specific configuration + +For configuring object storage in GitLab 13.1 and earlier, or for storage types not +supported by consolidated configuration form, refer to the following guides: + +|Object storage type|Supported by consolidated configuration?| +|-------------------|----------------------------------------| +| [Backups](../raketasks/backup_restore.md#uploading-backups-to-a-remote-cloud-storage)|No| +| [Job artifacts](job_artifacts.md#using-object-storage) and [incremental logging](job_logs.md#new-incremental-logging-architecture) | Yes | +| [LFS objects](lfs/index.md#storing-lfs-objects-in-remote-object-storage) | Yes | +| [Uploads](uploads.md#using-object-storage-core-only) | Yes | +| [Container Registry](packages/container_registry.md#use-object-storage) (optional feature) | No | +| [Merge request diffs](merge_request_diffs.md#using-object-storage) | Yes | +| [Mattermost](https://docs.mattermost.com/administration/config-settings.html#file-storage)| No | +| [Packages](packages/index.md#using-object-storage) (optional feature) **(PREMIUM ONLY)** | Yes | +| [Dependency Proxy](packages/dependency_proxy.md#using-object-storage) (optional feature) **(PREMIUM ONLY)** | Yes | +| [Pseudonymizer](pseudonymizer.md#configuration) (optional feature) **(ULTIMATE ONLY)** | No | +| [Autoscale Runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional for improved performance) | No | +| [Terraform state files](terraform_state.md#using-object-storage-core-only) | Yes | ### Other alternatives to filesystem storage @@ -69,7 +470,7 @@ backups might not be realised until the organisation had a critical requirement ### S3 API compatibility issues Not all S3 providers [are fully compatible](../raketasks/backup_restore.md#other-s3-providers) -with the Fog library that GitLab uses. Symptoms include: +with the Fog library that GitLab uses. Symptoms include an error in `production.log`: ```plaintext 411 Length Required @@ -143,14 +544,26 @@ might generate `ETag mismatch` errors. If you are seeing this ETag mismatch error with Amazon Web Services S3, it's likely this is due to [encryption settings on your bucket](https://docs.aws.amazon.com/AmazonS3/latest/API/RESTCommonResponseHeaders.html). -See the section on [using Amazon instance profiles](#using-amazon-instance-profiles) on how to fix this issue. +To fix this issue, you have two options: -When using GitLab direct upload, the +- [Use the consolidated object configuration](#consolidated-object-storage-configuration). +- [Use Amazon instance profiles](#using-amazon-instance-profiles). + +The first option is recommended for MinIO. Otherwise, the [workaround for MinIO](https://gitlab.com/gitlab-org/charts/gitlab/-/issues/1564#note_244497658) is to use the `--compat` parameter on the server. -We are working on a fix to the [GitLab Workhorse -component](https://gitlab.com/gitlab-org/gitlab-workhorse/-/issues/222). +Without consolidated object store configuration or instance profiles enabled, +GitLab Workhorse will upload files to S3 using pre-signed URLs that do +not have a `Content-MD5` HTTP header computed for them. To ensure data +is not corrupted, Workhorse checks that the MD5 hash of the data sent +equals the ETag header returned from the S3 server. When encryption is +enabled, this is not the case, which causes Workhorse to report an `ETag +mismatch` error during an upload. + +With the consolidated object configuration and instance profile, Workhorse has +S3 credentials so that it can compute the `Content-MD5` header. This +eliminates the need to compare ETag headers returned from the S3 server. ### Using Amazon instance profiles @@ -163,66 +576,57 @@ configuration. #### Encrypted S3 buckets -> Introduced in [GitLab 13.1](https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/466) only for instance profiles. +> - Introduced in [GitLab 13.1](https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/466) for instance profiles only. +> - Introduced in [GitLab 13.2](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/34460) for static credentials when [consolidated object storage configuration](#consolidated-object-storage-configuration) is used. -When configured to use an instance profile, GitLab Workhorse -will properly upload files to S3 buckets that have [SSE-S3 or SSE-KMS -encryption enabled by default](https://docs.aws.amazon.com/kms/latest/developerguide/services-s3.html). -Note that customer master keys (CMKs) and SSE-C encryption are not yet -supported since this requires supplying keys to the GitLab -configuration. +When configured either with an instance profile or with the consolidated +object configuration, GitLab Workhorse properly uploads files to S3 buckets +that have [SSE-S3 or SSE-KMS encryption enabled by +default](https://docs.aws.amazon.com/kms/latest/developerguide/services-s3.html). +Note that customer master keys (CMKs) and +SSE-C encryption are [not yet supported since this requires supplying +keys to the GitLab configuration](https://gitlab.com/gitlab-org/gitlab/-/issues/226006). -Without instance profiles enabled (or prior to GitLab 13.1), GitLab -Workhorse will upload files to S3 using pre-signed URLs that do not have -a `Content-MD5` HTTP header computed for them. To ensure data is not -corrupted, Workhorse checks that the MD5 hash of the data sent equals -the ETag header returned from the S3 server. When encryption is enabled, -this is not the case, which causes Workhorse to report an `ETag -mismatch` error during an upload. +##### Disabling the feature -With instance profiles enabled, GitLab Workhorse uses an AWS S3 client -that properly computes and sends the `Content-MD5` header to the server, -which eliminates the need for comparing ETag headers. If the data is -corrupted in transit, the S3 server will reject the file. +The Workhorse S3 client is enabled by default when the +[`use_iam_profile` configuration option](#iam-permissions) is set to `true`. -#### IAM Permissions - -To set up an instance profile, create an Amazon Identity Access and -Management (IAM) role with the necessary permissions. The following -example is a role for an S3 bucket named `test-bucket`: - -```json -{ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "VisualEditor0", - "Effect": "Allow", - "Action": [ - "s3:PutObject", - "s3:GetObject", - "s3:AbortMultipartUpload", - "s3:DeleteObject" - ], - "Resource": "arn:aws:s3:::test-bucket/*" - } - ] -} -``` - -Associate this role with your GitLab instance, and then configure GitLab -to use it via the `use_iam_profile` configuration option. For example, -when configuring uploads to use object storage, see the `AWS IAM profiles` -section in [S3 compatible connection settings](uploads.md#s3-compatible-connection-settings). - -#### Disabling the feature - -The Workhorse S3 client is only enabled when the `use_iam_profile` -configuration flag is `true`. - -To disable this feature, ask a GitLab administrator with [Rails console access](feature_flags.md#how-to-enable-and-disable-features-behind-flags) to run the +The feature can be disabled using the `:use_workhorse_s3_client` feature flag. To disable the +feature, ask a GitLab administrator with +[Rails console access](feature_flags.md#how-to-enable-and-disable-features-behind-flags) to run the following command: ```ruby Feature.disable(:use_workhorse_s3_client) ``` + +#### IAM Permissions + +To set up an instance profile: + +1. Create an Amazon Identity Access and Management (IAM) role with the necessary permissions. The + following example is a role for an S3 bucket named `test-bucket`: + + ```json + { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "VisualEditor0", + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObject", + "s3:AbortMultipartUpload", + "s3:DeleteObject" + ], + "Resource": "arn:aws:s3:::test-bucket/*" + } + ] + } + ``` + +1. [Attach this role](https://aws.amazon.com/premiumsupport/knowledge-center/attach-replace-ec2-instance-profile/) + to the EC2 instance hosting your GitLab instance. +1. Configure GitLab to use it via the `use_iam_profile` configuration option. diff --git a/doc/administration/operations/fast_ssh_key_lookup.md b/doc/administration/operations/fast_ssh_key_lookup.md index 9f67c927128..b874a4257f0 100644 --- a/doc/administration/operations/fast_ssh_key_lookup.md +++ b/doc/administration/operations/fast_ssh_key_lookup.md @@ -3,7 +3,8 @@ > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/1631) in [GitLab Starter](https://about.gitlab.com/pricing/) 9.3. > - [Available in](https://gitlab.com/gitlab-org/gitlab/-/issues/3953) GitLab Community Edition 10.4. -NOTE: **Note:** This document describes a drop-in replacement for the +NOTE: **Note:** +This document describes a drop-in replacement for the `authorized_keys` file. For normal (non-deploy key) users, consider using [SSH certificates](ssh_certificates.md). They are even faster, but are not a drop-in replacement. @@ -67,19 +68,25 @@ sudo service ssh reload sudo service sshd reload ``` -Confirm that SSH is working by removing your user's SSH key in the UI, adding a -new one, and attempting to pull a repository. +Confirm that SSH is working by commenting out your user's key in the `authorized_keys` +(start the line with a `#` to comment it), and attempting to pull a repository. -NOTE: **Note:** For Omnibus Docker, `AuthorizedKeysCommand` is setup by default in +A successful pull would mean that GitLab was able to find the key in the database, +since it is not present in the file anymore. + +NOTE: **Note:** +For Omnibus Docker, `AuthorizedKeysCommand` is setup by default in GitLab 11.11 and later. -NOTE: **Note:** For Installations from source, the command would be located at +NOTE: **Note:** +For Installations from source, the command would be located at `/home/git/gitlab-shell/bin/gitlab-shell-authorized-keys-check` if [the install from source](../../install/installation.md#install-gitlab-shell) instructions were followed. You might want to consider creating a wrapper script somewhere else since this command needs to be owned by `root` and not be writable by group or others. You could also consider changing the ownership of this command as required, but that might require temporary ownership changes during `gitlab-shell` upgrades. -CAUTION: **Caution:** Do not disable writes until SSH is confirmed to be working +CAUTION: **Caution:** +Do not disable writes until SSH is confirmed to be working perfectly, because the file will quickly become out-of-date. In the case of lookup failures (which are common), the `authorized_keys` @@ -96,6 +103,8 @@ Again, confirm that SSH is working by removing your user's SSH key in the UI, adding a new one, and attempting to pull a repository. Then you can backup and delete your `authorized_keys` file for best performance. +The current users' keys are already present in the database, so there is no need for migration +or for asking users to re-add their keys. ## How to go back to using the `authorized_keys` file @@ -200,3 +209,13 @@ the database. The following instructions can be used to build OpenSSH 7.5: # Only run this if you run into a problem logging in yum downgrade openssh-server openssh openssh-clients ``` + +## SELinux support and limitations + +> [Introduced](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/2855) in GitLab 10.5. + +GitLab supports `authorized_keys` database lookups with [SELinux](https://en.wikipedia.org/wiki/Security-Enhanced_Linux). + +Because the SELinux policy is static, GitLab doesn't support the ability to change +internal Unicorn ports at the moment. Admins would have to create a special `.te` +file for the environment, since it isn't generated dynamically. diff --git a/doc/administration/operations/filesystem_benchmarking.md b/doc/administration/operations/filesystem_benchmarking.md index c5c5a8b4313..856061348ed 100644 --- a/doc/administration/operations/filesystem_benchmarking.md +++ b/doc/administration/operations/filesystem_benchmarking.md @@ -65,7 +65,8 @@ operations per second. ### Simple benchmarking -NOTE: **Note:** This test is naive but may be useful if `fio` is not +NOTE: **Note:** +This test is naive but may be useful if `fio` is not available on the system. It's possible to receive good results on this test but still have poor performance due to read speed and various other factors. diff --git a/doc/administration/operations/puma.md b/doc/administration/operations/puma.md index af28335ef91..62b93d40a6b 100644 --- a/doc/administration/operations/puma.md +++ b/doc/administration/operations/puma.md @@ -1,11 +1,11 @@ # Switching to Puma -## Puma - As of GitLab 12.9, [Puma](https://github.com/puma/puma) has replaced [Unicorn](https://yhbt.net/unicorn/). -as the default web server. Starting with 13.0, both all-in-one package based -installations as well as Helm chart based installations will run Puma instead of -Unicorn unless explicitly specified not to. +as the default web server. From GitLab 13.0, the following run Puma instead of Unicorn unless +explicitly configured not to: + +- All-in-one package-based installations. +- Helm chart-based installations. ## Why switch to Puma? @@ -32,10 +32,12 @@ Additionally we strongly recommend that multi-node deployments [configure their ## Performance caveat when using Puma with Rugged For deployments where NFS is used to store Git repository, we allow GitLab to use -[Direct Git Access](../gitaly/#direct-git-access-in-gitlab-rails) to improve performance via usage of [Rugged](https://github.com/libgit2/rugged). +[direct Git access](../gitaly/index.md#direct-access-to-git-in-gitlab) to improve performance using +[Rugged](https://github.com/libgit2/rugged). -Rugged usage is automatically enabled if Direct Git Access is present, unless it -is disabled by [feature flags](../../development/gitaly.md#legacy-rugged-code). +Rugged usage is automatically enabled if direct Git access +[is available](../gitaly/index.md#how-it-works), unless it is disabled by +[feature flags](../../development/gitaly.md#legacy-rugged-code). MRI Ruby uses a GVL. This allows MRI Ruby to be multi-threaded, but running at most on a single core. Since Rugged can use a thread for long periods of diff --git a/doc/administration/operations/unicorn.md b/doc/administration/operations/unicorn.md index eabf99eb08c..a1593c3a6c3 100644 --- a/doc/administration/operations/unicorn.md +++ b/doc/administration/operations/unicorn.md @@ -45,7 +45,7 @@ master process has PID 56227 below. The main tunable options for Unicorn are the number of worker processes and the request timeout after which the Unicorn master terminates a worker process. See the [Omnibus GitLab Unicorn settings -documentation](https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/doc/settings/unicorn.md) +documentation](https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/doc/settings/unicorn.html) if you want to adjust these settings. ## unicorn-worker-killer diff --git a/doc/administration/packages/container_registry.md b/doc/administration/packages/container_registry.md index 8f55345a9a8..44f1d075a5e 100644 --- a/doc/administration/packages/container_registry.md +++ b/doc/administration/packages/container_registry.md @@ -76,7 +76,7 @@ where: | `port` | The port under which the external Registry domain will listen on. | | `api_url` | The internal API URL under which the Registry is exposed to. It defaults to `http://localhost:5000`. | | `key` | The private key location that is a pair of Registry's `rootcertbundle`. Read the [token auth configuration documentation](https://docs.docker.com/registry/configuration/#token). | -| `path` | This should be the same directory like specified in Registry's `rootdirectory`. Read the [storage configuration documentation](https://docs.docker.com/registry/configuration/#storage). This path needs to be readable by the GitLab user, the web-server user and the Registry user. Read more in [#container-registry-storage-path](#container-registry-storage-path). | +| `path` | This should be the same directory like specified in Registry's `rootdirectory`. Read the [storage configuration documentation](https://docs.docker.com/registry/configuration/#storage). This path needs to be readable by the GitLab user, the web-server user and the Registry user. Read more in [#configure-storage-for-the-container-registry](#configure-storage-for-the-container-registry). | | `issuer` | This should be the same value as configured in Registry's `issuer`. Read the [token auth configuration documentation](https://docs.docker.com/registry/configuration/#token). | NOTE: **Note:** @@ -313,11 +313,28 @@ the Container Registry by themselves, follow the steps below. 1. Save the file and [restart GitLab](../restart_gitlab.md#installations-from-source) for the changes to take effect. -## Container Registry storage path +## Configure storage for the Container Registry -NOTE: **Note:** -For configuring storage in the cloud instead of the filesystem, see the -[storage driver configuration](#container-registry-storage-driver). +You can configure the Container Registry to use various storage backends by +configuring a storage driver. By default the GitLab Container Registry +is configured to use the [filesystem driver](#use-filesystem) +configuration. + +The different supported drivers are: + +| Driver | Description | +|------------|-------------------------------------| +| filesystem | Uses a path on the local filesystem | +| Azure | Microsoft Azure Blob Storage | +| gcs | Google Cloud Storage | +| s3 | Amazon Simple Storage Service. Be sure to configure your storage bucket with the correct [S3 Permission Scopes](https://docs.docker.com/registry/storage-drivers/s3/#s3-permission-scopes). | +| swift | OpenStack Swift Object Storage | +| oss | Aliyun OSS | + +Read more about the individual driver's configuration options in the +[Docker Registry docs](https://docs.docker.com/registry/configuration/#storage). + +### Use filesystem If you want to store your images on the filesystem, you can change the storage path for the Container Registry, follow the steps below. @@ -327,7 +344,8 @@ This path is accessible to: - The user running the Container Registry daemon. - The user running GitLab. -CAUTION: **Warning** You should confirm that all GitLab, Registry and web server users +CAUTION: **Warning:** +You should confirm that all GitLab, Registry and web server users have access to this directory. **Omnibus GitLab installations** @@ -358,30 +376,15 @@ The default location where images are stored in source installations, is 1. Save the file and [restart GitLab](../restart_gitlab.md#installations-from-source) for the changes to take effect. -### Container Registry storage driver +### Use object storage -You can configure the Container Registry to use a different storage backend by -configuring a different storage driver. By default the GitLab Container Registry -is configured to use the filesystem driver, which makes use of [storage path](#container-registry-storage-path) -configuration. - -The different supported drivers are: - -| Driver | Description | -|------------|-------------------------------------| -| filesystem | Uses a path on the local filesystem | -| Azure | Microsoft Azure Blob Storage | -| gcs | Google Cloud Storage | -| s3 | Amazon Simple Storage Service. Be sure to configure your storage bucket with the correct [S3 Permission Scopes](https://docs.docker.com/registry/storage-drivers/s3/#s3-permission-scopes). | -| swift | OpenStack Swift Object Storage | -| oss | Aliyun OSS | - -Read more about the individual driver's configuration options in the -[Docker Registry docs](https://docs.docker.com/registry/configuration/#storage). +If you want to store your images on object storage, you can change the storage +driver for the Container Registry. [Read more about using object storage with GitLab](../object_storage.md). -CAUTION: **Warning:** GitLab will not backup Docker images that are not stored on the +CAUTION: **Warning:** +GitLab will not backup Docker images that are not stored on the filesystem. Remember to enable backups with your object storage provider if desired. @@ -435,21 +438,43 @@ storage: NOTE: **Note:** `your-s3-bucket` should only be the name of a bucket that exists, and can't include subdirectories. -**Migrate without downtime** +#### Migrate to object storage without downtime + +To migrate storage without stopping the Container Registry, set the Container Registry +to read-only mode. On large instances, this may require the Container Registry +to be in read-only mode for a while. During this time, +you can pull from the Container Registry, but you cannot push. + +1. Optional: To reduce the amount of data to be migrated, run the [garbage collection tool without downtime](#performing-garbage-collection-without-downtime). +1. Copy initial data to your S3 bucket, for example with the AWS CLI [`cp`](https://awscli.amazonaws.com/v2/documentation/api/latest/reference/s3/cp.html) + or [`sync`](https://awscli.amazonaws.com/v2/documentation/api/latest/reference/s3/sync.html) + command. Make sure to keep the `docker` folder as the top-level folder inside the bucket. + + ```shell + aws s3 sync registry s3://mybucket + ``` + +1. To perform the final data sync, + [put the Container Registry in `read-only` mode](#performing-garbage-collection-without-downtime) and + [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). +1. Sync any changes since the initial data load to your S3 bucket and delete files that exist in the destination bucket but not in the source: + + ```shell + aws s3 sync registry s3://mybucket --delete + ``` -To migrate the data to AWS S3 without downtime: + DANGER: **Danger:** + The `--delete` flag will delete files that exist in the destination but not in the source. + Make sure not to swap the source and destination, or you will delete all data in the Registry. -1. To reduce the amount of data to be migrated, run the [garbage collection tool without downtime](#performing-garbage-collection-without-downtime). Part of this process sets the registry to `read-only`. -1. Copy the data to your AWS S3 bucket, for example with [AWS CLI's `cp`](https://docs.aws.amazon.com/cli/latest/reference/s3/cp.html) command. -1. Configure your registry to use the S3 bucket for storage. -1. Put the registry back to `read-write`. -1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. +1. Configure your registry to [use the S3 bucket for storage](#use-object-storage). +1. For the changes to take effect, set the Registry back to `read-write` mode and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). ### Disable redirect for storage driver By default, users accessing a registry configured with a remote backend are redirected to the default backend for the storage driver. For example, registries can be configured using the `s3` storage driver, which redirects requests to a remote S3 bucket to alleviate load on the GitLab server. -However, this behavior is undesirable for registries used by internal hosts that usually can't access public servers. To disable redirects, set the `disable` flag to true as follows. This makes all traffic to always go through the Registry service. This results in improved security (less surface attack as the storage backend is not publicly accessible), but worse performance (all traffic is redirected via the service). +However, this behavior is undesirable for registries used by internal hosts that usually can't access public servers. To disable redirects and [proxy download](../object_storage.md#proxy-download), set the `disable` flag to true as follows. This makes all traffic always go through the Registry service. This results in improved security (less surface attack as the storage backend is not publicly accessible), but worse performance (all traffic is redirected via the service). **Omnibus GitLab installations** @@ -779,13 +804,15 @@ that you have backed up all registry data. > [Introduced](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/764) in GitLab 8.8. -You can perform a garbage collection without stopping the Container Registry by setting -it into a read-only mode and by not using the built-in command. During this time, +You can perform garbage collection without stopping the Container Registry by putting +it in read-only mode and by not using the built-in command. On large instances +this could require Container Registry to be in read-only mode for a while. +During this time, you will be able to pull from the Container Registry, but you will not be able to push. NOTE: **Note:** -By default, the [registry storage path](#container-registry-storage-path) +By default, the [registry storage path](#configure-storage-for-the-container-registry) is `/var/opt/gitlab/gitlab-rails/shared/registry`. To enable the read-only mode: @@ -927,7 +954,7 @@ larger images, or images that take longer than 5 minutes to push, users may encounter this error. Administrators can increase the token duration in **Admin area > Settings > -Container Registry > Authorization token duration (minutes)**. +CI/CD > Container Registry > Authorization token duration (minutes)**. ### AWS S3 with the GitLab registry error when pushing large images diff --git a/doc/administration/packages/dependency_proxy.md b/doc/administration/packages/dependency_proxy.md index 565a4521c2a..2f9cfecc9d4 100644 --- a/doc/administration/packages/dependency_proxy.md +++ b/doc/administration/packages/dependency_proxy.md @@ -87,6 +87,11 @@ store the blobs of the dependency proxy. [Read more about using object storage with GitLab](../object_storage.md). +NOTE: **Note:** +In GitLab 13.2 and later, we recommend using the +[consolidated object storage settings](../object_storage.md#consolidated-object-storage-configuration). +This section describes the earlier configuration format. + **Omnibus GitLab installations** 1. Edit `/etc/gitlab/gitlab.rb` and add the following lines (uncomment where diff --git a/doc/administration/packages/index.md b/doc/administration/packages/index.md index 5088dd86a86..5d07136ef40 100644 --- a/doc/administration/packages/index.md +++ b/doc/administration/packages/index.md @@ -99,6 +99,9 @@ store packages. [Read more about using object storage with GitLab](../object_storage.md). +NOTE: **Note:** +We recommend using the [consolidated object storage settings](../object_storage.md#consolidated-object-storage-configuration). The following instructions apply to the original config format. + **Omnibus GitLab installations** 1. Edit `/etc/gitlab/gitlab.rb` and add the following lines (uncomment where diff --git a/doc/administration/pages/index.md b/doc/administration/pages/index.md index a7a3a86de8e..4efd92eaa07 100644 --- a/doc/administration/pages/index.md +++ b/doc/administration/pages/index.md @@ -190,7 +190,7 @@ control over how the Pages daemon runs and serves content in your environment. | Setting | Description | | ------- | ----------- | | `pages_external_url` | The URL where GitLab Pages is accessible, including protocol (HTTP / HTTPS). If `https://` is used, you must also set `gitlab_pages['ssl_certificate']` and `gitlab_pages['ssl_certificate_key']`. -| **gitlab_pages[]** | | +| `gitlab_pages[]` | | | `access_control` | Whether to enable [access control](index.md#access-control). | `api_secret_key` | Full path to file with secret key used to authenticate with the GitLab API. Auto-generated when left unset. | `artifacts_server` | Enable viewing [artifacts](../job_artifacts.md) in GitLab Pages. @@ -213,7 +213,7 @@ control over how the Pages daemon runs and serves content in your environment. | `internal_gitlab_server` | Internal GitLab server address used exclusively for API requests. Useful if you want to send that traffic over an internal load balancer. Defaults to GitLab `external_url`. | `listen_proxy` | The addresses to listen on for reverse-proxy requests. Pages will bind to these addresses' network socket and receives incoming requests from it. Sets the value of `proxy_pass` in `$nginx-dir/conf/gitlab-pages.conf`. | `log_directory` | Absolute path to a log directory. -| `log_format` | The log output format: 'text' or 'json'. +| `log_format` | The log output format: `text` or `json`. | `log_verbose` | Verbose logging, true/false. | `max_connections` | Limit on the number of concurrent connections to the HTTP, HTTPS or proxy listeners. | `metrics_address` | The address to listen on for metrics requests. @@ -225,14 +225,14 @@ control over how the Pages daemon runs and serves content in your environment. | `tls_max_version` | Specifies the maximum SSL/TLS version ("ssl3", "tls1.0", "tls1.1" or "tls1.2"). | `tls_min_version` | Specifies the minimum SSL/TLS version ("ssl3", "tls1.0", "tls1.1" or "tls1.2"). | `use_http2` | Enable HTTP2 support. -| **gitlab_pages['env'][]** | | +| `gitlab_pages['env'][]` | | | `http_proxy` | Configure GitLab Pages to use an HTTP Proxy to mediate traffic between Pages and GitLab. Sets an environment variable `http_proxy` when starting Pages daemon. -| **gitlab_rails[]** | | +| `gitlab_rails[]` | | | `pages_domain_verification_cron_worker` | Schedule for verifying custom GitLab Pages domains. | `pages_domain_ssl_renewal_cron_worker` | Schedule for obtaining and renewing SSL certificates through Let's Encrypt for GitLab Pages domains. | `pages_domain_removal_cron_worker` | Schedule for removing unverified custom GitLab Pages domains. | `pages_path` | The directory on disk where pages are stored, defaults to `GITLAB-RAILS/shared/pages`. -| **pages_nginx[]** | | +| `pages_nginx[]` | | | `enable` | Include a virtual host `server{}` block for Pages inside NGINX. Needed for NGINX to proxy traffic back to the Pages daemon. Set to `false` if the Pages daemon should directly receive all requests, for example, when using [custom domains](index.md#custom-domains). --- @@ -408,6 +408,9 @@ pages: ### Using a custom Certificate Authority (CA) +NOTE: **Note:** +[Before 13.2](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/4289), when using Omnibus, a [workaround was required](https://docs.gitlab.com/13.1/ee/administration/pages/index.html#using-a-custom-certificate-authority-ca). + When using certificates issued by a custom CA, [Access Control](../../user/project/pages/pages_access_control.md#gitlab-pages-access-control) and the [online view of HTML job artifacts](../../ci/pipelines/job_artifacts.md#browsing-artifacts) will fail to work if the custom CA is not recognized. @@ -415,28 +418,10 @@ will fail to work if the custom CA is not recognized. This usually results in this error: `Post /oauth/token: x509: certificate signed by unknown authority`. -For installation from source this can be fixed by installing the custom Certificate +For installation from source, this can be fixed by installing the custom Certificate Authority (CA) in the system certificate store. -For Omnibus, normally this would be fixed by [installing a custom CA in Omnibus GitLab](https://docs.gitlab.com/omnibus/settings/ssl.html#install-custom-public-certificates) -but a [bug](https://gitlab.com/gitlab-org/gitlab/-/issues/25411) is currently preventing -that method from working. Use the following workaround: - -1. Append your GitLab server TLS/SSL certificate to `/opt/gitlab/embedded/ssl/certs/cacert.pem` where `gitlab-domain-example.com` is your GitLab application URL - - ```shell - printf "\ngitlab-domain-example.com\n===========================\n" | sudo tee --append /opt/gitlab/embedded/ssl/certs/cacert.pem - echo -n | openssl s_client -connect gitlab-domain-example.com:443 | sed -ne '/-BEGIN CERTIFICATE-/,/-END CERTIFICATE-/p' | sudo tee --append /opt/gitlab/embedded/ssl/certs/cacert.pem - ``` - -1. [Restart](../restart_gitlab.md) the GitLab Pages Daemon. For Omnibus GitLab instances: - - ```shell - sudo gitlab-ctl restart gitlab-pages - ``` - -CAUTION: **Caution:** -Some Omnibus GitLab upgrades will revert this workaround and you'll need to apply it again. +For Omnibus, this is fixed by [installing a custom CA in Omnibus GitLab](https://docs.gitlab.com/omnibus/settings/ssl.html#install-custom-public-certificates). ## Activate verbose logging for daemon @@ -526,10 +511,17 @@ The following procedure includes steps to back up and edit the `gitlab-secrets.json` file. This file contains secrets that control database encryption. Proceed with caution. +1. Create a backup of the secrets file on the **GitLab server**: + + ```shell + cp /etc/gitlab/gitlab-secrets.json /etc/gitlab/gitlab-secrets.json.bak + ``` + 1. On the **GitLab server**, to enable Pages, add the following to `/etc/gitlab/gitlab.rb`: ```ruby gitlab_pages['enable'] = true + pages_external_url "http://<pages_server_URL>" ``` 1. Optionally, to enable [access control](#access-control), add the following to `/etc/gitlab/gitlab.rb`: @@ -542,26 +534,25 @@ database encryption. Proceed with caution. changes to take effect. The `gitlab-secrets.json` file is now updated with the new configuration. -1. Create a backup of the secrets file on the **GitLab server**: - - ```shell - cp /etc/gitlab/gitlab-secrets.json /etc/gitlab/gitlab-secrets.json.bak - ``` - 1. Set up a new server. This will become the **Pages server**. -1. Create an [NFS share](../high_availability/nfs_host_client_setup.md) on the new server and configure this share to - allow access from your main **GitLab server**. For this example, we use the +1. Create an [NFS share](../high_availability/nfs_host_client_setup.md) + on the **Pages server** and configure this share to + allow access from your main **GitLab server**. + Note that the example there is more general and + shares several sub-directories from `/home` to several `/nfs/home` mountpoints. + For our Pages-specific example here, we instead share only the default GitLab Pages folder `/var/opt/gitlab/gitlab-rails/shared/pages` - as the shared folder on the new server and we will mount it to `/mnt/pages` + from the **Pages server** and we mount it to `/mnt/pages` on the **GitLab server**. + Therefore, omit "Step 4" there. 1. On the **Pages server**, install Omnibus GitLab and modify `/etc/gitlab/gitlab.rb` to include: ```ruby - external_url 'http://<ip-address-of-the-server>' - pages_external_url "http://<your-pages-server-URL>" + external_url 'http://<gitlab_server_IP_or_URL>' + pages_external_url "http://<pages_server_URL>" postgresql['enable'] = false redis['enable'] = false prometheus['enable'] = false @@ -581,7 +572,15 @@ database encryption. Proceed with caution. ``` 1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the **GitLab server** - to the **Pages server**. + to the **Pages server**, for example via the NFS share. + + ```shell + # On the GitLab server + cp /etc/gitlab/gitlab-secrets.json /mnt/pages/gitlab-secrets.json + + # On the Pages server + mv /var/opt/gitlab/gitlab-rails/shared/pages/gitlab-secrets.json /etc/gitlab/gitlab-secrets.json + ``` 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. @@ -589,13 +588,13 @@ database encryption. Proceed with caution. ```ruby gitlab_pages['enable'] = false - pages_external_url "http://<your-pages-server-URL>" + pages_external_url "http://<pages_server_URL>" gitlab_rails['pages_path'] = "/mnt/pages" ``` 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. -It is possible to run GitLab Pages on multiple servers if you wish to distribute +It's possible to run GitLab Pages on multiple servers if you wish to distribute the load. You can do this through standard load balancing practices such as configuring your DNS server to return multiple IPs for your Pages server, configuring a load balancer to work at the IP level, and so on. If you wish to @@ -655,3 +654,45 @@ The fix is to correct the source file permissions and restart Pages: sudo chmod 644 /opt/gitlab/embedded/ssl/certs/cacert.pem sudo gitlab-ctl restart gitlab-pages ``` + +### `dial tcp: lookup gitlab.example.com` and `x509: certificate signed by unknown authority` + +When setting both `inplace_chroot` and `access_control` to `true`, you might encounter errors like: + +```plaintext +dial tcp: lookup gitlab.example.com on [::1]:53: dial udp [::1]:53: connect: cannot assign requested address +``` + +Or: + +```plaintext +open /opt/gitlab/embedded/ssl/certs/cacert.pem: no such file or directory +x509: certificate signed by unknown authority +``` + +The reason for those errors is that the files `resolv.conf` and `ca-bundle.pem` are missing inside the chroot. +The fix is to copy the host's `/etc/resolv.conf` and GitLab's certificate bundle inside the chroot: + +```shell +sudo mkdir -p /var/opt/gitlab/gitlab-rails/shared/pages/etc/ssl +sudo mkdir -p /var/opt/gitlab/gitlab-rails/shared/pages/opt/gitlab/embedded/ssl/certs/ + +sudo cp /etc/resolv.conf /var/opt/gitlab/gitlab-rails/shared/pages/etc +sudo cp /opt/gitlab/embedded/ssl/certs/cacert.pem /var/opt/gitlab/gitlab-rails/shared/pages/opt/gitlab/embedded/ssl/certs/ +sudo cp /opt/gitlab/embedded/ssl/certs/cacert.pem /var/opt/gitlab/gitlab-rails/shared/pages/etc/ssl/ca-bundle.pem +``` + +### 404 error after transferring project to a different group or user + +If you encounter a `404 Not Found` error a Pages site after transferring a project to +another group or user, you must trigger adomain configuration update for Pages. To do +so, write something in the `.update` file. The Pages daemon monitors for changes to this +file, and reloads the configuration when changes occur. + +Use this example to fix a `404 Not Found` error after transferring a project with Pages: + +```shell +date > /var/opt/gitlab/gitlab-rails/shared/pages/.update +``` + +If you've customized the Pages storage path, adjust the command above to use your custom path. diff --git a/doc/administration/postgresql/index.md b/doc/administration/postgresql/index.md new file mode 100644 index 00000000000..7e0a2f3cae1 --- /dev/null +++ b/doc/administration/postgresql/index.md @@ -0,0 +1,36 @@ +--- +type: reference +--- + +# Configuring PostgreSQL for scaling + +In this section, you'll be guided through configuring a PostgreSQL database to +be used with GitLab in one of our [Scalable and Highly Available Setups](../reference_architectures/index.md). +There are essentially three setups to choose from. + +## PostgreSQL replication and failover with Omnibus GitLab **(PREMIUM ONLY)** + +This setup is for when you have installed GitLab using the +[Omnibus GitLab **Enterprise Edition** (EE) package](https://about.gitlab.com/install/?version=ee). + +All the tools that are needed like PostgreSQL, PgBouncer, Repmgr are bundled in +the package, so you can it to set up the whole PostgreSQL infrastructure (primary, replica). + +[> Read how to set up PostgreSQL replication and failover using Omnibus GitLab](replication_and_failover.md) + +## Standalone PostgreSQL using Omnibus GitLab **(CORE ONLY)** + +This setup is for when you have installed the +[Omnibus GitLab packages](https://about.gitlab.com/install/) (CE or EE), +to use the bundled PostgreSQL having only its service enabled. + +[> Read how to set up a standalone PostgreSQL instance using Omnibus GitLab](standalone.md) + +## Provide your own PostgreSQL instance **(CORE ONLY)** + +This setup is for when you have installed GitLab using the +[Omnibus GitLab packages](https://about.gitlab.com/install/) (CE or EE), +or installed it [from source](../../install/installation.md), but you want to use +your own external PostgreSQL server. + +[> Read how to set up an external PostgreSQL instance](external.md) diff --git a/doc/administration/postgresql/replication_and_failover.md b/doc/administration/postgresql/replication_and_failover.md index aa95b983d20..5f550f09e5b 100644 --- a/doc/administration/postgresql/replication_and_failover.md +++ b/doc/administration/postgresql/replication_and_failover.md @@ -1,16 +1,15 @@ # PostgreSQL replication and failover with Omnibus GitLab **(PREMIUM ONLY)** -> Important notes: -> -> - This document will focus only on configuration supported with [GitLab Premium](https://about.gitlab.com/pricing/), using the Omnibus GitLab package. -> - If you are a Community Edition or Starter user, consider using a cloud hosted solution. -> - This document will not cover installations from source. -> -> - If a setup with replication and failover is not what you were looking for, see the [database configuration document](https://docs.gitlab.com/omnibus/settings/database.html) -> for the Omnibus GitLab packages. -> -> Please read this document fully before attempting to configure PostgreSQL with -> replication and failover for GitLab. +This document will focus only on configuration supported with [GitLab Premium](https://about.gitlab.com/pricing/), using the Omnibus GitLab package. +If you are a Community Edition or Starter user, consider using a cloud hosted solution. +This document will not cover installations from source. + +If a setup with replication and failover is not what you were looking for, see +the [database configuration document](https://docs.gitlab.com/omnibus/settings/database.html) +for the Omnibus GitLab packages. + +It's recommended to read this document fully before attempting to configure PostgreSQL with +replication and failover for GitLab. ## Architecture @@ -967,7 +966,8 @@ after it has been restored to service. gitlab-ctl restart repmgrd ``` - CAUTION: **Warning:** When the server is brought back online, and before + CAUTION: **Warning:** + When the server is brought back online, and before you switch it to a standby node, repmgr will report that there are two masters. If there are any clients that are still attempting to write to the old master, this will cause a split, and the old master will need to be resynced from @@ -1127,3 +1127,213 @@ If you're running into an issue with a component not outlined here, be sure to c - [Consul](../high_availability/consul.md#troubleshooting) - [PostgreSQL](https://docs.gitlab.com/omnibus/settings/database.html#troubleshooting) - [GitLab application](../high_availability/gitlab.md#troubleshooting) + +## Patroni + +NOTE: **Note:** +Starting from GitLab 13.1, Patroni is available for **experimental** use to replace repmgr. Due to its +experimental nature, Patroni support is **subject to change without notice.** + +Patroni is an opinionated solution for PostgreSQL high-availability. It takes the control of PostgreSQL, overrides its +configuration and manages its lifecycle (start, stop, restart). This is a more active approach when compared to repmgr. +Both repmgr and Patroni are both supported and available. But Patroni will be the default (and perhaps the only) option +for PostgreSQL 12 clustering and cascading replication for Geo deployments. + +The [architecture](#example-recommended-setup-manual-steps) (that was mentioned above) does not change for Patroni. +You do not need any special consideration for Patroni while provisioning your database nodes. Patroni heavily relies on +Consul to store the state of the cluster and elect a leader. Any failure in Consul cluster and its leader election will +propagate to Patroni cluster as well. + +Similar to repmgr, Patroni monitors the cluster and handles failover. When the primary node fails it works with Consul +to notify PgBouncer. However, as opposed to repmgr, on failure, Patroni handles the transitioning of the old primary to +a replica and rejoins it to the cluster automatically. So you do not need any manual operation for recovering the +cluster as you do with repmgr. + +With Patroni the connection flow is slightly different. Patroni on each node connects to Consul agent to join the +cluster. Only after this point it decides if the node is the primary or a replica. Based on this decision, it configures +and starts PostgreSQL which it communicates with directly over a Unix socket. This implies that if Consul cluster is not +functional or does not have a leader, Patroni and by extension PostgreSQL will not start. Patroni also exposes a REST +API which can be accessed via its [default port](https://docs.gitlab.com/omnibus/package-information/defaults.html#patroni) +on each node. + +### Configuring Patroni cluster + +You must enable Patroni explicitly to be able to use it (with `patroni['enable'] = true`). When Patroni is enabled +repmgr will be disabled automatically. + +Any PostgreSQL configuration item that controls replication, for example `wal_level`, `max_wal_senders`, etc, are strictly +controlled by Patroni and will override the original settings that you make with the `postgresql[...]` configuration key. +Hence, they are all separated and placed under `patroni['postgresql'][...]`. This behavior is limited to replication. +Patroni honours any other PostgreSQL configuration that was made with the `postgresql[...]` configuration key. For example, +`max_wal_senders` by default is set to `5`. If you wish to change this you must set it with the `patroni['postgresql']['max_wal_senders']` +configuration key. + +The configuration of Patroni node is very similar to a repmgr but shorter. When Patroni is enabled, first you can ignore +any replication setting of PostgreSQL (it will be overwritten anyway). Then you can remove any `repmgr[...]` or +repmgr-specific configuration as well. Especially, make sure that you remove `postgresql['shared_preload_libraries'] = 'repmgr_funcs'`. + +Here is an example similar to [the one that was done with repmgr](#configuring-the-database-nodes): + +```ruby +# Disable all components except PostgreSQL and Repmgr and Consul +roles['postgres_role'] + +# Enable Patroni +patroni['enable'] = true + +# PostgreSQL configuration +postgresql['listen_address'] = '0.0.0.0' + +# Disable automatic database migrations +gitlab_rails['auto_migrate'] = false + +# Configure the Consul agent +consul['services'] = %w(postgresql) + +# START user configuration +# Please set the real values as explained in Required Information section +# +# Replace PGBOUNCER_PASSWORD_HASH with a generated md5 value +postgresql['pgbouncer_user_password'] = 'PGBOUNCER_PASSWORD_HASH' +# Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value +postgresql['sql_user_password'] = 'POSTGRESQL_PASSWORD_HASH' + +# Replace X with value of number of db nodes + 1 (OPTIONAL the default value is 5) +patroni['postgresql']['max_wal_senders'] = X +patroni['postgresql']['max_replication_slots'] = X + +# Replace XXX.XXX.XXX.XXX/YY with Network Address +postgresql['trust_auth_cidr_addresses'] = %w(XXX.XXX.XXX.XXX/YY) + +# Replace placeholders: +# +# Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z +# with the addresses gathered for CONSUL_SERVER_NODES +consul['configuration'] = { + retry_join: %w(Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z) +} +# +# END user configuration +``` + +You do not need an additional or different configuration for replica nodes. As a matter of fact, you don't have to have +a predetermined primary node. Therefore all database nodes use the same configuration. + +Once the configuration of a node is done, you must [reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) +on each node for the changes to take effect. + +Generally, when Consul cluster is ready, the first node that [reconfigures](../restart_gitlab.md#omnibus-gitlab-reconfigure) +becomes the leader. You do not need to sequence the nodes reconfiguration. You can run them in parallel or in any order. +If you choose an arbitrary order you do not have any predetermined master. + +As opposed to repmgr, once the nodes are reconfigured you do not need any further action or additional command to join +the replicas. + +#### Database authorization for Patroni + +Patroni uses Unix socket to manage PostgreSQL instance. Therefore, the connection from the `local` socket must be trusted. + +Also, replicas use the replication user (`gitlab_replicator` by default) to communicate with the leader. For this user, +you can choose between `trust` and `md5` authentication. If you set `postgresql['sql_replication_password']`, +Patroni will use `md5` authentication, otherwise it falls back to `trust`. You must to specify the cluster CIDR in +`postgresql['md5_auth_cidr_addresses']` or `postgresql['trust_auth_cidr_addresses']` respectively. + +### Interacting with Patroni cluster + +You can use `gitlab-ctl patroni members` to check the status of the cluster members. To check the status of each node +`gitlab-ctl patroni` provides two additional sub-commands, `check-leader` and `check-replica` which indicate if a node +is the primary or a replica. + +When Patroni is enabled, you don't have direct control over `postgresql` service. Patroni will signal PostgreSQL's startup, +shutdown, and restart. For example, for shutting down PostgreSQL on a node, you must shutdown Patroni on the same node +with: + +```shell +sudo gitlab-ctl stop patroni +``` + +### Manual failover procedure for Patroni + +While Patroni supports automatic failover, you also have the ability to perform +a manual one, where you have two slightly different options: + +- **Failover**: allows you to perform a manual failover when there are no healthy nodes. + You can perform this action in any PostgreSQL node: + + ```shell + sudo gitlab-ctl patroni failover + ``` + +- **Switchover**: only works when the cluster is healthy and allows you to schedule a switchover (it can happen immediately). + You can perform this action in any PostgreSQL node: + + ```shell + sudo gitlab-ctl patroni switchover + ``` + +For further details on this subject, see the +[Patroni documentation](https://patroni.readthedocs.io/en/latest/rest_api.html#switchover-and-failover-endpoints). + +### Recovering the Patroni cluster + +To recover the old primary and rejoin it to the cluster as a replica, you can simply start Patroni with: + +```shell +sudo gitlab-ctl start patroni +``` + +No further configuration or intervention is needed. + +### Maintenance procedure for Patroni + +With Patroni enabled, you can run a planned maintenance. If you want to do some maintenance work on one node and you +don't want Patroni to manage it, you can use put it into maintenance mode: + +```shell +sudo gitlab-ctl patroni pause +``` + +When Patroni runs in a paused mode, it does not change the state of PostgreSQL. Once you are done you can resume Patroni: + +```shell +sudo gitlab-ctl patroni resume +``` + +For further details, see [Patroni documentation on this subject](https://patroni.readthedocs.io/en/latest/pause.html). + +### Switching from repmgr to Patroni + +CAUTION: **Warning:** +Although switching from repmgr to Patroni is fairly straightforward the other way around is not. Rolling back from +Patroni to repmgr can be complicated and may involve deletion of data directory. If you need to do that, please contact +GitLab support. + +You can switch an exiting database cluster to use Patroni instead of repmgr with the following steps: + +1. Stop repmgr on all replica nodes and lastly with the primary node: + + ```shell + sudo gitlab-ctl stop repmgrd + ``` + +1. Stop PostgreSQL on all replica nodes: + + ```shell + sudo gitlab-ctl stop postgresql + ``` + + NOTE: **Note:** + Ensure that there is no `walsender` process running on the primary node. + `ps aux | grep walsender` must not show any running process. + +1. On the primary node, [configure Patroni](#configuring-patroni-cluster). Remove `repmgr` and any other + repmgr-specific configuration. Also remove any configuration that is related to PostgreSQL replication. +1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) on the primary node. It will become + the leader. You can check this with: + + ```shell + sudo gitlab-ctl tail patroni + ``` + +1. Repeat the last two steps for all replica nodes. `gitlab.rb` should look the same on all nodes. +1. Optional: You can remove `gitlab_repmgr` database and role on the primary. diff --git a/doc/administration/postgresql/standalone.md b/doc/administration/postgresql/standalone.md index 3e7826ce009..2747749066e 100644 --- a/doc/administration/postgresql/standalone.md +++ b/doc/administration/postgresql/standalone.md @@ -53,7 +53,8 @@ together with Omnibus GitLab. This is recommended as part of our gitlab_rails['auto_migrate'] = false ``` - NOTE: **Note:** The role `postgres_role` was introduced with GitLab 10.3 + NOTE: **Note:** + The role `postgres_role` was introduced with GitLab 10.3 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. 1. Note the PostgreSQL node's IP address or hostname, port, and diff --git a/doc/administration/raketasks/ldap.md b/doc/administration/raketasks/ldap.md index d168e3d568c..30bb9828aa0 100644 --- a/doc/administration/raketasks/ldap.md +++ b/doc/administration/raketasks/ldap.md @@ -36,7 +36,7 @@ The following task will run a [group sync](../auth/ldap/index.md#group-sync-star when you'd like to update all configured group memberships against LDAP without waiting for the next scheduled group sync to be run. -NOTE: **NOTE:** +NOTE: **Note:** If you'd like to change the frequency at which a group sync is performed, [adjust the cron schedule](../auth/ldap/index.md#adjusting-ldap-group-sync-schedule-starter-only) instead. diff --git a/doc/administration/raketasks/maintenance.md b/doc/administration/raketasks/maintenance.md index 78094f00a43..19781b6a5db 100644 --- a/doc/administration/raketasks/maintenance.md +++ b/doc/administration/raketasks/maintenance.md @@ -260,7 +260,7 @@ clear it. To clear all exclusive leases: -DANGER: **DANGER**: +DANGER: **Danger:** Don't run it while GitLab or Sidekiq is running ```shell @@ -317,7 +317,7 @@ migrations are completed (have an `up` status). Sometimes you may need to re-import the common metrics that power the Metrics dashboards. -This could be as a result of [updating existing metrics](../../development/prometheus_metrics.md#update-existing-metrics), or as a [troubleshooting measure](../../user/project/integrations/prometheus.md#troubleshooting). +This could be as a result of [updating existing metrics](../../development/prometheus_metrics.md#update-existing-metrics), or as a [troubleshooting measure](../../operations/metrics/dashboards/index.md#troubleshooting). To re-import the metrics you can run: diff --git a/doc/administration/raketasks/praefect.md b/doc/administration/raketasks/praefect.md index c48e23df77a..ca9c2065904 100644 --- a/doc/administration/raketasks/praefect.md +++ b/doc/administration/raketasks/praefect.md @@ -1,13 +1,23 @@ -# Praefect Rake Tasks **(CORE ONLY)** +--- +stage: Create +group: Gitaly +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: reference +--- + +# Praefect Rake tasks **(CORE ONLY)** > [Introduced]( https://gitlab.com/gitlab-org/gitlab/-/merge_requests/28369) in GitLab 12.10. +Rake tasks are available for projects that have been created on Praefect storage. See the +[Praefect documentation](../gitaly/praefect.md) for information on configuring Praefect. + ## Replica checksums -Prints out checksums of the repository of a given project_id on the primary as well as secondary internal Gitaly nodes. +`gitlab:praefect:replicas` prints out checksums of the repository of a given `project_id` on: -NOTE: **Note:** -This only is relevant and works for projects that have been created on a Praefect storage. See the [Praefect Documentation](../gitaly/praefect.md) for configuring Praefect. +- The primary Gitaly node. +- Secondary internal Gitaly nodes. **Omnibus Installation** diff --git a/doc/administration/raketasks/project_import_export.md b/doc/administration/raketasks/project_import_export.md index 2e65e889c90..b807e03b01f 100644 --- a/doc/administration/raketasks/project_import_export.md +++ b/doc/administration/raketasks/project_import_export.md @@ -7,6 +7,16 @@ GitLab provides Rake tasks relating to project import and export. For more infor - [Project import/export documentation](../../user/project/settings/import_export.md). - [Project import/export API](../../api/project_import_export.md). +- [Developer documentation: project import/export](../../development/import_export.md) + +## Project import status + +You can query an import through the [Project import/export API](../../api/project_import_export.md#import-status). +As described in the API documentation, the query may return an import error or exceptions. + +## Import large projects + +If you have a larger project, consider using a Rake task, as described in our [developer documentation](../../development/import_project.md#importing-via-a-rake-task). ## Import/export tasks diff --git a/doc/administration/redis/index.md b/doc/administration/redis/index.md new file mode 100644 index 00000000000..0bd56666ab8 --- /dev/null +++ b/doc/administration/redis/index.md @@ -0,0 +1,42 @@ +--- +type: index +stage: Enablement +group: Distribution +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Configuring Redis for scaling + +Based on your infrastructure setup and how you have installed GitLab, there are +multiple ways to configure Redis. + +You can choose to install and manage Redis and Sentinel yourself, use a hosted +cloud solution, or you can use the ones that come bundled with the Omnibus GitLab +packages so you only need to focus on configuration. Pick the one that suits your needs. + +## Redis replication and failover using Omnibus GitLab + +This setup is for when you have installed GitLab using the +[Omnibus GitLab **Enterprise Edition** (EE) package](https://about.gitlab.com/install/?version=ee). + +Both Redis and Sentinel are bundled in the package, so you can it to set up the whole +Redis infrastructure (primary, replica and sentinel). + +[> Read how to set up Redis replication and failover using Omnibus GitLab](replication_and_failover.md) + +## Redis replication and failover using the non-bundled Redis + +This setup is for when you have installed GitLab using the +[Omnibus GitLab packages](https://about.gitlab.com/install/) (CE or EE), +or installed it [from source](../../install/installation.md), but you want to use +your own external Redis and sentinel servers. + +[> Read how to set up Redis replication and failover using the non-bundled Redis](replication_and_failover_external.md) + +## Standalone Redis using Omnibus GitLab + +This setup is for when you have installed the +[Omnibus GitLab **Community Edition** (CE) package](https://about.gitlab.com/install/?version=ce) +to use the bundled Redis, so you can use the package with only the Redis service enabled. + +[> Read how to set up a standalone Redis instance using Omnibus GitLab](standalone.md) diff --git a/doc/administration/redis/replication_and_failover.md b/doc/administration/redis/replication_and_failover.md new file mode 100644 index 00000000000..ac31b909c89 --- /dev/null +++ b/doc/administration/redis/replication_and_failover.md @@ -0,0 +1,741 @@ +--- +type: howto +stage: Enablement +group: Distribution +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Redis replication and failover with Omnibus GitLab **(PREMIUM ONLY)** + +NOTE: **Note:** +This is the documentation for the Omnibus GitLab packages. For using your own +non-bundled Redis, follow the [relevant documentation](replication_and_failover_external.md). + +NOTE: **Note:** +In Redis lingo, primary is called master. In this document, primary is used +instead of master, except the settings where `master` is required. + +Using [Redis](https://redis.io/) in scalable environment is possible using a **Primary** x **Replica** +topology with a [Redis Sentinel](https://redis.io/topics/sentinel) service to watch and automatically +start the failover procedure. + +Redis requires authentication if used with Sentinel. See +[Redis Security](https://redis.io/topics/security) documentation for more +information. We recommend using a combination of a Redis password and tight +firewall rules to secure your Redis service. +You are highly encouraged to read the [Redis Sentinel](https://redis.io/topics/sentinel) documentation +before configuring Redis with GitLab to fully understand the topology and +architecture. + +Before diving into the details of setting up Redis and Redis Sentinel for a +replicated topology, make sure you read this document once as a whole to better +understand how the components are tied together. + +You need at least `3` independent machines: physical, or VMs running into +distinct physical machines. It is essential that all primary and replica Redis +instances run in different machines. If you fail to provision the machines in +that specific way, any issue with the shared environment can bring your entire +setup down. + +It is OK to run a Sentinel alongside of a primary or replica Redis instance. +There should be no more than one Sentinel on the same machine though. + +You also need to take into consideration the underlying network topology, +making sure you have redundant connectivity between Redis / Sentinel and +GitLab instances, otherwise the networks will become a single point of +failure. + +Running Redis in a scaled environment requires a few things: + +- Multiple Redis instances +- Run Redis in a **Primary** x **Replica** topology +- Multiple Sentinel instances +- Application support and visibility to all Sentinel and Redis instances + +Redis Sentinel can handle the most important tasks in an HA environment and that's +to help keep servers online with minimal to no downtime. Redis Sentinel: + +- Monitors **Primary** and **Replicas** instances to see if they are available +- Promotes a **Replica** to **Primary** when the **Primary** fails +- Demotes a **Primary** to **Replica** when the failed **Primary** comes back online + (to prevent data-partitioning) +- Can be queried by the application to always connect to the current **Primary** + server + +When a **Primary** fails to respond, it's the application's responsibility +(in our case GitLab) to handle timeout and reconnect (querying a **Sentinel** +for a new **Primary**). + +To get a better understanding on how to correctly set up Sentinel, please read +the [Redis Sentinel documentation](https://redis.io/topics/sentinel) first, as +failing to configure it correctly can lead to data loss or can bring your +whole cluster down, invalidating the failover effort. + +## Recommended setup + +For a minimal setup, you will install the Omnibus GitLab package in `3` +**independent** machines, both with **Redis** and **Sentinel**: + +- Redis Primary + Sentinel +- Redis Replica + Sentinel +- Redis Replica + Sentinel + +If you are not sure or don't understand why and where the amount of nodes come +from, read [Redis setup overview](#redis-setup-overview) and +[Sentinel setup overview](#sentinel-setup-overview). + +For a recommended setup that can resist more failures, you will install +the Omnibus GitLab package in `5` **independent** machines, both with +**Redis** and **Sentinel**: + +- Redis Primary + Sentinel +- Redis Replica + Sentinel +- Redis Replica + Sentinel +- Redis Replica + Sentinel +- Redis Replica + Sentinel + +### Redis setup overview + +You must have at least `3` Redis servers: `1` primary, `2` Replicas, and they +need to each be on independent machines (see explanation above). + +You can have additional Redis nodes, that will help survive a situation +where more nodes goes down. Whenever there is only `2` nodes online, a failover +will not be initiated. + +As an example, if you have `6` Redis nodes, a maximum of `3` can be +simultaneously down. + +Please note that there are different requirements for Sentinel nodes. +If you host them in the same Redis machines, you may need to take +that restrictions into consideration when calculating the amount of +nodes to be provisioned. See [Sentinel setup overview](#sentinel-setup-overview) +documentation for more information. + +All Redis nodes should be configured the same way and with similar server specs, as +in a failover situation, any **Replica** can be promoted as the new **Primary** by +the Sentinel servers. + +The replication requires authentication, so you need to define a password to +protect all Redis nodes and the Sentinels. They will all share the same +password, and all instances must be able to talk to +each other over the network. + +### Sentinel setup overview + +Sentinels watch both other Sentinels and Redis nodes. Whenever a Sentinel +detects that a Redis node is not responding, it will announce that to the +other Sentinels. They have to reach the **quorum**, that is the minimum amount +of Sentinels that agrees a node is down, in order to be able to start a failover. + +Whenever the **quorum** is met, the **majority** of all known Sentinel nodes +need to be available and reachable, so that they can elect the Sentinel **leader** +who will take all the decisions to restore the service availability by: + +- Promoting a new **Primary** +- Reconfiguring the other **Replicas** and make them point to the new **Primary** +- Announce the new **Primary** to every other Sentinel peer +- Reconfigure the old **Primary** and demote to **Replica** when it comes back online + +You must have at least `3` Redis Sentinel servers, and they need to +be each in an independent machine (that are believed to fail independently), +ideally in different geographical areas. + +You can configure them in the same machines where you've configured the other +Redis servers, but understand that if a whole node goes down, you loose both +a Sentinel and a Redis instance. + +The number of sentinels should ideally always be an **odd** number, for the +consensus algorithm to be effective in the case of a failure. + +In a `3` nodes topology, you can only afford `1` Sentinel node going down. +Whenever the **majority** of the Sentinels goes down, the network partition +protection prevents destructive actions and a failover **will not be started**. + +Here are some examples: + +- With `5` or `6` sentinels, a maximum of `2` can go down for a failover begin. +- With `7` sentinels, a maximum of `3` nodes can go down. + +The **Leader** election can sometimes fail the voting round when **consensus** +is not achieved (see the odd number of nodes requirement above). In that case, +a new attempt will be made after the amount of time defined in +`sentinel['failover_timeout']` (in milliseconds). + +NOTE: **Note:** +We will see where `sentinel['failover_timeout']` is defined later. + +The `failover_timeout` variable has a lot of different use cases. According to +the official documentation: + +- The time needed to re-start a failover after a previous failover was + already tried against the same primary by a given Sentinel, is two + times the failover timeout. + +- The time needed for a replica replicating to a wrong primary according + to a Sentinel current configuration, to be forced to replicate + with the right primary, is exactly the failover timeout (counting since + the moment a Sentinel detected the misconfiguration). + +- The time needed to cancel a failover that is already in progress but + did not produced any configuration change (REPLICAOF NO ONE yet not + acknowledged by the promoted replica). + +- The maximum time a failover in progress waits for all the replicas to be + reconfigured as replicas of the new primary. However even after this time + the replicas will be reconfigured by the Sentinels anyway, but not with + the exact parallel-syncs progression as specified. + +## Configuring Redis + +This is the section where we install and set up the new Redis instances. + +It is assumed that you have installed GitLab and all its components from scratch. +If you already have Redis installed and running, read how to +[switch from a single-machine installation](#switching-from-an-existing-single-machine-installation). + +NOTE: **Note:** +Redis nodes (both primary and replica) will need the same password defined in +`redis['password']`. At any time during a failover the Sentinels can +reconfigure a node and change its status from primary to replica and vice versa. + +### Requirements + +The requirements for a Redis setup are the following: + +1. Provision the minimum required number of instances as specified in the + [recommended setup](#recommended-setup) section. +1. We **Do not** recommend installing Redis or Redis Sentinel in the same machines your + GitLab application is running on as this weakens your HA configuration. You can however opt in to install Redis + and Sentinel in the same machine. +1. All Redis nodes must be able to talk to each other and accept incoming + connections over Redis (`6379`) and Sentinel (`26379`) ports (unless you + change the default ones). +1. The server that hosts the GitLab application must be able to access the + Redis nodes. +1. Protect the nodes from access from external networks ([Internet](https://gitlab.com/gitlab-org/gitlab-foss/uploads/c4cc8cd353604bd80315f9384035ff9e/The_Internet_IT_Crowd.png)), using + firewall. + +### Switching from an existing single-machine installation + +If you already have a single-machine GitLab install running, you will need to +replicate from this machine first, before de-activating the Redis instance +inside it. + +Your single-machine install will be the initial **Primary**, and the `3` others +should be configured as **Replica** pointing to this machine. + +After replication catches up, you will need to stop services in the +single-machine install, to rotate the **Primary** to one of the new nodes. + +Make the required changes in configuration and restart the new nodes again. + +To disable Redis in the single install, edit `/etc/gitlab/gitlab.rb`: + +```ruby +redis['enable'] = false +``` + +If you fail to replicate first, you may loose data (unprocessed background jobs). + +### Step 1. Configuring the primary Redis instance + +1. SSH into the **Primary** Redis server. +1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab + package you want using **steps 1 and 2** from the GitLab downloads page. + - Make sure you select the correct Omnibus package, with the same version + and type (Community, Enterprise editions) of your current install. + - Do not complete any other steps on the download page. + +1. Edit `/etc/gitlab/gitlab.rb` and add the contents: + + ```ruby + # Specify server role as 'redis_master_role' + roles ['redis_master_role'] + + # IP address pointing to a local IP that the other machines can reach to. + # You can also set bind to '0.0.0.0' which listen in all interfaces. + # If you really need to bind to an external accessible IP, make + # sure you add extra firewall rules to prevent unauthorized access. + redis['bind'] = '10.0.0.1' + + # Define a port so Redis can listen for TCP requests which will allow other + # machines to connect to it. + redis['port'] = 6379 + + # Set up password authentication for Redis (use the same password in all nodes). + redis['password'] = 'redis-password-goes-here' + ``` + +1. Only the primary GitLab application server should handle migrations. To + prevent database migrations from running on upgrade, add the following + configuration to your `/etc/gitlab/gitlab.rb` file: + + ```ruby + gitlab_rails['auto_migrate'] = false + ``` + +1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + +NOTE: **Note:** +You can specify multiple roles like sentinel and Redis as: +`roles ['redis_sentinel_role', 'redis_master_role']`. +Read more about [roles](https://docs.gitlab.com/omnibus/roles/). + +### Step 2. Configuring the replica Redis instances + +1. SSH into the **replica** Redis server. +1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab + package you want using **steps 1 and 2** from the GitLab downloads page. + - Make sure you select the correct Omnibus package, with the same version + and type (Community, Enterprise editions) of your current install. + - Do not complete any other steps on the download page. + +1. Edit `/etc/gitlab/gitlab.rb` and add the contents: + + ```ruby + # Specify server role as 'redis_replica_role' + roles ['redis_replica_role'] + + # IP address pointing to a local IP that the other machines can reach to. + # You can also set bind to '0.0.0.0' which listen in all interfaces. + # If you really need to bind to an external accessible IP, make + # sure you add extra firewall rules to prevent unauthorized access. + redis['bind'] = '10.0.0.2' + + # Define a port so Redis can listen for TCP requests which will allow other + # machines to connect to it. + redis['port'] = 6379 + + # The same password for Redis authentication you set up for the primary node. + redis['password'] = 'redis-password-goes-here' + + # The IP of the primary Redis node. + redis['master_ip'] = '10.0.0.1' + + # Port of primary Redis server, uncomment to change to non default. Defaults + # to `6379`. + #redis['master_port'] = 6379 + ``` + +1. To prevent reconfigure from running automatically on upgrade, run: + + ```shell + sudo touch /etc/gitlab/skip-auto-reconfigure + ``` + +1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. +1. Go through the steps again for all the other replica nodes. + +NOTE: **Note:** +You can specify multiple roles like sentinel and Redis as: +`roles ['redis_sentinel_role', 'redis_master_role']`. +Read more about [roles](https://docs.gitlab.com/omnibus/roles/). + +These values don't have to be changed again in `/etc/gitlab/gitlab.rb` after +a failover, as the nodes will be managed by the Sentinels, and even after a +`gitlab-ctl reconfigure`, they will get their configuration restored by +the same Sentinels. + +### Step 3. Configuring the Redis Sentinel instances + +NOTE: **Note:** +If you are using an external Redis Sentinel instance, be sure +to exclude the `requirepass` parameter from the Sentinel +configuration. This parameter will cause clients to report `NOAUTH +Authentication required.`. [Redis Sentinel 3.2.x does not support +password authentication](https://github.com/antirez/redis/issues/3279). + +Now that the Redis servers are all set up, let's configure the Sentinel +servers. + +If you are not sure if your Redis servers are working and replicating +correctly, please read the [Troubleshooting Replication](troubleshooting.md#troubleshooting-redis-replication) +and fix it before proceeding with Sentinel setup. + +You must have at least `3` Redis Sentinel servers, and they need to +be each in an independent machine. You can configure them in the same +machines where you've configured the other Redis servers. + +With GitLab Enterprise Edition, you can use the Omnibus package to set up +multiple machines with the Sentinel daemon. + +--- + +1. SSH into the server that will host Redis Sentinel. +1. **You can omit this step if the Sentinels will be hosted in the same node as + the other Redis instances.** + + [Download/install](https://about.gitlab.com/install/) the + Omnibus GitLab Enterprise Edition package using **steps 1 and 2** from the + GitLab downloads page. + - Make sure you select the correct Omnibus package, with the same version + the GitLab application is running. + - Do not complete any other steps on the download page. + +1. Edit `/etc/gitlab/gitlab.rb` and add the contents (if you are installing the + Sentinels in the same node as the other Redis instances, some values might + be duplicate below): + + ```ruby + roles ['redis_sentinel_role'] + + # Must be the same in every sentinel node + redis['master_name'] = 'gitlab-redis' + + # The same password for Redis authentication you set up for the primary node. + redis['master_password'] = 'redis-password-goes-here' + + # The IP of the primary Redis node. + redis['master_ip'] = '10.0.0.1' + + # Define a port so Redis can listen for TCP requests which will allow other + # machines to connect to it. + redis['port'] = 6379 + + # Port of primary Redis server, uncomment to change to non default. Defaults + # to `6379`. + #redis['master_port'] = 6379 + + ## Configure Sentinel + sentinel['bind'] = '10.0.0.1' + + # Port that Sentinel listens on, uncomment to change to non default. Defaults + # to `26379`. + # sentinel['port'] = 26379 + + ## Quorum must reflect the amount of voting sentinels it take to start a failover. + ## Value must NOT be greater then the amount of sentinels. + ## + ## The quorum can be used to tune Sentinel in two ways: + ## 1. If a the quorum is set to a value smaller than the majority of Sentinels + ## we deploy, we are basically making Sentinel more sensible to primary failures, + ## triggering a failover as soon as even just a minority of Sentinels is no longer + ## able to talk with the primary. + ## 1. If a quorum is set to a value greater than the majority of Sentinels, we are + ## making Sentinel able to failover only when there are a very large number (larger + ## than majority) of well connected Sentinels which agree about the primary being down.s + sentinel['quorum'] = 2 + + ## Consider unresponsive server down after x amount of ms. + # sentinel['down_after_milliseconds'] = 10000 + + ## Specifies the failover timeout in milliseconds. It is used in many ways: + ## + ## - The time needed to re-start a failover after a previous failover was + ## already tried against the same primary by a given Sentinel, is two + ## times the failover timeout. + ## + ## - The time needed for a replica replicating to a wrong primary according + ## to a Sentinel current configuration, to be forced to replicate + ## with the right primary, is exactly the failover timeout (counting since + ## the moment a Sentinel detected the misconfiguration). + ## + ## - The time needed to cancel a failover that is already in progress but + ## did not produced any configuration change (REPLICAOF NO ONE yet not + ## acknowledged by the promoted replica). + ## + ## - The maximum time a failover in progress waits for all the replica to be + ## reconfigured as replicas of the new primary. However even after this time + ## the replicas will be reconfigured by the Sentinels anyway, but not with + ## the exact parallel-syncs progression as specified. + # sentinel['failover_timeout'] = 60000 + ``` + +1. To prevent database migrations from running on upgrade, run: + + ```shell + sudo touch /etc/gitlab/skip-auto-reconfigure + ``` + + Only the primary GitLab application server should handle migrations. + +1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. +1. Go through the steps again for all the other Sentinel nodes. + +### Step 4. Configuring the GitLab application + +The final part is to inform the main GitLab application server of the Redis +Sentinels servers and authentication credentials. + +You can enable or disable Sentinel support at any time in new or existing +installations. From the GitLab application perspective, all it requires is +the correct credentials for the Sentinel nodes. + +While it doesn't require a list of all Sentinel nodes, in case of a failure, +it needs to access at least one of the listed. + +NOTE: **Note:** +The following steps should be performed in the [GitLab application server](../high_availability/gitlab.md) +which ideally should not have Redis or Sentinels on it for a HA setup. + +1. SSH into the server where the GitLab application is installed. +1. Edit `/etc/gitlab/gitlab.rb` and add/change the following lines: + + ```ruby + ## Must be the same in every sentinel node + redis['master_name'] = 'gitlab-redis' + + ## The same password for Redis authentication you set up for the primary node. + redis['master_password'] = 'redis-password-goes-here' + + ## A list of sentinels with `host` and `port` + gitlab_rails['redis_sentinels'] = [ + {'host' => '10.0.0.1', 'port' => 26379}, + {'host' => '10.0.0.2', 'port' => 26379}, + {'host' => '10.0.0.3', 'port' => 26379} + ] + ``` + +1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + +### Step 5. Enable Monitoring + +> [Introduced](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/3786) in GitLab 12.0. + +If you enable Monitoring, it must be enabled on **all** Redis servers. + +1. Make sure to collect [`CONSUL_SERVER_NODES`](../postgresql/replication_and_failover.md#consul-information), which are the IP addresses or DNS records of the Consul server nodes, for the next step. Note they are presented as `Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z` + +1. Create/edit `/etc/gitlab/gitlab.rb` and add the following configuration: + + ```ruby + # Enable service discovery for Prometheus + consul['enable'] = true + consul['monitoring_service_discovery'] = true + + # Replace placeholders + # Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z + # with the addresses of the Consul server nodes + consul['configuration'] = { + retry_join: %w(Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z), + } + + # Set the network addresses that the exporters will listen on + node_exporter['listen_address'] = '0.0.0.0:9100' + redis_exporter['listen_address'] = '0.0.0.0:9121' + ``` + +1. Run `sudo gitlab-ctl reconfigure` to compile the configuration. + +## Example of a minimal configuration with 1 primary, 2 replicas and 3 Sentinels + +In this example we consider that all servers have an internal network +interface with IPs in the `10.0.0.x` range, and that they can connect +to each other using these IPs. + +In a real world usage, you would also set up firewall rules to prevent +unauthorized access from other machines and block traffic from the +outside (Internet). + +We will use the same `3` nodes with **Redis** + **Sentinel** topology +discussed in [Redis setup overview](#redis-setup-overview) and +[Sentinel setup overview](#sentinel-setup-overview) documentation. + +Here is a list and description of each **machine** and the assigned **IP**: + +- `10.0.0.1`: Redis primary + Sentinel 1 +- `10.0.0.2`: Redis Replica 1 + Sentinel 2 +- `10.0.0.3`: Redis Replica 2 + Sentinel 3 +- `10.0.0.4`: GitLab application + +Please note that after the initial configuration, if a failover is initiated +by the Sentinel nodes, the Redis nodes will be reconfigured and the **Primary** +will change permanently (including in `redis.conf`) from one node to the other, +until a new failover is initiated again. + +The same thing will happen with `sentinel.conf` that will be overridden after the +initial execution, after any new sentinel node starts watching the **Primary**, +or a failover promotes a different **Primary** node. + +### Example configuration for Redis primary and Sentinel 1 + +In `/etc/gitlab/gitlab.rb`: + +```ruby +roles ['redis_sentinel_role', 'redis_master_role'] +redis['bind'] = '10.0.0.1' +redis['port'] = 6379 +redis['password'] = 'redis-password-goes-here' +redis['master_name'] = 'gitlab-redis' # must be the same in every sentinel node +redis['master_password'] = 'redis-password-goes-here' # the same value defined in redis['password'] in the primary instance +redis['master_ip'] = '10.0.0.1' # ip of the initial primary redis instance +#redis['master_port'] = 6379 # port of the initial primary redis instance, uncomment to change to non default +sentinel['bind'] = '10.0.0.1' +# sentinel['port'] = 26379 # uncomment to change default port +sentinel['quorum'] = 2 +# sentinel['down_after_milliseconds'] = 10000 +# sentinel['failover_timeout'] = 60000 +``` + +[Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + +### Example configuration for Redis replica 1 and Sentinel 2 + +In `/etc/gitlab/gitlab.rb`: + +```ruby +roles ['redis_sentinel_role', 'redis_replica_role'] +redis['bind'] = '10.0.0.2' +redis['port'] = 6379 +redis['password'] = 'redis-password-goes-here' +redis['master_password'] = 'redis-password-goes-here' +redis['master_ip'] = '10.0.0.1' # IP of primary Redis server +#redis['master_port'] = 6379 # Port of primary Redis server, uncomment to change to non default +redis['master_name'] = 'gitlab-redis' # must be the same in every sentinel node +sentinel['bind'] = '10.0.0.2' +# sentinel['port'] = 26379 # uncomment to change default port +sentinel['quorum'] = 2 +# sentinel['down_after_milliseconds'] = 10000 +# sentinel['failover_timeout'] = 60000 +``` + +[Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + +### Example configuration for Redis replica 2 and Sentinel 3 + +In `/etc/gitlab/gitlab.rb`: + +```ruby +roles ['redis_sentinel_role', 'redis_replica_role'] +redis['bind'] = '10.0.0.3' +redis['port'] = 6379 +redis['password'] = 'redis-password-goes-here' +redis['master_password'] = 'redis-password-goes-here' +redis['master_ip'] = '10.0.0.1' # IP of primary Redis server +#redis['master_port'] = 6379 # Port of primary Redis server, uncomment to change to non default +redis['master_name'] = 'gitlab-redis' # must be the same in every sentinel node +sentinel['bind'] = '10.0.0.3' +# sentinel['port'] = 26379 # uncomment to change default port +sentinel['quorum'] = 2 +# sentinel['down_after_milliseconds'] = 10000 +# sentinel['failover_timeout'] = 60000 +``` + +[Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + +### Example configuration for the GitLab application + +In `/etc/gitlab/gitlab.rb`: + +```ruby +redis['master_name'] = 'gitlab-redis' +redis['master_password'] = 'redis-password-goes-here' +gitlab_rails['redis_sentinels'] = [ + {'host' => '10.0.0.1', 'port' => 26379}, + {'host' => '10.0.0.2', 'port' => 26379}, + {'host' => '10.0.0.3', 'port' => 26379} +] +``` + +[Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + +## Advanced configuration + +Omnibus GitLab configures some things behind the curtains to make the sysadmins' +lives easier. If you want to know what happens underneath keep reading. + +### Running multiple Redis clusters + +GitLab supports running [separate Redis clusters for different persistent +classes](https://docs.gitlab.com/omnibus/settings/redis.html#running-with-multiple-redis-instances): +cache, queues, and shared_state. To make this work with Sentinel: + +1. Set the appropriate variable in `/etc/gitlab/gitlab.rb` for each instance you are using: + + ```ruby + gitlab_rails['redis_cache_instance'] = REDIS_CACHE_URL + gitlab_rails['redis_queues_instance'] = REDIS_QUEUES_URL + gitlab_rails['redis_shared_state_instance'] = REDIS_SHARED_STATE_URL + ``` + + **Note**: Redis URLs should be in the format: `redis://:PASSWORD@SENTINEL_PRIMARY_NAME` + + 1. PASSWORD is the plaintext password for the Redis instance + 1. SENTINEL_PRIMARY_NAME is the Sentinel primary name (e.g. `gitlab-redis-cache`) + +1. Include an array of hashes with host/port combinations, such as the following: + + ```ruby + gitlab_rails['redis_cache_sentinels'] = [ + { host: REDIS_CACHE_SENTINEL_HOST, port: PORT1 }, + { host: REDIS_CACHE_SENTINEL_HOST2, port: PORT2 } + ] + gitlab_rails['redis_queues_sentinels'] = [ + { host: REDIS_QUEUES_SENTINEL_HOST, port: PORT1 }, + { host: REDIS_QUEUES_SENTINEL_HOST2, port: PORT2 } + ] + gitlab_rails['redis_shared_state_sentinels'] = [ + { host: SHARED_STATE_SENTINEL_HOST, port: PORT1 }, + { host: SHARED_STATE_SENTINEL_HOST2, port: PORT2 } + ] + ``` + +1. Note that for each persistence class, GitLab will default to using the + configuration specified in `gitlab_rails['redis_sentinels']` unless + overridden by the settings above. +1. Be sure to include BOTH configuration options for each persistent classes. For example, + if you choose to configure a cache instance, you must specify both `gitlab_rails['redis_cache_instance']` + and `gitlab_rails['redis_cache_sentinels']` for GitLab to generate the proper configuration files. +1. Run `gitlab-ctl reconfigure` + +### Control running services + +In the previous example, we've used `redis_sentinel_role` and +`redis_master_role` which simplifies the amount of configuration changes. + +If you want more control, here is what each one sets for you automatically +when enabled: + +```ruby +## Redis Sentinel Role +redis_sentinel_role['enable'] = true + +# When Sentinel Role is enabled, the following services are also enabled +sentinel['enable'] = true + +# The following services are disabled +redis['enable'] = false +bootstrap['enable'] = false +nginx['enable'] = false +postgresql['enable'] = false +gitlab_rails['enable'] = false +mailroom['enable'] = false + +------- + +## Redis primary/replica Role +redis_master_role['enable'] = true # enable only one of them +redis_replica_role['enable'] = true # enable only one of them + +# When Redis primary or Replica role are enabled, the following services are +# enabled/disabled. Note that if Redis and Sentinel roles are combined, both +# services will be enabled. + +# The following services are disabled +sentinel['enable'] = false +bootstrap['enable'] = false +nginx['enable'] = false +postgresql['enable'] = false +gitlab_rails['enable'] = false +mailroom['enable'] = false + +# For Redis Replica role, also change this setting from default 'true' to 'false': +redis['master'] = false +``` + +You can find the relevant attributes defined in [`gitlab_rails.rb`](https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/libraries/gitlab_rails.rb). + +## Troubleshooting + +See the [Redis troubleshooting guide](troubleshooting.md). + +## Further reading + +Read more: + +1. [Reference architectures](../reference_architectures/index.md) +1. [Configure the database](../postgresql/replication_and_failover.md) +1. [Configure NFS](../high_availability/nfs.md) +1. [Configure the GitLab application servers](../high_availability/gitlab.md) +1. [Configure the load balancers](../high_availability/load_balancer.md) diff --git a/doc/administration/redis/replication_and_failover_external.md b/doc/administration/redis/replication_and_failover_external.md new file mode 100644 index 00000000000..244b44dd76a --- /dev/null +++ b/doc/administration/redis/replication_and_failover_external.md @@ -0,0 +1,376 @@ +--- +type: howto +stage: Enablement +group: Distribution +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Redis replication and failover providing your own instance **(CORE ONLY)** + +If you’re hosting GitLab on a cloud provider, you can optionally use a managed +service for Redis. For example, AWS offers ElastiCache that runs Redis. + +Alternatively, you may opt to manage your own Redis instance separate from the +Omnibus GitLab package. + +## Requirements + +The following are the requirements for providing your own Redis instance: + +- Redis version 5.0 or higher is recommended, as this is what ships with + Omnibus GitLab packages starting with GitLab 12.7. +- Support for Redis 3.2 is deprecated with GitLab 12.10 and will be completely + removed in GitLab 13.0. +- GitLab 12.0 and later requires Redis version 3.2 or higher. Older Redis + versions do not support an optional count argument to SPOP which is now + required for [Merge Trains](../../ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md). +- In addition, if Redis 4 or later is available, GitLab makes use of certain + commands like `UNLINK` and `USAGE` which were introduced only in Redis 4. +- Standalone Redis or Redis high availability with Sentinel are supported. Redis + Cluster is not supported. +- Managed Redis from cloud providers such as AWS ElastiCache will work. If these + services support high availability, be sure it is **not** the Redis Cluster type. + +Note the Redis node's IP address or hostname, port, and password (if required). + +## Redis as a managed service in a cloud provider + +1. Set up Redis according to the [requirements](#requirements). +1. Configure the GitLab application servers with the appropriate connection details + for your external Redis service in your `/etc/gitlab/gitlab.rb` file: + + ```ruby + redis['enable'] = false + + gitlab_rails['redis_host'] = 'redis.example.com' + gitlab_rails['redis_port'] = 6379 + + # Required if Redis authentication is configured on the Redis node + gitlab_rails['redis_password'] = 'Redis Password' + ``` + +1. Reconfigure for the changes to take effect: + + ```shell + sudo gitlab-ctl reconfigure + ``` + +## Redis replication and failover with your own Redis servers + +This is the documentation for configuring a scalable Redis setup when +you have installed Redis all by yourself and not using the bundled one that +comes with the Omnibus packages, although using the Omnibus GitLab packages is +highly recommend as we optimize them specifically for GitLab, and we will take +care of upgrading Redis to the latest supported version. + +Note also that you may elect to override all references to +`/home/git/gitlab/config/resque.yml` in accordance with the advanced Redis +settings outlined in +[Configuration Files Documentation](https://gitlab.com/gitlab-org/gitlab/blob/master/config/README.md). + +We cannot stress enough the importance of reading the +[replication and failover](replication_and_failover.md) documentation of the +Omnibus Redis HA as it provides some invaluable information to the configuration +of Redis. Please proceed to read it before going forward with this guide. + +Before proceeding on setting up the new Redis instances, here are some +requirements: + +- All Redis servers in this guide must be configured to use a TCP connection + instead of a socket. To configure Redis to use TCP connections you need to + define both `bind` and `port` in the Redis config file. You can bind to all + interfaces (`0.0.0.0`) or specify the IP of the desired interface + (e.g., one from an internal network). +- Since Redis 3.2, you must define a password to receive external connections + (`requirepass`). +- If you are using Redis with Sentinel, you will also need to define the same + password for the replica password definition (`masterauth`) in the same instance. + +In addition, read the prerequisites as described in the +[Omnibus Redis document](replication_and_failover.md#requirements) since they provide some +valuable information for the general setup. + +### Step 1. Configuring the primary Redis instance + +Assuming that the Redis primary instance IP is `10.0.0.1`: + +1. [Install Redis](../../install/installation.md#7-redis). +1. Edit `/etc/redis/redis.conf`: + + ```conf + ## Define a `bind` address pointing to a local IP that your other machines + ## can reach you. If you really need to bind to an external accessible IP, make + ## sure you add extra firewall rules to prevent unauthorized access: + bind 10.0.0.1 + + ## Define a `port` to force redis to listen on TCP so other machines can + ## connect to it (default port is `6379`). + port 6379 + + ## Set up password authentication (use the same password in all nodes). + ## The password should be defined equal for both `requirepass` and `masterauth` + ## when setting up Redis to use with Sentinel. + requirepass redis-password-goes-here + masterauth redis-password-goes-here + ``` + +1. Restart the Redis service for the changes to take effect. + +### Step 2. Configuring the replica Redis instances + +Assuming that the Redis replica instance IP is `10.0.0.2`: + +1. [Install Redis](../../install/installation.md#7-redis). +1. Edit `/etc/redis/redis.conf`: + + ```conf + ## Define a `bind` address pointing to a local IP that your other machines + ## can reach you. If you really need to bind to an external accessible IP, make + ## sure you add extra firewall rules to prevent unauthorized access: + bind 10.0.0.2 + + ## Define a `port` to force redis to listen on TCP so other machines can + ## connect to it (default port is `6379`). + port 6379 + + ## Set up password authentication (use the same password in all nodes). + ## The password should be defined equal for both `requirepass` and `masterauth` + ## when setting up Redis to use with Sentinel. + requirepass redis-password-goes-here + masterauth redis-password-goes-here + + ## Define `replicaof` pointing to the Redis primary instance with IP and port. + replicaof 10.0.0.1 6379 + ``` + +1. Restart the Redis service for the changes to take effect. +1. Go through the steps again for all the other replica nodes. + +### Step 3. Configuring the Redis Sentinel instances + +Sentinel is a special type of Redis server. It inherits most of the basic +configuration options you can define in `redis.conf`, with specific ones +starting with `sentinel` prefix. + +Assuming that the Redis Sentinel is installed on the same instance as Redis +primary with IP `10.0.0.1` (some settings might overlap with the primary): + +1. [Install Redis Sentinel](https://redis.io/topics/sentinel). +1. Edit `/etc/redis/sentinel.conf`: + + ```conf + ## Define a `bind` address pointing to a local IP that your other machines + ## can reach you. If you really need to bind to an external accessible IP, make + ## sure you add extra firewall rules to prevent unauthorized access: + bind 10.0.0.1 + + ## Define a `port` to force Sentinel to listen on TCP so other machines can + ## connect to it (default port is `6379`). + port 26379 + + ## Set up password authentication (use the same password in all nodes). + ## The password should be defined equal for both `requirepass` and `masterauth` + ## when setting up Redis to use with Sentinel. + requirepass redis-password-goes-here + masterauth redis-password-goes-here + + ## Define with `sentinel auth-pass` the same shared password you have + ## defined for both Redis primary and replicas instances. + sentinel auth-pass gitlab-redis redis-password-goes-here + + ## Define with `sentinel monitor` the IP and port of the Redis + ## primary node, and the quorum required to start a failover. + sentinel monitor gitlab-redis 10.0.0.1 6379 2 + + ## Define with `sentinel down-after-milliseconds` the time in `ms` + ## that an unresponsive server will be considered down. + sentinel down-after-milliseconds gitlab-redis 10000 + + ## Define a value for `sentinel failover_timeout` in `ms`. This has multiple + ## meanings: + ## + ## * The time needed to re-start a failover after a previous failover was + ## already tried against the same primary by a given Sentinel, is two + ## times the failover timeout. + ## + ## * The time needed for a replica replicating to a wrong primary according + ## to a Sentinel current configuration, to be forced to replicate + ## with the right primary, is exactly the failover timeout (counting since + ## the moment a Sentinel detected the misconfiguration). + ## + ## * The time needed to cancel a failover that is already in progress but + ## did not produced any configuration change (REPLICAOF NO ONE yet not + ## acknowledged by the promoted replica). + ## + ## * The maximum time a failover in progress waits for all the replicas to be + ## reconfigured as replicas of the new primary. However even after this time + ## the replicas will be reconfigured by the Sentinels anyway, but not with + ## the exact parallel-syncs progression as specified. + sentinel failover_timeout 30000 + ``` + +1. Restart the Redis service for the changes to take effect. +1. Go through the steps again for all the other Sentinel nodes. + +### Step 4. Configuring the GitLab application + +You can enable or disable Sentinel support at any time in new or existing +installations. From the GitLab application perspective, all it requires is +the correct credentials for the Sentinel nodes. + +While it doesn't require a list of all Sentinel nodes, in case of a failure, +it needs to access at least one of listed ones. + +The following steps should be performed in the [GitLab application server](../high_availability/gitlab.md) +which ideally should not have Redis or Sentinels in the same machine: + +1. Edit `/home/git/gitlab/config/resque.yml` following the example in + [resque.yml.example](https://gitlab.com/gitlab-org/gitlab/blob/master/config/resque.yml.example), and uncomment the Sentinel lines, pointing to + the correct server credentials: + + ```yaml + # resque.yaml + production: + url: redis://:redi-password-goes-here@gitlab-redis/ + sentinels: + - + host: 10.0.0.1 + port: 26379 # point to sentinel, not to redis port + - + host: 10.0.0.2 + port: 26379 # point to sentinel, not to redis port + - + host: 10.0.0.3 + port: 26379 # point to sentinel, not to redis port + ``` + +1. [Restart GitLab](../restart_gitlab.md#installations-from-source) for the changes to take effect. + +## Example of minimal configuration with 1 primary, 2 replicas and 3 sentinels + +In this example we consider that all servers have an internal network +interface with IPs in the `10.0.0.x` range, and that they can connect +to each other using these IPs. + +In a real world usage, you would also set up firewall rules to prevent +unauthorized access from other machines, and block traffic from the +outside ([Internet](https://gitlab.com/gitlab-org/gitlab-foss/uploads/c4cc8cd353604bd80315f9384035ff9e/The_Internet_IT_Crowd.png)). + +For this example, **Sentinel 1** will be configured in the same machine as the +**Redis Primary**, **Sentinel 2** and **Sentinel 3** in the same machines as the +**Replica 1** and **Replica 2** respectively. + +Here is a list and description of each **machine** and the assigned **IP**: + +- `10.0.0.1`: Redis Primary + Sentinel 1 +- `10.0.0.2`: Redis Replica 1 + Sentinel 2 +- `10.0.0.3`: Redis Replica 2 + Sentinel 3 +- `10.0.0.4`: GitLab application + +Please note that after the initial configuration, if a failover is initiated +by the Sentinel nodes, the Redis nodes will be reconfigured and the **Primary** +will change permanently (including in `redis.conf`) from one node to the other, +until a new failover is initiated again. + +The same thing will happen with `sentinel.conf` that will be overridden after the +initial execution, after any new sentinel node starts watching the **Primary**, +or a failover promotes a different **Primary** node. + +### Example configuration for Redis primary and Sentinel 1 + +1. In `/etc/redis/redis.conf`: + + ```conf + bind 10.0.0.1 + port 6379 + requirepass redis-password-goes-here + masterauth redis-password-goes-here + ``` + +1. In `/etc/redis/sentinel.conf`: + + ```conf + bind 10.0.0.1 + port 26379 + sentinel auth-pass gitlab-redis redis-password-goes-here + sentinel monitor gitlab-redis 10.0.0.1 6379 2 + sentinel down-after-milliseconds gitlab-redis 10000 + sentinel failover_timeout 30000 + ``` + +1. Restart the Redis service for the changes to take effect. + +### Example configuration for Redis replica 1 and Sentinel 2 + +1. In `/etc/redis/redis.conf`: + + ```conf + bind 10.0.0.2 + port 6379 + requirepass redis-password-goes-here + masterauth redis-password-goes-here + replicaof 10.0.0.1 6379 + ``` + +1. In `/etc/redis/sentinel.conf`: + + ```conf + bind 10.0.0.2 + port 26379 + sentinel auth-pass gitlab-redis redis-password-goes-here + sentinel monitor gitlab-redis 10.0.0.1 6379 2 + sentinel down-after-milliseconds gitlab-redis 10000 + sentinel failover_timeout 30000 + ``` + +1. Restart the Redis service for the changes to take effect. + +### Example configuration for Redis replica 2 and Sentinel 3 + +1. In `/etc/redis/redis.conf`: + + ```conf + bind 10.0.0.3 + port 6379 + requirepass redis-password-goes-here + masterauth redis-password-goes-here + replicaof 10.0.0.1 6379 + ``` + +1. In `/etc/redis/sentinel.conf`: + + ```conf + bind 10.0.0.3 + port 26379 + sentinel auth-pass gitlab-redis redis-password-goes-here + sentinel monitor gitlab-redis 10.0.0.1 6379 2 + sentinel down-after-milliseconds gitlab-redis 10000 + sentinel failover_timeout 30000 + ``` + +1. Restart the Redis service for the changes to take effect. + +### Example configuration of the GitLab application + +1. Edit `/home/git/gitlab/config/resque.yml`: + + ```yaml + production: + url: redis://:redi-password-goes-here@gitlab-redis/ + sentinels: + - + host: 10.0.0.1 + port: 26379 # point to sentinel, not to redis port + - + host: 10.0.0.2 + port: 26379 # point to sentinel, not to redis port + - + host: 10.0.0.3 + port: 26379 # point to sentinel, not to redis port + ``` + +1. [Restart GitLab](../restart_gitlab.md#installations-from-source) for the changes to take effect. + +## Troubleshooting + +See the [Redis troubleshooting guide](troubleshooting.md). diff --git a/doc/administration/redis/standalone.md b/doc/administration/redis/standalone.md new file mode 100644 index 00000000000..12e932dbc5e --- /dev/null +++ b/doc/administration/redis/standalone.md @@ -0,0 +1,63 @@ +--- +type: howto +stage: Enablement +group: Distribution +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Standalone Redis using Omnibus GitLab **(CORE ONLY)** + +The Omnibus GitLab package can be used to configure a standalone Redis server. +In this configuration, Redis is not scaled, and represents a single +point of failure. However, in a scaled environment the objective is to allow +the environment to handle more users or to increase throughput. Redis itself +is generally stable and can handle many requests, so it is an acceptable +trade off to have only a single instance. See the [reference architectures](../reference_architectures/index.md) +page for an overview of GitLab scaling options. + +## Set up a standalone Redis instance + +The steps below are the minimum necessary to configure a Redis server with +Omnibus GitLab: + +1. SSH into the Redis server. +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package you want by using **steps 1 and 2** from the GitLab downloads page. + Do not complete any other steps on the download page. + +1. Edit `/etc/gitlab/gitlab.rb` and add the contents: + + ```ruby + ## Enable Redis + redis['enable'] = true + + ## Disable all other services + sidekiq['enable'] = false + gitlab_workhorse['enable'] = false + puma['enable'] = false + postgresql['enable'] = false + nginx['enable'] = false + prometheus['enable'] = false + alertmanager['enable'] = false + pgbouncer_exporter['enable'] = false + gitlab_exporter['enable'] = false + gitaly['enable'] = false + + redis['bind'] = '0.0.0.0' + redis['port'] = 6379 + redis['password'] = 'SECRET_PASSWORD_HERE' + + gitlab_rails['enable'] = false + ``` + +1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. +1. Note the Redis node's IP address or hostname, port, and + Redis password. These will be necessary when configuring the GitLab + application servers later. + +[Advanced configuration options](https://docs.gitlab.com/omnibus/settings/redis.html) +are supported and can be added if needed. + +## Troubleshooting + +See the [Redis troubleshooting guide](troubleshooting.md). diff --git a/doc/administration/redis/troubleshooting.md b/doc/administration/redis/troubleshooting.md new file mode 100644 index 00000000000..402b60e5b7b --- /dev/null +++ b/doc/administration/redis/troubleshooting.md @@ -0,0 +1,158 @@ +--- +type: reference +stage: Enablement +group: Distribution +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Troubleshooting Redis + +There are a lot of moving parts that needs to be taken care carefully +in order for the HA setup to work as expected. + +Before proceeding with the troubleshooting below, check your firewall rules: + +- Redis machines + - Accept TCP connection in `6379` + - Connect to the other Redis machines via TCP in `6379` +- Sentinel machines + - Accept TCP connection in `26379` + - Connect to other Sentinel machines via TCP in `26379` + - Connect to the Redis machines via TCP in `6379` + +## Troubleshooting Redis replication + +You can check if everything is correct by connecting to each server using +`redis-cli` application, and sending the `info replication` command as below. + +```shell +/opt/gitlab/embedded/bin/redis-cli -h <redis-host-or-ip> -a '<redis-password>' info replication +``` + +When connected to a `Primary` Redis, you will see the number of connected +`replicas`, and a list of each with connection details: + +```plaintext +# Replication +role:master +connected_replicas:1 +replica0:ip=10.133.5.21,port=6379,state=online,offset=208037514,lag=1 +master_repl_offset:208037658 +repl_backlog_active:1 +repl_backlog_size:1048576 +repl_backlog_first_byte_offset:206989083 +repl_backlog_histlen:1048576 +``` + +When it's a `replica`, you will see details of the primary connection and if +its `up` or `down`: + +```plaintext +# Replication +role:replica +master_host:10.133.1.58 +master_port:6379 +master_link_status:up +master_last_io_seconds_ago:1 +master_sync_in_progress:0 +replica_repl_offset:208096498 +replica_priority:100 +replica_read_only:1 +connected_replicas:0 +master_repl_offset:0 +repl_backlog_active:0 +repl_backlog_size:1048576 +repl_backlog_first_byte_offset:0 +repl_backlog_histlen:0 +``` + +## Troubleshooting Sentinel + +If you get an error like: `Redis::CannotConnectError: No sentinels available.`, +there may be something wrong with your configuration files or it can be related +to [this issue](https://github.com/redis/redis-rb/issues/531). + +You must make sure you are defining the same value in `redis['master_name']` +and `redis['master_pasword']` as you defined for your sentinel node. + +The way the Redis connector `redis-rb` works with sentinel is a bit +non-intuitive. We try to hide the complexity in omnibus, but it still requires +a few extra configurations. + +--- + +To make sure your configuration is correct: + +1. SSH into your GitLab application server +1. Enter the Rails console: + + ```shell + # For Omnibus installations + sudo gitlab-rails console + + # For source installations + sudo -u git rails console -e production + ``` + +1. Run in the console: + + ```ruby + redis = Redis.new(Gitlab::Redis::SharedState.params) + redis.info + ``` + + Keep this screen open and try to simulate a failover below. + +1. To simulate a failover on primary Redis, SSH into the Redis server and run: + + ```shell + # port must match your primary redis port, and the sleep time must be a few seconds bigger than defined one + redis-cli -h localhost -p 6379 DEBUG sleep 20 + ``` + +1. Then back in the Rails console from the first step, run: + + ```ruby + redis.info + ``` + + You should see a different port after a few seconds delay + (the failover/reconnect time). + +## Troubleshooting a non-bundled Redis with an installation from source + +If you get an error in GitLab like `Redis::CannotConnectError: No sentinels available.`, +there may be something wrong with your configuration files or it can be related +to [this upstream issue](https://github.com/redis/redis-rb/issues/531). + +You must make sure that `resque.yml` and `sentinel.conf` are configured correctly, +otherwise `redis-rb` will not work properly. + +The `master-group-name` (`gitlab-redis`) defined in (`sentinel.conf`) +**must** be used as the hostname in GitLab (`resque.yml`): + +```conf +# sentinel.conf: +sentinel monitor gitlab-redis 10.0.0.1 6379 2 +sentinel down-after-milliseconds gitlab-redis 10000 +sentinel config-epoch gitlab-redis 0 +sentinel leader-epoch gitlab-redis 0 +``` + +```yaml +# resque.yaml +production: + url: redis://:myredispassword@gitlab-redis/ + sentinels: + - + host: 10.0.0.1 + port: 26379 # point to sentinel, not to redis port + - + host: 10.0.0.2 + port: 26379 # point to sentinel, not to redis port + - + host: 10.0.0.3 + port: 26379 # point to sentinel, not to redis port +``` + +When in doubt, read the [Redis Sentinel documentation](https://redis.io/topics/sentinel). diff --git a/doc/administration/reference_architectures/10k_users.md b/doc/administration/reference_architectures/10k_users.md index a15fcf722a5..5367021af4e 100644 --- a/doc/administration/reference_architectures/10k_users.md +++ b/doc/administration/reference_architectures/10k_users.md @@ -47,7 +47,7 @@ For a full list of reference architectures, see For medium sized installs (3,000 - 5,000) we suggest one Redis cluster for all classes and that Redis Sentinel is hosted alongside Consul. For larger architectures (10,000 users or more) we suggest running a separate - [Redis Cluster](../high_availability/redis.md#running-multiple-redis-clusters) for the Cache class + [Redis Cluster](../redis/replication_and_failover.md#running-multiple-redis-clusters) for the Cache class and another for the Queues and Shared State classes respectively. We also recommend that you run the Redis Sentinel clusters separately for each Redis Cluster. diff --git a/doc/administration/reference_architectures/1k_users.md b/doc/administration/reference_architectures/1k_users.md index 34805a8ac68..def23619a5c 100644 --- a/doc/administration/reference_architectures/1k_users.md +++ b/doc/administration/reference_architectures/1k_users.md @@ -57,7 +57,7 @@ added performance and reliability at a reduced complexity cost. For medium sized installs (3,000 - 5,000) we suggest one Redis cluster for all classes and that Redis Sentinel is hosted alongside Consul. For larger architectures (10,000 users or more) we suggest running a separate - [Redis Cluster](../high_availability/redis.md#running-multiple-redis-clusters) for the Cache class + [Redis Cluster](../redis/replication_and_failover.md#running-multiple-redis-clusters) for the Cache class and another for the Queues and Shared State classes respectively. We also recommend that you run the Redis Sentinel clusters separately for each Redis Cluster. diff --git a/doc/administration/reference_architectures/25k_users.md b/doc/administration/reference_architectures/25k_users.md index d851fa124c6..17f4300eb03 100644 --- a/doc/administration/reference_architectures/25k_users.md +++ b/doc/administration/reference_architectures/25k_users.md @@ -47,7 +47,7 @@ For a full list of reference architectures, see For medium sized installs (3,000 - 5,000) we suggest one Redis cluster for all classes and that Redis Sentinel is hosted alongside Consul. For larger architectures (10,000 users or more) we suggest running a separate - [Redis Cluster](../high_availability/redis.md#running-multiple-redis-clusters) for the Cache class + [Redis Cluster](../redis/replication_and_failover.md#running-multiple-redis-clusters) for the Cache class and another for the Queues and Shared State classes respectively. We also recommend that you run the Redis Sentinel clusters separately for each Redis Cluster. diff --git a/doc/administration/reference_architectures/2k_users.md b/doc/administration/reference_architectures/2k_users.md index 0a3ade1acf1..d182daf45b3 100644 --- a/doc/administration/reference_architectures/2k_users.md +++ b/doc/administration/reference_architectures/2k_users.md @@ -12,16 +12,16 @@ For a full list of reference architectures, see > - **High Availability:** False > - **Test requests per second (RPS) rates:** API: 40 RPS, Web: 4 RPS, Git: 4 RPS -| Service | Nodes | Configuration | GCP | AWS | Azure | -|--------------------------------------------------------------|-----------|---------------------------------|---------------|-----------------------|----------------| -| Load balancer | 1 | 2 vCPU, 1.8GB memory | n1-highcpu-2 | c5.large | F2s v2 | -| Object storage | n/a | n/a | n/a | n/a | n/a | -| NFS server (optional, not recommended) | 1 | 4 vCPU, 3.6GB memory | n1-highcpu-4 | c5.xlarge | F4s v2 | -| PostgreSQL | 1 | 2 vCPU, 7.5GB memory | n1-standard-2 | m5.large | D2s v3 | -| Redis | 1 | 1 vCPU, 3.75GB memory | n1-standard-1 | m5.large | D2s v3 | -| Gitaly | 1 | 4 vCPU, 15GB memory | n1-standard-4 | m5.xlarge | D4s v3 | -| GitLab Rails | 2 | 8 vCPU, 7.2GB memory | n1-highcpu-8 | c5.2xlarge | F8s v2 | -| Monitoring node | 1 | 2 vCPU, 1.8GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| Service | Nodes | Configuration | GCP | AWS | Azure | +|------------------------------------------|--------|-------------------------|-----------------|----------------|-----------| +| Load balancer | 1 | 2 vCPU, 1.8GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| PostgreSQL | 1 | 2 vCPU, 7.5GB memory | `n1-standard-2` | `m5.large` | `D2s v3` | +| Redis | 1 | 1 vCPU, 3.75GB memory | `n1-standard-1` | `m5.large` | `D2s v3` | +| Gitaly | 1 | 4 vCPU, 15GB memory | `n1-standard-4` | `m5.xlarge` | `D4s v3` | +| GitLab Rails | 2 | 8 vCPU, 7.2GB memory | `n1-highcpu-8` | `c5.2xlarge` | `F8s v2` | +| Monitoring node | 1 | 2 vCPU, 1.8GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| Object storage | n/a | n/a | n/a | n/a | n/a | +| NFS server (optional, not recommended) | 1 | 4 vCPU, 3.6GB memory | `n1-highcpu-4` | `c5.xlarge` | `F4s v2` | The Google Cloud Platform (GCP) architectures were built and tested using the [Intel Xeon E5 v3 (Haswell)](https://cloud.google.com/compute/docs/cpu-platforms) @@ -30,9 +30,6 @@ or higher, are required for your CPU or node counts. For more information, see our [Sysbench](https://github.com/akopytov/sysbench)-based [CPU benchmark](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Reference-Architectures/GCP-CPU-Benchmarks). -AWS-equivalent and Azure-equivalent configurations are rough suggestions that -may change in the future, and haven't been tested or validated. - Due to better performance and availability, for data objects (such as LFS, uploads, or artifacts), using an [object storage service](#configure-the-object-storage) is recommended instead of using NFS. Using an object storage service also @@ -44,12 +41,6 @@ To set up GitLab and its components to accommodate up to 2,000 users: 1. [Configure the external load balancing node](#configure-the-load-balancer) to handle the load balancing of the two GitLab application services nodes. -1. [Configure the object storage](#configure-the-object-storage) used for - shared data objects. -1. [Configure NFS](#configure-nfs-optional) (optional, and not recommended) - to have shared disk storage service as an alternative to Gitaly or object - storage. You can skip this step if you're not using GitLab Pages (which - requires NFS). 1. [Configure PostgreSQL](#configure-postgresql), the database for GitLab. 1. [Configure Redis](#configure-redis). 1. [Configure Gitaly](#configure-gitaly), which provides access to the Git @@ -59,6 +50,12 @@ To set up GitLab and its components to accommodate up to 2,000 users: requests (which include UI, API, and Git over HTTP/SSH). 1. [Configure Prometheus](#configure-prometheus) to monitor your GitLab environment. +1. [Configure the object storage](#configure-the-object-storage) used for + shared data objects. +1. [Configure NFS](#configure-nfs-optional) (optional, and not recommended) + to have shared disk storage service as an alternative to Gitaly or object + storage. You can skip this step if you're not using GitLab Pages (which + requires NFS). ## Configure the load balancer @@ -173,73 +170,6 @@ Configure DNS for an alternate SSH hostname, such as `altssh.gitlab.example.com` </a> </div> -## Configure the object storage - -GitLab supports using an object storage service for holding several types of -data, and is recommended over [NFS](#configure-nfs-optional). In general, -object storage services are better for larger environments, as object storage -is typically much more performant, reliable, and scalable. - -Object storage options that GitLab has either tested or is aware of customers -using, includes: - -- SaaS/Cloud solutions (such as [Amazon S3](https://aws.amazon.com/s3/) or - [Google Cloud Storage](https://cloud.google.com/storage)). -- On-premises hardware and appliances, from various storage vendors. -- MinIO ([Deployment guide](https://docs.gitlab.com/charts/advanced/external-object-storage/minio.html)). - -To configure GitLab to use object storage, refer to the following guides based -on the features you intend to use: - -1. [Object storage for backups](../../raketasks/backup_restore.md#uploading-backups-to-a-remote-cloud-storage). -1. [Object storage for job artifacts](../job_artifacts.md#using-object-storage) - including [incremental logging](../job_logs.md#new-incremental-logging-architecture). -1. [Object storage for LFS objects](../lfs/index.md#storing-lfs-objects-in-remote-object-storage). -1. [Object storage for uploads](../uploads.md#using-object-storage-core-only). -1. [Object storage for merge request diffs](../merge_request_diffs.md#using-object-storage). -1. [Object storage for Container Registry](../packages/container_registry.md#container-registry-storage-driver) (optional feature). -1. [Object storage for Mattermost](https://docs.mattermost.com/administration/config-settings.html#file-storage) (optional feature). -1. [Object storage for packages](../packages/index.md#using-object-storage) (optional feature). **(PREMIUM ONLY)** -1. [Object storage for Dependency Proxy](../packages/dependency_proxy.md#using-object-storage) (optional feature). **(PREMIUM ONLY)** -1. [Object storage for Pseudonymizer](../pseudonymizer.md#configuration) (optional feature). **(ULTIMATE ONLY)** -1. [Object storage for autoscale Runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional, for improved performance). -1. [Object storage for Terraform state files](../terraform_state.md#using-object-storage-core-only). - -Using separate buckets for each data type is the recommended approach for GitLab. - -A limitation of our configuration is that each use of object storage is -separately configured. We have an [issue](https://gitlab.com/gitlab-org/gitlab/-/issues/23345) -for improving this, which would allow for one bucket with separate folders. - -Using a single bucket when GitLab is deployed with the Helm chart causes -restoring from a backup to -[not function properly](https://docs.gitlab.com/charts/advanced/external-object-storage/#lfs-artifacts-uploads-packages-external-diffs-pseudonymizer). -Although you may not be using a Helm deployment right now, if you migrate -GitLab to a Helm deployment later, GitLab would still work, but you may not -realize backups aren't working correctly until a critical requirement for -functioning backups is encountered. - -<div align="right"> - <a type="button" class="btn btn-default" href="#setup-components"> - Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> - </a> -</div> - -## Configure NFS (optional) - -For improved performance, [object storage](#configure-the-object-storage), -along with [Gitaly](#configure-gitaly), are recommended over using NFS whenever -possible. However, if you intend to use GitLab Pages, -[you must use NFS](troubleshooting.md#gitlab-pages-requires-nfs). - -For information about configuring NFS, see the [NFS documentation page](../high_availability/nfs.md). - -<div align="right"> - <a type="button" class="btn btn-default" href="#setup-components"> - Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> - </a> -</div> - ## Configure PostgreSQL In this section, you'll be guided through configuring an external PostgreSQL database @@ -543,7 +473,7 @@ nodes (including the Gitaly node using the certificate) and on all client nodes that communicate with it following the procedure described in [GitLab custom certificate configuration](https://docs.gitlab.com/omnibus/settings/ssl.html#install-custom-public-certificates). -NOTE: **Note** +NOTE: **Note:** The self-signed certificate must specify the address you use to access the Gitaly server. If you are addressing the Gitaly server by a hostname, you can either use the Common Name field for this, or add it as a Subject Alternative @@ -728,7 +658,8 @@ On each node perform the following: sudo gitlab-ctl tail gitaly ``` -NOTE: **Note:** When you specify `https` in the `external_url`, as in the example +NOTE: **Note:** +When you specify `https` in the `external_url`, as in the example above, GitLab assumes you have SSL certificates in `/etc/gitlab/ssl/`. If certificates are not present, NGINX will fail to start. See the [NGINX documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https) @@ -862,6 +793,73 @@ running [Prometheus](../monitoring/prometheus/index.md) and </a> </div> +## Configure the object storage + +GitLab supports using an object storage service for holding several types of +data, and is recommended over [NFS](#configure-nfs-optional). In general, +object storage services are better for larger environments, as object storage +is typically much more performant, reliable, and scalable. + +Object storage options that GitLab has either tested or is aware of customers +using, includes: + +- SaaS/Cloud solutions (such as [Amazon S3](https://aws.amazon.com/s3/) or + [Google Cloud Storage](https://cloud.google.com/storage)). +- On-premises hardware and appliances, from various storage vendors. +- MinIO ([Deployment guide](https://docs.gitlab.com/charts/advanced/external-object-storage/minio.html)). + +To configure GitLab to use object storage, refer to the following guides based +on the features you intend to use: + +1. [Object storage for backups](../../raketasks/backup_restore.md#uploading-backups-to-a-remote-cloud-storage). +1. [Object storage for job artifacts](../job_artifacts.md#using-object-storage) + including [incremental logging](../job_logs.md#new-incremental-logging-architecture). +1. [Object storage for LFS objects](../lfs/index.md#storing-lfs-objects-in-remote-object-storage). +1. [Object storage for uploads](../uploads.md#using-object-storage-core-only). +1. [Object storage for merge request diffs](../merge_request_diffs.md#using-object-storage). +1. [Object storage for Container Registry](../packages/container_registry.md#use-object-storage) (optional feature). +1. [Object storage for Mattermost](https://docs.mattermost.com/administration/config-settings.html#file-storage) (optional feature). +1. [Object storage for packages](../packages/index.md#using-object-storage) (optional feature). **(PREMIUM ONLY)** +1. [Object storage for Dependency Proxy](../packages/dependency_proxy.md#using-object-storage) (optional feature). **(PREMIUM ONLY)** +1. [Object storage for Pseudonymizer](../pseudonymizer.md#configuration) (optional feature). **(ULTIMATE ONLY)** +1. [Object storage for autoscale Runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional, for improved performance). +1. [Object storage for Terraform state files](../terraform_state.md#using-object-storage-core-only). + +Using separate buckets for each data type is the recommended approach for GitLab. + +A limitation of our configuration is that each use of object storage is +separately configured. We have an [issue](https://gitlab.com/gitlab-org/gitlab/-/issues/23345) +for improving this, which would allow for one bucket with separate folders. + +Using a single bucket when GitLab is deployed with the Helm chart causes +restoring from a backup to +[not function properly](https://docs.gitlab.com/charts/advanced/external-object-storage/#lfs-artifacts-uploads-packages-external-diffs-pseudonymizer). +Although you may not be using a Helm deployment right now, if you migrate +GitLab to a Helm deployment later, GitLab would still work, but you may not +realize backups aren't working correctly until a critical requirement for +functioning backups is encountered. + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure NFS (optional) + +For improved performance, [object storage](#configure-the-object-storage), +along with [Gitaly](#configure-gitaly), are recommended over using NFS whenever +possible. However, if you intend to use GitLab Pages, +[you must use NFS](troubleshooting.md#gitlab-pages-requires-nfs). + +For information about configuring NFS, see the [NFS documentation page](../high_availability/nfs.md). + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + ## Troubleshooting See the [troubleshooting documentation](troubleshooting.md). diff --git a/doc/administration/reference_architectures/3k_users.md b/doc/administration/reference_architectures/3k_users.md index efeed3e9ffd..04cb9fa4769 100644 --- a/doc/administration/reference_architectures/3k_users.md +++ b/doc/administration/reference_architectures/3k_users.md @@ -1,10 +1,15 @@ +--- +reading_time: true +--- + # Reference architecture: up to 3,000 users This page describes GitLab reference architecture for up to 3,000 users. For a full list of reference architectures, see [Available reference architectures](index.md#available-reference-architectures). -NOTE: **Note:** The 3,000-user reference architecture documented below is +NOTE: **Note:** +The 3,000-user reference architecture documented below is designed to help your organization achieve a highly-available GitLab deployment. If you do not have the expertise or need to maintain a highly-available environment, you can have a simpler and less costly-to-operate environment by @@ -14,66 +19,1749 @@ following the [2,000-user reference architecture](2k_users.md). > - **High Availability:** True > - **Test RPS rates:** API: 60 RPS, Web: 6 RPS, Git: 6 RPS -| Service | Nodes | Configuration ([8](#footnotes)) | GCP | AWS | Azure | -|--------------------------------------------------------------|-------|---------------------------------|---------------|-----------------------|----------------| -| GitLab Rails ([1](#footnotes)) | 3 | 8 vCPU, 7.2GB Memory | `n1-highcpu-8` | `c5.2xlarge` | F8s v2 | -| PostgreSQL | 3 | 2 vCPU, 7.5GB Memory | `n1-standard-2` | `m5.large` | D2s v3 | -| PgBouncer | 3 | 2 vCPU, 1.8GB Memory | `n1-highcpu-2` | `c5.large` | F2s v2 | -| Gitaly ([2](#footnotes)) ([5](#footnotes)) ([7](#footnotes)) | X | 4 vCPU, 15GB Memory | `n1-standard-4` | `m5.xlarge` | D4s v3 | -| Redis ([3](#footnotes)) | 3 | 2 vCPU, 7.5GB Memory | `n1-standard-2` | `m5.large` | D2s v3 | -| Consul + Sentinel ([3](#footnotes)) | 3 | 2 vCPU, 1.8GB Memory | `n1-highcpu-2` | `c5.large` | F2s v2 | -| Sidekiq | 4 | 2 vCPU, 7.5GB Memory | `n1-standard-2` | `m5.large` | D2s v3 | -| Object Storage ([4](#footnotes)) | - | - | - | - | - | -| NFS Server ([5](#footnotes)) ([7](#footnotes)) | 1 | 4 vCPU, 3.6GB Memory | `n1-highcpu-4` | `c5.xlarge` | F4s v2 | -| Monitoring node | 1 | 2 vCPU, 1.8GB Memory | `n1-highcpu-2` | `c5.large` | F2s v2 | -| External load balancing node ([6](#footnotes)) | 1 | 2 vCPU, 1.8GB Memory | `n1-highcpu-2` | `c5.large` | F2s v2 | -| Internal load balancing node ([6](#footnotes)) | 1 | 2 vCPU, 1.8GB Memory | `n1-highcpu-2` | `c5.large` | F2s v2 | - -## Footnotes - -1. In our architectures we run each GitLab Rails node using the Puma webserver - and have its number of workers set to 90% of available CPUs along with four threads. For - nodes that are running Rails with other components the worker value should be reduced - accordingly where we've found 50% achieves a good balance but this is dependent - on workload. - -1. Gitaly node requirements are dependent on customer data, specifically the number of - projects and their sizes. We recommend two nodes as an absolute minimum for HA environments - and at least four nodes should be used when supporting 50,000 or more users. - We also recommend that each Gitaly node should store no more than 5TB of data - and have the number of [`gitaly-ruby` workers](../gitaly/index.md#gitaly-ruby) - set to 20% of available CPUs. Additional nodes should be considered in conjunction - with a review of expected data size and spread based on the recommendations above. - -1. Recommended Redis setup differs depending on the size of the architecture. - For smaller architectures (less than 3,000 users) a single instance should suffice. - For medium sized installs (3,000 - 5,000) we suggest one Redis cluster for all - classes and that Redis Sentinel is hosted alongside Consul. - For larger architectures (10,000 users or more) we suggest running a separate - [Redis Cluster](../high_availability/redis.md#running-multiple-redis-clusters) for the Cache class - and another for the Queues and Shared State classes respectively. We also recommend - that you run the Redis Sentinel clusters separately for each Redis Cluster. - -1. For data objects such as LFS, Uploads, Artifacts, etc. We recommend an [Object Storage service](../object_storage.md) - over NFS where possible, due to better performance and availability. - -1. NFS can be used as an alternative for both repository data (replacing Gitaly) and - object storage but this isn't typically recommended for performance reasons. Note however it is required for - [GitLab Pages](https://gitlab.com/gitlab-org/gitlab-pages/-/issues/196). - -1. Our architectures have been tested and validated with [HAProxy](https://www.haproxy.org/) - as the load balancer. Although other load balancers with similar feature sets - could also be used, those load balancers have not been validated. - -1. We strongly recommend that any Gitaly or NFS nodes be set up with SSD disks over - HDD with a throughput of at least 8,000 IOPS for read operations and 2,000 IOPS for write - as these components have heavy I/O. These IOPS values are recommended only as a starter - as with time they may be adjusted higher or lower depending on the scale of your - environment's workload. If you're running the environment on a Cloud provider - you may need to refer to their documentation on how configure IOPS correctly. - -1. The architectures were built and tested with the [Intel Xeon E5 v3 (Haswell)](https://cloud.google.com/compute/docs/cpu-platforms) - CPU platform on GCP. On different hardware you may find that adjustments, either lower - or higher, are required for your CPU or Node counts accordingly. For more information, a - [Sysbench](https://github.com/akopytov/sysbench) benchmark of the CPU can be found - [here](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Reference-Architectures/GCP-CPU-Benchmarks). +| Service | Nodes | Configuration | GCP | AWS | Azure | +|--------------------------------------------------------------|-------|---------------------------------|-----------------|-------------------------|----------------| +| External load balancing node | 1 | 2 vCPU, 1.8GB Memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| Redis | 3 | 2 vCPU, 7.5GB Memory | `n1-standard-2` | `m5.large` | `D2s v3` | +| Consul + Sentinel | 3 | 2 vCPU, 1.8GB Memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| PostgreSQL | 3 | 2 vCPU, 7.5GB Memory | `n1-standard-2` | `m5.large` | `D2s v3` | +| PgBouncer | 3 | 2 vCPU, 1.8GB Memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| Internal load balancing node | 1 | 2 vCPU, 1.8GB Memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| Gitaly | 2 minimum | 4 vCPU, 15GB Memory | `n1-standard-4` | `m5.xlarge` | `D4s v3` | +| Sidekiq | 4 | 2 vCPU, 7.5GB Memory | `n1-standard-2` | `m5.large` | `D2s v3` | +| GitLab Rails | 3 | 8 vCPU, 7.2GB Memory | `n1-highcpu-8` | `c5.2xlarge` | `F8s v2` | +| Monitoring node | 1 | 2 vCPU, 1.8GB Memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| Object Storage | n/a | n/a | n/a | n/a | n/a | +| NFS Server (optional, not recommended) | 1 | 4 vCPU, 3.6GB Memory | `n1-highcpu-4` | `c5.xlarge` | `F4s v2` | + +The architectures were built and tested with the [Intel Xeon E5 v3 (Haswell)](https://cloud.google.com/compute/docs/cpu-platforms) +CPU platform on GCP. On different hardware you may find that adjustments, either lower +or higher, are required for your CPU or Node counts accordingly. For more information, a +[Sysbench](https://github.com/akopytov/sysbench) benchmark of the CPU can be found +[here](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Reference-Architectures/GCP-CPU-Benchmarks). + +For data objects such as LFS, Uploads, Artifacts, etc, an [object storage service](#configure-the-object-storage) +is recommended over NFS where possible, due to better performance and availability. +Since this doesn't require a node to be set up, it's marked as not applicable (n/a) +in the table above. + +## Setup components + +To set up GitLab and its components to accommodate up to 3,000 users: + +1. [Configure the external load balancing node](#configure-the-external-load-balancer) + that will handle the load balancing of the two GitLab application services nodes. +1. [Configure Redis](#configure-redis). +1. [Configure Consul and Sentinel](#configure-consul-and-sentinel). +1. [Configure PostgreSQL](#configure-postgresql), the database for GitLab. +1. [Configure PgBouncer](#configure-pgbouncer). +1. [Configure the internal load balancing node](#configure-the-internal-load-balancer) +1. [Configure Gitaly](#configure-gitaly), + which provides access to the Git repositories. +1. [Configure Sidekiq](#configure-sidekiq). +1. [Configure the main GitLab Rails application](#configure-gitlab-rails) + to run Puma/Unicorn, Workhorse, GitLab Shell, and to serve all frontend requests (UI, API, Git + over HTTP/SSH). +1. [Configure Prometheus](#configure-prometheus) to monitor your GitLab environment. +1. [Configure the Object Storage](#configure-the-object-storage) + used for shared data objects. +1. [Configure NFS (Optional)](#configure-nfs-optional) + to have shared disk storage service as an alternative to Gitaly and/or Object Storage (although + not recommended). NFS is required for GitLab Pages, you can skip this step if you're not using + that feature. + +We start with all servers on the same 10.6.0.0/16 private network range, they +can connect to each other freely on those addresses. + +Here is a list and description of each machine and the assigned IP: + +- `10.6.0.10`: External Load Balancer +- `10.6.0.61`: Redis Primary +- `10.6.0.62`: Redis Replica 1 +- `10.6.0.63`: Redis Replica 2 +- `10.6.0.11`: Consul/Sentinel 1 +- `10.6.0.12`: Consul/Sentinel 2 +- `10.6.0.13`: Consul/Sentinel 3 +- `10.6.0.31`: PostgreSQL primary +- `10.6.0.32`: PostgreSQL secondary 1 +- `10.6.0.33`: PostgreSQL secondary 2 +- `10.6.0.21`: PgBouncer 1 +- `10.6.0.22`: PgBouncer 2 +- `10.6.0.23`: PgBouncer 3 +- `10.6.0.20`: Internal Load Balancer +- `10.6.0.51`: Gitaly 1 +- `10.6.0.52`: Gitaly 2 +- `10.6.0.71`: Sidekiq 1 +- `10.6.0.72`: Sidekiq 2 +- `10.6.0.73`: Sidekiq 3 +- `10.6.0.74`: Sidekiq 4 +- `10.6.0.41`: GitLab application 1 +- `10.6.0.42`: GitLab application 2 +- `10.6.0.43`: GitLab application 3 +- `10.6.0.81`: Prometheus + +## Configure the external load balancer + +NOTE: **Note:** +This architecture has been tested and validated with [HAProxy](https://www.haproxy.org/) +as the load balancer. Although other load balancers with similar feature sets +could also be used, those load balancers have not been validated. + +In an active/active GitLab configuration, you will need a load balancer to route +traffic to the application servers. The specifics on which load balancer to use +or the exact configuration is beyond the scope of GitLab documentation. We hope +that if you're managing multi-node systems like GitLab you have a load balancer of +choice already. Some examples including HAProxy (open-source), F5 Big-IP LTM, +and Citrix Net Scaler. This documentation will outline what ports and protocols +you need to use with GitLab. + +The next question is how you will handle SSL in your environment. +There are several different options: + +- [The application node terminates SSL](#application-node-terminates-ssl). +- [The load balancer terminates SSL without backend SSL](#load-balancer-terminates-ssl-without-backend-ssl) + and communication is not secure between the load balancer and the application node. +- [The load balancer terminates SSL with backend SSL](#load-balancer-terminates-ssl-with-backend-ssl) + and communication is *secure* between the load balancer and the application node. + +### Application node terminates SSL + +Configure your load balancer to pass connections on port 443 as `TCP` rather +than `HTTP(S)` protocol. This will pass the connection to the application node's +NGINX service untouched. NGINX will have the SSL certificate and listen on port 443. + +See the [NGINX HTTPS documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https) +for details on managing SSL certificates and configuring NGINX. + +### Load balancer terminates SSL without backend SSL + +Configure your load balancer to use the `HTTP(S)` protocol rather than `TCP`. +The load balancer will then be responsible for managing SSL certificates and +terminating SSL. + +Since communication between the load balancer and GitLab will not be secure, +there is some additional configuration needed. See the +[NGINX proxied SSL documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#supporting-proxied-ssl) +for details. + +### Load balancer terminates SSL with backend SSL + +Configure your load balancer(s) to use the 'HTTP(S)' protocol rather than 'TCP'. +The load balancer(s) will be responsible for managing SSL certificates that +end users will see. + +Traffic will also be secure between the load balancer(s) and NGINX in this +scenario. There is no need to add configuration for proxied SSL since the +connection will be secure all the way. However, configuration will need to be +added to GitLab to configure SSL certificates. See +[NGINX HTTPS documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https) +for details on managing SSL certificates and configuring NGINX. + +### Ports + +The basic ports to be used are shown in the table below. + +| LB Port | Backend Port | Protocol | +| ------- | ------------ | ------------------------ | +| 80 | 80 | HTTP (*1*) | +| 443 | 443 | TCP or HTTPS (*1*) (*2*) | +| 22 | 22 | TCP | + +- (*1*): [Web terminal](../../ci/environments/index.md#web-terminals) support requires + your load balancer to correctly handle WebSocket connections. When using + HTTP or HTTPS proxying, this means your load balancer must be configured + to pass through the `Connection` and `Upgrade` hop-by-hop headers. See the + [web terminal](../integration/terminal.md) integration guide for + more details. +- (*2*): When using HTTPS protocol for port 443, you will need to add an SSL + certificate to the load balancers. If you wish to terminate SSL at the + GitLab application server instead, use TCP protocol. + +If you're using GitLab Pages with custom domain support you will need some +additional port configurations. +GitLab Pages requires a separate virtual IP address. Configure DNS to point the +`pages_external_url` from `/etc/gitlab/gitlab.rb` at the new virtual IP address. See the +[GitLab Pages documentation](../pages/index.md) for more information. + +| LB Port | Backend Port | Protocol | +| ------- | ------------- | --------- | +| 80 | Varies (*1*) | HTTP | +| 443 | Varies (*1*) | TCP (*2*) | + +- (*1*): The backend port for GitLab Pages depends on the + `gitlab_pages['external_http']` and `gitlab_pages['external_https']` + setting. See [GitLab Pages documentation](../pages/index.md) for more details. +- (*2*): Port 443 for GitLab Pages should always use the TCP protocol. Users can + configure custom domains with custom SSL, which would not be possible + if SSL was terminated at the load balancer. + +#### Alternate SSH Port + +Some organizations have policies against opening SSH port 22. In this case, +it may be helpful to configure an alternate SSH hostname that allows users +to use SSH on port 443. An alternate SSH hostname will require a new virtual IP address +compared to the other GitLab HTTP configuration above. + +Configure DNS for an alternate SSH hostname such as `altssh.gitlab.example.com`. + +| LB Port | Backend Port | Protocol | +| ------- | ------------ | -------- | +| 443 | 22 | TCP | + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure Redis + +Using [Redis](https://redis.io/) in scalable environment is possible using a **Primary** x **Replica** +topology with a [Redis Sentinel](https://redis.io/topics/sentinel) service to watch and automatically +start the failover procedure. + +Redis requires authentication if used with Sentinel. See +[Redis Security](https://redis.io/topics/security) documentation for more +information. We recommend using a combination of a Redis password and tight +firewall rules to secure your Redis service. +You are highly encouraged to read the [Redis Sentinel](https://redis.io/topics/sentinel) documentation +before configuring Redis with GitLab to fully understand the topology and +architecture. + +In this section, you'll be guided through configuring an external Redis instance +to be used with GitLab. The following IPs will be used as an example: + +- `10.6.0.61`: Redis Primary +- `10.6.0.62`: Redis Replica 1 +- `10.6.0.63`: Redis Replica 2 + +### Provide your own Redis instance + +Managed Redis from cloud providers such as AWS ElastiCache will work. If these +services support high availability, be sure it is **not** the Redis Cluster type. + +Redis version 5.0 or higher is required, as this is what ships with +Omnibus GitLab packages starting with GitLab 13.0. Older Redis versions +do not support an optional count argument to SPOP which is now required for +[Merge Trains](../../ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md). + +Note the Redis node's IP address or hostname, port, and password (if required). +These will be necessary when configuring the +[GitLab application servers](#configure-gitlab-rails) later. + +### Standalone Redis using Omnibus GitLab + +This is the section where we install and set up the new Redis instances. + +The requirements for a Redis setup are the following: + +1. All Redis nodes must be able to talk to each other and accept incoming + connections over Redis (`6379`) and Sentinel (`26379`) ports (unless you + change the default ones). +1. The server that hosts the GitLab application must be able to access the + Redis nodes. +1. Protect the nodes from access from external networks + ([Internet](https://gitlab.com/gitlab-org/gitlab-foss/uploads/c4cc8cd353604bd80315f9384035ff9e/The_Internet_IT_Crowd.png)), + using a firewall. + +NOTE: **Note:** +Redis nodes (both primary and replica) will need the same password defined in +`redis['password']`. At any time during a failover the Sentinels can +reconfigure a node and change its status from primary to replica and vice versa. + +#### Configuring the primary Redis instance + +1. SSH into the **Primary** Redis server. +1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab + package you want using **steps 1 and 2** from the GitLab downloads page. + - Make sure you select the correct Omnibus package, with the same version + and type (Community, Enterprise editions) of your current install. + - Do not complete any other steps on the download page. + +1. Edit `/etc/gitlab/gitlab.rb` and add the contents: + + ```ruby + # Specify server role as 'redis_master_role' + roles ['redis_master_role'] + + # IP address pointing to a local IP that the other machines can reach to. + # You can also set bind to '0.0.0.0' which listen in all interfaces. + # If you really need to bind to an external accessible IP, make + # sure you add extra firewall rules to prevent unauthorized access. + redis['bind'] = '10.6.0.61' + + # Define a port so Redis can listen for TCP requests which will allow other + # machines to connect to it. + redis['port'] = 6379 + + # Set up password authentication for Redis (use the same password in all nodes). + redis['password'] = 'redis-password-goes-here' + + ## Enable service discovery for Prometheus + consul['enable'] = true + consul['monitoring_service_discovery'] = true + + ## The IPs of the Consul server nodes + ## You can also use FQDNs and intermix them with IPs + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), + } + + # Set the network addresses that the exporters will listen on + node_exporter['listen_address'] = '0.0.0.0:9100' + redis_exporter['listen_address'] = '0.0.0.0:9121' + redis_exporter['flags'] = { + 'redis.addr' => 'redis://10.6.0.61:6379', + 'redis.password' => 'redis-password-goes-here', + } + + # Disable auto migrations + gitlab_rails['auto_migrate'] = false + ``` + +1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + +NOTE: **Note:** +You can specify multiple roles like sentinel and Redis as: +`roles ['redis_sentinel_role', 'redis_master_role']`. +Read more about [roles](https://docs.gitlab.com/omnibus/roles/). + +You can list the current Redis Primary, Replica status via: + +```shell +/opt/gitlab/embedded/bin/redis-cli -h <host> -a 'redis-password-goes-here' info replication +``` + +Show running GitLab services via: + +```shell +gitlab-ctl status +``` + +The output should be similar to the following: + +```plaintext +run: consul: (pid 30043) 76863s; run: log: (pid 29691) 76892s +run: logrotate: (pid 31152) 3070s; run: log: (pid 29595) 76908s +run: node-exporter: (pid 30064) 76862s; run: log: (pid 29624) 76904s +run: redis: (pid 30070) 76861s; run: log: (pid 29573) 76914s +run: redis-exporter: (pid 30075) 76861s; run: log: (pid 29674) 76896s +``` + +#### Configuring the replica Redis instances + +1. SSH into the **replica** Redis server. +1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab + package you want using **steps 1 and 2** from the GitLab downloads page. + - Make sure you select the correct Omnibus package, with the same version + and type (Community, Enterprise editions) of your current install. + - Do not complete any other steps on the download page. + +1. Edit `/etc/gitlab/gitlab.rb` and add the contents: + + ```ruby + # Specify server role as 'redis_replica_role' + roles ['redis_replica_role'] + + # IP address pointing to a local IP that the other machines can reach to. + # You can also set bind to '0.0.0.0' which listen in all interfaces. + # If you really need to bind to an external accessible IP, make + # sure you add extra firewall rules to prevent unauthorized access. + redis['bind'] = '10.6.0.62' + + # Define a port so Redis can listen for TCP requests which will allow other + # machines to connect to it. + redis['port'] = 6379 + + # The same password for Redis authentication you set up for the primary node. + redis['password'] = 'redis-password-goes-here' + + # The IP of the primary Redis node. + redis['master_ip'] = '10.6.0.61' + + # Port of primary Redis server, uncomment to change to non default. Defaults + # to `6379`. + #redis['master_port'] = 6379 + + ## Enable service discovery for Prometheus + consul['enable'] = true + consul['monitoring_service_discovery'] = true + + ## The IPs of the Consul server nodes + ## You can also use FQDNs and intermix them with IPs + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), + } + + # Set the network addresses that the exporters will listen on + node_exporter['listen_address'] = '0.0.0.0:9100' + redis_exporter['listen_address'] = '0.0.0.0:9121' + redis_exporter['flags'] = { + 'redis.addr' => 'redis://10.6.0.62:6379', + 'redis.password' => 'redis-password-goes-here', + } + + # Disable auto migrations + gitlab_rails['auto_migrate'] = false + ``` + +1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. +1. Go through the steps again for all the other replica nodes, and + make sure to set up the IPs correctly. + +NOTE: **Note:** +You can specify multiple roles like sentinel and Redis as: +`roles ['redis_sentinel_role', 'redis_master_role']`. +Read more about [roles](https://docs.gitlab.com/omnibus/roles/). + +These values don't have to be changed again in `/etc/gitlab/gitlab.rb` after +a failover, as the nodes will be managed by the [Sentinels](#configure-consul-and-sentinel), and even after a +`gitlab-ctl reconfigure`, they will get their configuration restored by +the same Sentinels. + +Advanced [configuration options](https://docs.gitlab.com/omnibus/settings/redis.html) +are supported and can be added if needed. + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure Consul and Sentinel + +NOTE: **Note:** +If you are using an external Redis Sentinel instance, be sure +to exclude the `requirepass` parameter from the Sentinel +configuration. This parameter will cause clients to report `NOAUTH +Authentication required.`. [Redis Sentinel 3.2.x does not support +password authentication](https://github.com/antirez/redis/issues/3279). + +Now that the Redis servers are all set up, let's configure the Sentinel +servers. The following IPs will be used as an example: + +- `10.6.0.11`: Consul/Sentinel 1 +- `10.6.0.12`: Consul/Sentinel 2 +- `10.6.0.13`: Consul/Sentinel 3 + +To configure the Sentinel: + +1. SSH into the server that will host Consul/Sentinel. +1. [Download/install](https://about.gitlab.com/install/) the + Omnibus GitLab Enterprise Edition package using **steps 1 and 2** from the + GitLab downloads page. + - Make sure you select the correct Omnibus package, with the same version + the GitLab application is running. + - Do not complete any other steps on the download page. + +1. Edit `/etc/gitlab/gitlab.rb` and add the contents: + + ```ruby + roles ['redis_sentinel_role', 'consul_role'] + + # Must be the same in every sentinel node + redis['master_name'] = 'gitlab-redis' + + # The same password for Redis authentication you set up for the primary node. + redis['master_password'] = 'redis-password-goes-here' + + # The IP of the primary Redis node. + redis['master_ip'] = '10.6.0.61' + + # Define a port so Redis can listen for TCP requests which will allow other + # machines to connect to it. + redis['port'] = 6379 + + # Port of primary Redis server, uncomment to change to non default. Defaults + # to `6379`. + #redis['master_port'] = 6379 + + ## Configure Sentinel + sentinel['bind'] = '10.6.0.11' + + # Port that Sentinel listens on, uncomment to change to non default. Defaults + # to `26379`. + # sentinel['port'] = 26379 + + ## Quorum must reflect the amount of voting sentinels it take to start a failover. + ## Value must NOT be greater then the amount of sentinels. + ## + ## The quorum can be used to tune Sentinel in two ways: + ## 1. If a the quorum is set to a value smaller than the majority of Sentinels + ## we deploy, we are basically making Sentinel more sensible to primary failures, + ## triggering a failover as soon as even just a minority of Sentinels is no longer + ## able to talk with the primary. + ## 1. If a quorum is set to a value greater than the majority of Sentinels, we are + ## making Sentinel able to failover only when there are a very large number (larger + ## than majority) of well connected Sentinels which agree about the primary being down.s + sentinel['quorum'] = 2 + + ## Consider unresponsive server down after x amount of ms. + # sentinel['down_after_milliseconds'] = 10000 + + ## Specifies the failover timeout in milliseconds. It is used in many ways: + ## + ## - The time needed to re-start a failover after a previous failover was + ## already tried against the same primary by a given Sentinel, is two + ## times the failover timeout. + ## + ## - The time needed for a replica replicating to a wrong primary according + ## to a Sentinel current configuration, to be forced to replicate + ## with the right primary, is exactly the failover timeout (counting since + ## the moment a Sentinel detected the misconfiguration). + ## + ## - The time needed to cancel a failover that is already in progress but + ## did not produced any configuration change (REPLICAOF NO ONE yet not + ## acknowledged by the promoted replica). + ## + ## - The maximum time a failover in progress waits for all the replica to be + ## reconfigured as replicas of the new primary. However even after this time + ## the replicas will be reconfigured by the Sentinels anyway, but not with + ## the exact parallel-syncs progression as specified. + # sentinel['failover_timeout'] = 60000 + + ## Enable service discovery for Prometheus + consul['enable'] = true + consul['monitoring_service_discovery'] = true + + ## The IPs of the Consul server nodes + ## You can also use FQDNs and intermix them with IPs + consul['configuration'] = { + server: true, + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), + } + + # Set the network addresses that the exporters will listen on + node_exporter['listen_address'] = '0.0.0.0:9100' + redis_exporter['listen_address'] = '0.0.0.0:9121' + + # Disable auto migrations + gitlab_rails['auto_migrate'] = false + ``` + +1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. +1. Go through the steps again for all the other Consul/Sentinel nodes, and + make sure you set up the correct IPs. + +NOTE: **Note:** +A Consul leader will be elected when the provisioning of the third Consul server is completed. +Viewing the Consul logs `sudo gitlab-ctl tail consul` will display +`...[INFO] consul: New leader elected: ...` + +You can list the current Consul members (server, client): + +```shell +sudo /opt/gitlab/embedded/bin/consul members +``` + +You can verify the GitLab services are running: + +```shell +sudo gitlab-ctl status +``` + +The output should be similar to the following: + +```plaintext +run: consul: (pid 30074) 76834s; run: log: (pid 29740) 76844s +run: logrotate: (pid 30925) 3041s; run: log: (pid 29649) 76861s +run: node-exporter: (pid 30093) 76833s; run: log: (pid 29663) 76855s +run: sentinel: (pid 30098) 76832s; run: log: (pid 29704) 76850s +``` + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure PostgreSQL + +In this section, you'll be guided through configuring an external PostgreSQL database +to be used with GitLab. + +### Provide your own PostgreSQL instance + +If you're hosting GitLab on a cloud provider, you can optionally use a +managed service for PostgreSQL. For example, AWS offers a managed Relational +Database Service (RDS) that runs PostgreSQL. + +If you use a cloud-managed service, or provide your own PostgreSQL: + +1. Set up PostgreSQL according to the + [database requirements document](../../install/requirements.md#database). +1. Set up a `gitlab` username with a password of your choice. The `gitlab` user + needs privileges to create the `gitlabhq_production` database. +1. Configure the GitLab application servers with the appropriate details. + This step is covered in [Configuring the GitLab Rails application](#configure-gitlab-rails). + +### Standalone PostgreSQL using Omnibus GitLab + +The following IPs will be used as an example: + +- `10.6.0.31`: PostgreSQL primary +- `10.6.0.32`: PostgreSQL secondary 1 +- `10.6.0.33`: PostgreSQL secondary 2 + +First, make sure to [install](https://about.gitlab.com/install/) +the Linux GitLab package **on each node**. Following the steps, +install the necessary dependencies from step 1, and add the +GitLab package repository from step 2. When installing GitLab +in the second step, do not supply the `EXTERNAL_URL` value. + +#### PostgreSQL primary node + +1. SSH into the PostgreSQL primary node. +1. Generate a password hash for the PostgreSQL username/password pair. This assumes you will use the default + username of `gitlab` (recommended). The command will request a password + and confirmation. Use the value that is output by this command in the next + step as the value of `<postgresql_password_hash>`: + + ```shell + sudo gitlab-ctl pg-password-md5 gitlab + ``` + +1. Generate a password hash for the PgBouncer username/password pair. This assumes you will use the default + username of `pgbouncer` (recommended). The command will request a password + and confirmation. Use the value that is output by this command in the next + step as the value of `<pgbouncer_password_hash>`: + + ```shell + sudo gitlab-ctl pg-password-md5 pgbouncer + ``` + +1. Generate a password hash for the Consul database username/password pair. This assumes you will use the default + username of `gitlab-consul` (recommended). The command will request a password + and confirmation. Use the value that is output by this command in the next + step as the value of `<consul_password_hash>`: + + ```shell + sudo gitlab-ctl pg-password-md5 gitlab-consul + ``` + +1. On the primary database node, edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: + + ```ruby + # Disable all components except PostgreSQL and Repmgr and Consul + roles ['postgres_role'] + + # PostgreSQL configuration + postgresql['listen_address'] = '0.0.0.0' + postgresql['hot_standby'] = 'on' + postgresql['wal_level'] = 'replica' + postgresql['shared_preload_libraries'] = 'repmgr_funcs' + + # Disable automatic database migrations + gitlab_rails['auto_migrate'] = false + + # Configure the Consul agent + consul['services'] = %w(postgresql) + + # START user configuration + # Please set the real values as explained in Required Information section + # + # Replace PGBOUNCER_PASSWORD_HASH with a generated md5 value + postgresql['pgbouncer_user_password'] = '<pgbouncer_password_hash>' + # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value + postgresql['sql_user_password'] = '<postgresql_password_hash>' + # Set `max_wal_senders` to one more than the number of database nodes in the cluster. + # This is used to prevent replication from using up all of the + # available database connections. + postgresql['max_wal_senders'] = 4 + postgresql['max_replication_slots'] = 4 + + # Replace XXX.XXX.XXX.XXX/YY with Network Address + postgresql['trust_auth_cidr_addresses'] = %w(127.0.0.1/32 10.6.0.0/24) + repmgr['trust_auth_cidr_addresses'] = %w(127.0.0.1/32 10.6.0.0/24) + + ## Enable service discovery for Prometheus + consul['enable'] = true + consul['monitoring_service_discovery'] = true + + # Set the network addresses that the exporters will listen on for monitoring + node_exporter['listen_address'] = '0.0.0.0:9100' + postgres_exporter['listen_address'] = '0.0.0.0:9187' + postgres_exporter['dbname'] = 'gitlabhq_production' + postgres_exporter['password'] = '<postgresql_password_hash>' + + ## The IPs of the Consul server nodes + ## You can also use FQDNs and intermix them with IPs + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), + } + # + # END user configuration + ``` + +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. +1. You can list the current PostgreSQL primary, secondary nodes status via: + + ```shell + sudo /opt/gitlab/bin/gitlab-ctl repmgr cluster show + ``` + +1. Verify the GitLab services are running: + + ```shell + sudo gitlab-ctl status + ``` + + The output should be similar to the following: + + ```plaintext + run: consul: (pid 30593) 77133s; run: log: (pid 29912) 77156s + run: logrotate: (pid 23449) 3341s; run: log: (pid 29794) 77175s + run: node-exporter: (pid 30613) 77133s; run: log: (pid 29824) 77170s + run: postgres-exporter: (pid 30620) 77132s; run: log: (pid 29894) 77163s + run: postgresql: (pid 30630) 77132s; run: log: (pid 29618) 77181s + run: repmgrd: (pid 30639) 77132s; run: log: (pid 29985) 77150s + ``` + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +#### PostgreSQL secondary nodes + +1. On both the secondary nodes, add the same configuration specified above for the primary node + with an additional setting that will inform `gitlab-ctl` that they are standby nodes initially + and there's no need to attempt to register them as a primary node: + + ```ruby + # Disable all components except PostgreSQL and Repmgr and Consul + roles ['postgres_role'] + + # PostgreSQL configuration + postgresql['listen_address'] = '0.0.0.0' + postgresql['hot_standby'] = 'on' + postgresql['wal_level'] = 'replica' + postgresql['shared_preload_libraries'] = 'repmgr_funcs' + + # Disable automatic database migrations + gitlab_rails['auto_migrate'] = false + + # Configure the Consul agent + consul['services'] = %w(postgresql) + + # Specify if a node should attempt to be primary on initialization. + repmgr['master_on_initialization'] = false + + # START user configuration + # Please set the real values as explained in Required Information section + # + # Replace PGBOUNCER_PASSWORD_HASH with a generated md5 value + postgresql['pgbouncer_user_password'] = '<pgbouncer_password_hash>' + # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value + postgresql['sql_user_password'] = '<postgresql_password_hash>' + # Set `max_wal_senders` to one more than the number of database nodes in the cluster. + # This is used to prevent replication from using up all of the + # available database connections. + postgresql['max_wal_senders'] = 4 + postgresql['max_replication_slots'] = 4 + + # Replace XXX.XXX.XXX.XXX/YY with Network Address + postgresql['trust_auth_cidr_addresses'] = %w(127.0.0.1/32 10.6.0.0/24) + repmgr['trust_auth_cidr_addresses'] = %w(127.0.0.1/32 10.6.0.0/24) + + ## Enable service discovery for Prometheus + consul['enable'] = true + consul['monitoring_service_discovery'] = true + + # Set the network addresses that the exporters will listen on for monitoring + node_exporter['listen_address'] = '0.0.0.0:9100' + postgres_exporter['listen_address'] = '0.0.0.0:9187' + postgres_exporter['dbname'] = 'gitlabhq_production' + postgres_exporter['password'] = '<postgresql_password_hash>' + + ## The IPs of the Consul server nodes + ## You can also use FQDNs and intermix them with IPs + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), + } + # END user configuration + ``` + +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + +Advanced [configuration options](https://docs.gitlab.com/omnibus/settings/database.html) +are supported and can be added if needed. + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +#### PostgreSQL post-configuration + +SSH into the **primary node**: + +1. Open a database prompt: + + ```shell + gitlab-psql -d gitlabhq_production + ``` + +1. Enable the `pg_trgm` extension: + + ```shell + CREATE EXTENSION pg_trgm; + ``` + +1. Exit the database prompt by typing `\q` and Enter. + +1. Verify the cluster is initialized with one node: + + ```shell + gitlab-ctl repmgr cluster show + ``` + + The output should be similar to the following: + + ```plaintext + Role | Name | Upstream | Connection String + ----------+----------|----------|---------------------------------------- + * master | HOSTNAME | | host=HOSTNAME user=gitlab_repmgr dbname=gitlab_repmgr + ``` + +1. Note down the hostname or IP address in the connection string: `host=HOSTNAME`. We will + refer to the hostname in the next section as `<primary_node_name>`. If the value + is not an IP address, it will need to be a resolvable name (via DNS or + `/etc/hosts`) + +SSH into the **secondary node**: + +1. Set up the repmgr standby: + + ```shell + gitlab-ctl repmgr standby setup <primary_node_name> + ``` + + Do note that this will remove the existing data on the node. The command + has a wait time. + + The output should be similar to the following: + + ```console + Doing this will delete the entire contents of /var/opt/gitlab/postgresql/data + If this is not what you want, hit Ctrl-C now to exit + To skip waiting, rerun with the -w option + Sleeping for 30 seconds + Stopping the database + Removing the data + Cloning the data + Starting the database + Registering the node with the cluster + ok: run: repmgrd: (pid 19068) 0s + ``` + +Before moving on, make sure the databases are configured correctly. Run the +following command on the **primary** node to verify that replication is working +properly and the secondary nodes appear in the cluster: + +```shell +gitlab-ctl repmgr cluster show +``` + +The output should be similar to the following: + +```plaintext +Role | Name | Upstream | Connection String +----------+---------|-----------|------------------------------------------------ +* master | MASTER | | host=<primary_node_name> user=gitlab_repmgr dbname=gitlab_repmgr + standby | STANDBY | MASTER | host=<secondary_node_name> user=gitlab_repmgr dbname=gitlab_repmgr + standby | STANDBY | MASTER | host=<secondary_node_name> user=gitlab_repmgr dbname=gitlab_repmgr +``` + +If the 'Role' column for any node says "FAILED", check the +[Troubleshooting section](troubleshooting.md) before proceeding. + +Also, check that the `repmgr-check-master` command works successfully on each node: + +```shell +su - gitlab-consul +gitlab-ctl repmgr-check-master || echo 'This node is a standby repmgr node' +``` + +This command relies on exit codes to tell Consul whether a particular node is a master +or secondary. The most important thing here is that this command does not produce errors. +If there are errors it's most likely due to incorrect `gitlab-consul` database user permissions. +Check the [Troubleshooting section](troubleshooting.md) before proceeding. + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure PgBouncer + +Now that the PostgreSQL servers are all set up, let's configure PgBouncer. +The following IPs will be used as an example: + +- `10.6.0.21`: PgBouncer 1 +- `10.6.0.22`: PgBouncer 2 +- `10.6.0.23`: PgBouncer 3 + +1. On each PgBouncer node, edit `/etc/gitlab/gitlab.rb`, and replace + `<consul_password_hash>` and `<pgbouncer_password_hash>` with the + password hashes you [set up previously](#postgresql-primary-node): + + ```ruby + # Disable all components except Pgbouncer and Consul agent + roles ['pgbouncer_role'] + + # Configure PgBouncer + pgbouncer['admin_users'] = %w(pgbouncer gitlab-consul) + + pgbouncer['users'] = { + 'gitlab-consul': { + password: '<consul_password_hash>' + }, + 'pgbouncer': { + password: '<pgbouncer_password_hash>' + } + } + + # Configure Consul agent + consul['watchers'] = %w(postgresql) + consul['enable'] = true + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13) + } + + # Enable service discovery for Prometheus + consul['monitoring_service_discovery'] = true + + # Set the network addresses that the exporters will listen on + node_exporter['listen_address'] = '0.0.0.0:9100' + pgbouncer_exporter['listen_address'] = '0.0.0.0:9188' + ``` + +1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + +1. Create a `.pgpass` file so Consul is able to + reload PgBouncer. Enter the PgBouncer password twice when asked: + + ```shell + gitlab-ctl write-pgpass --host 127.0.0.1 --database pgbouncer --user pgbouncer --hostuser gitlab-consul + ``` + +1. Ensure each node is talking to the current master: + + ```shell + gitlab-ctl pgb-console # You will be prompted for PGBOUNCER_PASSWORD + ``` + + If there is an error `psql: ERROR: Auth failed` after typing in the + password, ensure you previously generated the MD5 password hashes with the correct + format. The correct format is to concatenate the password and the username: + `PASSWORDUSERNAME`. For example, `Sup3rS3cr3tpgbouncer` would be the text + needed to generate an MD5 password hash for the `pgbouncer` user. + +1. Once the console prompt is available, run the following queries: + + ```shell + show databases ; show clients ; + ``` + + The output should be similar to the following: + + ```plaintext + name | host | port | database | force_user | pool_size | reserve_pool | pool_mode | max_connections | current_connections + ---------------------+-------------+------+---------------------+------------+-----------+--------------+-----------+-----------------+--------------------- + gitlabhq_production | MASTER_HOST | 5432 | gitlabhq_production | | 20 | 0 | | 0 | 0 + pgbouncer | | 6432 | pgbouncer | pgbouncer | 2 | 0 | statement | 0 | 0 + (2 rows) + + type | user | database | state | addr | port | local_addr | local_port | connect_time | request_time | ptr | link | remote_pid | tls + ------+-----------+---------------------+---------+----------------+-------+------------+------------+---------------------+---------------------+-----------+------+------------+----- + C | pgbouncer | pgbouncer | active | 127.0.0.1 | 56846 | 127.0.0.1 | 6432 | 2017-08-21 18:09:59 | 2017-08-21 18:10:48 | 0x22b3880 | | 0 | + (2 rows) + ``` + +1. Verify the GitLab services are running: + + ```shell + sudo gitlab-ctl status + ``` + + The output should be similar to the following: + + ```plaintext + run: consul: (pid 31530) 77150s; run: log: (pid 31106) 77182s + run: logrotate: (pid 32613) 3357s; run: log: (pid 30107) 77500s + run: node-exporter: (pid 31550) 77149s; run: log: (pid 30138) 77493s + run: pgbouncer: (pid 32033) 75593s; run: log: (pid 31117) 77175s + run: pgbouncer-exporter: (pid 31558) 77148s; run: log: (pid 31498) 77156s + ``` + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +### Configure the internal load balancer + +If you're running more than one PgBouncer node as recommended, then at this time you'll need to set +up a TCP internal load balancer to serve each correctly. + +The following IP will be used as an example: + +- `10.6.0.20`: Internal Load Balancer + +Here's how you could do it with [HAProxy](https://www.haproxy.org/): + +```plaintext +global + log /dev/log local0 + log localhost local1 notice + log stdout format raw local0 + +defaults + log global + default-server inter 10s fall 3 rise 2 + balance leastconn + +frontend internal-pgbouncer-tcp-in + bind *:6432 + mode tcp + option tcplog + + default_backend pgbouncer + +backend pgbouncer + mode tcp + option tcp-check + + server pgbouncer1 10.6.0.21:6432 check + server pgbouncer2 10.6.0.22:6432 check + server pgbouncer3 10.6.0.23:6432 check +``` + +Refer to your preferred Load Balancer's documentation for further guidance. + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure Gitaly + +Deploying Gitaly in its own server can benefit GitLab installations that are +larger than a single machine. + +The Gitaly node requirements are dependent on customer data, specifically the number of +projects and their repository sizes. Two nodes are recommended as an absolute minimum. +Each Gitaly node should store no more than 5TB of data and have the number of +[`gitaly-ruby` workers](../gitaly/index.md#gitaly-ruby) set to 20% of available CPUs. +Additional nodes should be considered in conjunction with a review of expected +data size and spread based on the recommendations above. + +It is also strongly recommended that all Gitaly nodes be set up with SSD disks with +a throughput of at least 8,000 IOPS for read operations and 2,000 IOPS for write, +as Gitaly has heavy I/O. These IOPS values are recommended only as a starter as with +time they may be adjusted higher or lower depending on the scale of your environment's workload. +If you're running the environment on a Cloud provider, you may need to refer to +their documentation on how to configure IOPS correctly. + +Some things to note: + +- The GitLab Rails application shards repositories into [repository storages](../repository_storage_paths.md). +- A Gitaly server can host one or more storages. +- A GitLab server can use one or more Gitaly servers. +- Gitaly addresses must be specified in such a way that they resolve + correctly for ALL Gitaly clients. +- Gitaly servers must not be exposed to the public internet, as Gitaly's network + traffic is unencrypted by default. The use of a firewall is highly recommended + to restrict access to the Gitaly server. Another option is to + [use TLS](#gitaly-tls-support). + +TIP: **Tip:** +For more information about Gitaly's history and network architecture see the +[standalone Gitaly documentation](../gitaly/index.md). + +Note: **Note:** The token referred to throughout the Gitaly documentation is +just an arbitrary password selected by the administrator. It is unrelated to +tokens created for the GitLab API or other similar web API tokens. + +Below we describe how to configure two Gitaly servers, with IPs and +domain names: + +- `10.6.0.51`: Gitaly 1 (`gitaly1.internal`) +- `10.6.0.52`: Gitaly 2 (`gitaly2.internal`) + +The secret token is assumed to be `gitalysecret` and that +your GitLab installation has three repository storages: + +- `default` on Gitaly 1 +- `storage1` on Gitaly 1 +- `storage2` on Gitaly 2 + +On each node: + +1. [Download/Install](https://about.gitlab.com/install/) the Omnibus GitLab + package you want using **steps 1 and 2** from the GitLab downloads page but + **without** providing the `EXTERNAL_URL` value. +1. Edit `/etc/gitlab/gitlab.rb` to configure storage paths, enable + the network listener and configure the token: + + <!-- + updates to following example must also be made at + https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab + --> + + ```ruby + # /etc/gitlab/gitlab.rb + + # Gitaly and GitLab use two shared secrets for authentication, one to authenticate gRPC requests + # to Gitaly, and a second for authentication callbacks from GitLab-Shell to the GitLab internal API. + # The following two values must be the same as their respective values + # of the GitLab Rails application setup + gitaly['auth_token'] = 'gitlaysecret' + gitlab_shell['secret_token'] = 'shellsecret' + + # Avoid running unnecessary services on the Gitaly server + postgresql['enable'] = false + redis['enable'] = false + nginx['enable'] = false + puma['enable'] = false + unicorn['enable'] = false + sidekiq['enable'] = false + gitlab_workhorse['enable'] = false + grafana['enable'] = false + gitlab_exporter['enable'] = false + + # If you run a seperate monitoring node you can disable these services + alertmanager['enable'] = false + prometheus['enable'] = false + + # Prevent database connections during 'gitlab-ctl reconfigure' + gitlab_rails['rake_cache_clear'] = false + gitlab_rails['auto_migrate'] = false + + # Configure the gitlab-shell API callback URL. Without this, `git push` will + # fail. This can be your 'front door' GitLab URL or an internal load + # balancer. + # Don't forget to copy `/etc/gitlab/gitlab-secrets.json` from web server to Gitaly server. + gitlab_rails['internal_api_url'] = 'https://gitlab.example.com' + + # Make Gitaly accept connections on all network interfaces. You must use + # firewalls to restrict access to this address/port. + # Comment out following line if you only want to support TLS connections + gitaly['listen_addr'] = "0.0.0.0:8075" + + ## Enable service discovery for Prometheus + consul['enable'] = true + consul['monitoring_service_discovery'] = true + + # Set the network addresses that the exporters will listen on for monitoring + gitaly['prometheus_listen_addr'] = "0.0.0.0:9236" + node_exporter['listen_address'] = '0.0.0.0:9100' + gitlab_rails['prometheus_address'] = '10.6.0.81:9090' + + ## The IPs of the Consul server nodes + ## You can also use FQDNs and intermix them with IPs + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), + } + ``` + +1. Append the following to `/etc/gitlab/gitlab.rb` for each respective server: + 1. On `gitaly1.internal`: + + ```ruby + git_data_dirs({ + 'default' => { + 'path' => '/var/opt/gitlab/git-data' + }, + 'storage1' => { + 'path' => '/mnt/gitlab/git-data' + }, + }) + ``` + + 1. On `gitaly2.internal`: + + ```ruby + git_data_dirs({ + 'storage2' => { + 'path' => '/mnt/gitlab/git-data' + }, + }) + ``` + + <!-- + updates to following example must also be made at + https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab + --> + +1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). +1. Confirm that Gitaly can perform callbacks to the internal API: + + ```shell + sudo /opt/gitlab/embedded/service/gitlab-shell/bin/check -config /opt/gitlab/embedded/service/gitlab-shell/config.yml + ``` + +1. Verify the GitLab services are running: + + ```shell + sudo gitlab-ctl status + ``` + + The output should be similar to the following: + + ```plaintext + run: consul: (pid 30339) 77006s; run: log: (pid 29878) 77020s + run: gitaly: (pid 30351) 77005s; run: log: (pid 29660) 77040s + run: logrotate: (pid 7760) 3213s; run: log: (pid 29782) 77032s + run: node-exporter: (pid 30378) 77004s; run: log: (pid 29812) 77026s + ``` + +### Gitaly TLS support + +Gitaly supports TLS encryption. To be able to communicate +with a Gitaly instance that listens for secure connections you will need to use `tls://` URL +scheme in the `gitaly_address` of the corresponding storage entry in the GitLab configuration. + +You will need to bring your own certificates as this isn't provided automatically. +The certificate, or its certificate authority, must be installed on all Gitaly +nodes (including the Gitaly node using the certificate) and on all client nodes +that communicate with it following the procedure described in +[GitLab custom certificate configuration](https://docs.gitlab.com/omnibus/settings/ssl.html#install-custom-public-certificates). + +NOTE: **Note:** +The self-signed certificate must specify the address you use to access the +Gitaly server. If you are addressing the Gitaly server by a hostname, you can +either use the Common Name field for this, or add it as a Subject Alternative +Name. If you are addressing the Gitaly server by its IP address, you must add it +as a Subject Alternative Name to the certificate. +[gRPC does not support using an IP address as Common Name in a certificate](https://github.com/grpc/grpc/issues/2691). + +NOTE: **Note:** +It is possible to configure Gitaly servers with both an +unencrypted listening address `listen_addr` and an encrypted listening +address `tls_listen_addr` at the same time. This allows you to do a +gradual transition from unencrypted to encrypted traffic, if necessary. + +To configure Gitaly with TLS: + +1. Create the `/etc/gitlab/ssl` directory and copy your key and certificate there: + + ```shell + sudo mkdir -p /etc/gitlab/ssl + sudo chmod 755 /etc/gitlab/ssl + sudo cp key.pem cert.pem /etc/gitlab/ssl/ + sudo chmod 644 key.pem cert.pem + ``` + +1. Copy the cert to `/etc/gitlab/trusted-certs` so Gitaly will trust the cert when + calling into itself: + + ```shell + sudo cp /etc/gitlab/ssl/cert.pem /etc/gitlab/trusted-certs/ + ``` + +1. Edit `/etc/gitlab/gitlab.rb` and add: + + <!-- + updates to following example must also be made at + https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab + --> + + ```ruby + gitaly['tls_listen_addr'] = "0.0.0.0:9999" + gitaly['certificate_path'] = "/etc/gitlab/ssl/cert.pem" + gitaly['key_path'] = "/etc/gitlab/ssl/key.pem" + ``` + +1. Delete `gitaly['listen_addr']` to allow only encrypted connections. +1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure Sidekiq + +Sidekiq requires connection to the Redis, PostgreSQL and Gitaly instance. +The following IPs will be used as an example: + +- `10.6.0.71`: Sidekiq 1 +- `10.6.0.72`: Sidekiq 2 +- `10.6.0.73`: Sidekiq 3 +- `10.6.0.74`: Sidekiq 4 + +To configure the Sidekiq nodes, one each one: + +1. SSH into the Sidekiq server. +1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab package +you want using steps 1 and 2 from the GitLab downloads page. +**Do not complete any other steps on the download page.** +1. Open `/etc/gitlab/gitlab.rb` with your editor: + + ```ruby + ######################################## + ##### Services Disabled ### + ######################################## + + nginx['enable'] = false + grafana['enable'] = false + prometheus['enable'] = false + gitlab_rails['auto_migrate'] = false + alertmanager['enable'] = false + gitaly['enable'] = false + gitlab_workhorse['enable'] = false + nginx['enable'] = false + puma['enable'] = false + postgres_exporter['enable'] = false + postgresql['enable'] = false + redis['enable'] = false + redis_exporter['enable'] = false + gitlab_exporter['enable'] = false + + ######################################## + #### Redis ### + ######################################## + + ## Must be the same in every sentinel node + redis['master_name'] = 'gitlab-redis' + + ## The same password for Redis authentication you set up for the master node. + redis['master_password'] = '<redis_primary_password>' + + ## A list of sentinels with `host` and `port` + gitlab_rails['redis_sentinels'] = [ + {'host' => '10.6.0.11', 'port' => 26379}, + {'host' => '10.6.0.12', 'port' => 26379}, + {'host' => '10.6.0.13', 'port' => 26379}, + ] + + ####################################### + ### Gitaly ### + ####################################### + + git_data_dirs({ + 'default' => { 'gitaly_address' => 'tcp://gitaly1.internal:8075' }, + 'storage1' => { 'gitaly_address' => 'tcp://gitaly1.internal:8075' }, + 'storage2' => { 'gitaly_address' => 'tcp://gitaly2.internal:8075' }, + }) + gitlab_rails['gitaly_token'] = 'YOUR_TOKEN' + + ####################################### + ### Postgres ### + ####################################### + gitlab_rails['db_host'] = '10.6.0.20' # internal load balancer IP + gitlab_rails['db_port'] = 6432 + gitlab_rails['db_password'] = '<postgresql_user_password>' + gitlab_rails['db_adapter'] = 'postgresql' + gitlab_rails['db_encoding'] = 'unicode' + gitlab_rails['auto_migrate'] = false + + ####################################### + ### Sidekiq configuration ### + ####################################### + sidekiq['listen_address'] = "0.0.0.0" + + ####################################### + ### Monitoring configuration ### + ####################################### + consul['enable'] = true + consul['monitoring_service_discovery'] = true + + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13) + } + + # Set the network addresses that the exporters will listen on + node_exporter['listen_address'] = '0.0.0.0:9100' + + # Rails Status for prometheus + gitlab_rails['monitoring_whitelist'] = ['10.6.0.81/32', '127.0.0.0/8'] + gitlab_rails['prometheus_address'] = '10.6.0.81:9090' + ``` + +1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). +1. Verify the GitLab services are running: + + ```shell + sudo gitlab-ctl status + ``` + + The output should be similar to the following: + + ```plaintext + run: consul: (pid 30114) 77353s; run: log: (pid 29756) 77367s + run: logrotate: (pid 9898) 3561s; run: log: (pid 29653) 77380s + run: node-exporter: (pid 30134) 77353s; run: log: (pid 29706) 77372s + run: sidekiq: (pid 30142) 77351s; run: log: (pid 29638) 77386s + ``` + +TIP: **Tip:** +You can also run [multiple Sidekiq processes](../operations/extra_sidekiq_processes.md). + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure GitLab Rails + +NOTE: **Note:** +In our architectures we run each GitLab Rails node using the Puma webserver +and have its number of workers set to 90% of available CPUs along with four threads. For +nodes that are running Rails with other components the worker value should be reduced +accordingly where we've found 50% achieves a good balance but this is dependent +on workload. + +This section describes how to configure the GitLab application (Rails) component. +On each node perform the following: + +1. If you're [using NFS](#configure-nfs-optional): + + 1. If necessary, install the NFS client utility packages using the following + commands: + + ```shell + # Ubuntu/Debian + apt-get install nfs-common + + # CentOS/Red Hat + yum install nfs-utils nfs-utils-lib + ``` + + 1. Specify the necessary NFS mounts in `/etc/fstab`. + The exact contents of `/etc/fstab` will depend on how you chose + to configure your NFS server. See the [NFS documentation](../high_availability/nfs.md) + for examples and the various options. + + 1. Create the shared directories. These may be different depending on your NFS + mount locations. + + ```shell + mkdir -p /var/opt/gitlab/.ssh /var/opt/gitlab/gitlab-rails/uploads /var/opt/gitlab/gitlab-rails/shared /var/opt/gitlab/gitlab-ci/builds /var/opt/gitlab/git-data + ``` + +1. Download/install Omnibus GitLab using **steps 1 and 2** from + [GitLab downloads](https://about.gitlab.com/install/). Do not complete other + steps on the download page. +1. Create/edit `/etc/gitlab/gitlab.rb` and use the following configuration. + To maintain uniformity of links across nodes, the `external_url` + on the application server should point to the external URL that users will use + to access GitLab. This would be the URL of the [external load balancer](#configure-the-external-load-balancer) + which will route traffic to the GitLab application server: + + ```ruby + external_url 'https://gitlab.example.com' + + # Gitaly and GitLab use two shared secrets for authentication, one to authenticate gRPC requests + # to Gitaly, and a second for authentication callbacks from GitLab-Shell to the GitLab internal API. + # The following two values must be the same as their respective values + # of the Gitaly setup + gitlab_rails['gitaly_token'] = 'gitalyecret' + gitlab_shell['secret_token'] = 'shellsecret' + + git_data_dirs({ + 'default' => { 'gitaly_address' => 'tcp://gitaly1.internal:8075' }, + 'storage1' => { 'gitaly_address' => 'tcp://gitaly1.internal:8075' }, + 'storage2' => { 'gitaly_address' => 'tcp://gitaly2.internal:8075' }, + }) + + ## Disable components that will not be on the GitLab application server + roles ['application_role'] + gitaly['enable'] = false + nginx['enable'] = true + sidekiq['enable'] = false + + ## PostgreSQL connection details + # Disable PostgreSQL on the application node + postgresql['enable'] = false + gitlab_rails['db_host'] = '10.6.0.20' # internal load balancer IP + gitlab_rails['db_port'] = 6432 + gitlab_rails['db_password'] = '<postgresql_user_password>' + gitlab_rails['auto_migrate'] = false + + ## Redis connection details + ## Must be the same in every sentinel node + redis['master_name'] = 'gitlab-redis' + + ## The same password for Redis authentication you set up for the Redis primary node. + redis['master_password'] = '<redis_primary_password>' + + ## A list of sentinels with `host` and `port` + gitlab_rails['redis_sentinels'] = [ + {'host' => '10.6.0.11', 'port' => 26379}, + {'host' => '10.6.0.12', 'port' => 26379}, + {'host' => '10.6.0.13', 'port' => 26379} + ] + + ## Enable service discovery for Prometheus + consul['enable'] = true + consul['monitoring_service_discovery'] = true + + # Set the network addresses that the exporters used for monitoring will listen on + node_exporter['listen_address'] = '0.0.0.0:9100' + gitlab_workhorse['prometheus_listen_addr'] = '0.0.0.0:9229' + sidekiq['listen_address'] = "0.0.0.0" + puma['listen'] = '0.0.0.0' + + ## The IPs of the Consul server nodes + ## You can also use FQDNs and intermix them with IPs + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), + } + + # Add the monitoring node's IP address to the monitoring whitelist and allow it to + # scrape the NGINX metrics + gitlab_rails['monitoring_whitelist'] = ['10.6.0.81/32', '127.0.0.0/8'] + nginx['status']['options']['allow'] = ['10.6.0.81/32', '127.0.0.0/8'] + gitlab_rails['prometheus_address'] = '10.6.0.81:9090' + + ## Uncomment and edit the following options if you have set up NFS + ## + ## Prevent GitLab from starting if NFS data mounts are not available + ## + #high_availability['mountpoint'] = '/var/opt/gitlab/git-data' + ## + ## Ensure UIDs and GIDs match between servers for permissions via NFS + ## + #user['uid'] = 9000 + #user['gid'] = 9000 + #web_server['uid'] = 9001 + #web_server['gid'] = 9001 + #registry['uid'] = 9002 + #registry['gid'] = 9002 + ``` + +1. If you're using [Gitaly with TLS support](#gitaly-tls-support), make sure the + `git_data_dirs` entry is configured with `tls` instead of `tcp`: + + ```ruby + git_data_dirs({ + 'default' => { 'gitaly_address' => 'tls://gitaly1.internal:9999' }, + 'storage1' => { 'gitaly_address' => 'tls://gitaly1.internal:9999' }, + 'storage2' => { 'gitaly_address' => 'tls://gitaly2.internal:9999' }, + }) + ``` + + 1. Copy the cert into `/etc/gitlab/trusted-certs`: + + ```shell + sudo cp cert.pem /etc/gitlab/trusted-certs/ + ``` + +1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). +1. Run `sudo gitlab-rake gitlab:gitaly:check` to confirm the node can connect to Gitaly. +1. Tail the logs to see the requests: + + ```shell + sudo gitlab-ctl tail gitaly + ``` + +1. Verify the GitLab services are running: + + ```shell + sudo gitlab-ctl status + ``` + + The output should be similar to the following: + + ```plaintext + run: consul: (pid 4890) 8647s; run: log: (pid 29962) 79128s + run: gitlab-exporter: (pid 4902) 8647s; run: log: (pid 29913) 79134s + run: gitlab-workhorse: (pid 4904) 8646s; run: log: (pid 29713) 79155s + run: logrotate: (pid 12425) 1446s; run: log: (pid 29798) 79146s + run: nginx: (pid 4925) 8646s; run: log: (pid 29726) 79152s + run: node-exporter: (pid 4931) 8645s; run: log: (pid 29855) 79140s + run: puma: (pid 4936) 8645s; run: log: (pid 29656) 79161s + ``` + +NOTE: **Note:** +When you specify `https` in the `external_url`, as in the example +above, GitLab assumes you have SSL certificates in `/etc/gitlab/ssl/`. If +certificates are not present, NGINX will fail to start. See the +[NGINX documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https) +for more information. + +### GitLab Rails post-configuration + +1. Ensure that all migrations ran: + + ```shell + gitlab-rake gitlab:db:configure + ``` + + NOTE: **Note:** + If you encounter a `rake aborted!` error stating that PgBouncer is failing to connect to + PostgreSQL it may be that your PgBouncer node's IP address is missing from + PostgreSQL's `trust_auth_cidr_addresses` in `gitlab.rb` on your database nodes. See + [PgBouncer error `ERROR: pgbouncer cannot connect to server`](troubleshooting.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server) + in the Troubleshooting section before proceeding. + +1. [Configure fast lookup of authorized SSH keys in the database](../operations/fast_ssh_key_lookup.md). + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure Prometheus + +The Omnibus GitLab package can be used to configure a standalone Monitoring node +running [Prometheus](../monitoring/prometheus/index.md) and +[Grafana](../monitoring/performance/grafana_configuration.md): + +1. SSH into the Monitoring node. +1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab + package you want using **steps 1 and 2** from the GitLab downloads page. + Do not complete any other steps on the download page. +1. Edit `/etc/gitlab/gitlab.rb` and add the contents: + + ```ruby + external_url 'http://gitlab.example.com' + + # Disable all other services + gitlab_rails['auto_migrate'] = false + alertmanager['enable'] = false + gitaly['enable'] = false + gitlab_exporter['enable'] = false + gitlab_workhorse['enable'] = false + nginx['enable'] = true + postgres_exporter['enable'] = false + postgresql['enable'] = false + redis['enable'] = false + redis_exporter['enable'] = false + sidekiq['enable'] = false + puma['enable'] = false + unicorn['enable'] = false + node_exporter['enable'] = false + gitlab_exporter['enable'] = false + + # Enable Prometheus + prometheus['enable'] = true + prometheus['listen_address'] = '0.0.0.0:9090' + prometheus['monitor_kubernetes'] = false + + # Enable Login form + grafana['disable_login_form'] = false + + # Enable Grafana + grafana['enable'] = true + grafana['admin_password'] = '<grafana_password>' + + # Enable service discovery for Prometheus + consul['enable'] = true + consul['monitoring_service_discovery'] = true + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13) + } + ``` + +1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). +1. In the GitLab UI, set `admin/application_settings/metrics_and_profiling` > Metrics - Grafana to `/-/grafana` to + `http[s]://<MONITOR NODE>/-/grafana`. +1. Verify the GitLab services are running: + + ```shell + sudo gitlab-ctl status + ``` + + The output should be similar to the following: + + ```plaintext + run: consul: (pid 31637) 17337s; run: log: (pid 29748) 78432s + run: grafana: (pid 31644) 17337s; run: log: (pid 29719) 78438s + run: logrotate: (pid 31809) 2936s; run: log: (pid 29581) 78462s + run: nginx: (pid 31665) 17335s; run: log: (pid 29556) 78468s + run: prometheus: (pid 31672) 17335s; run: log: (pid 29633) 78456s + ``` + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure the object storage + +GitLab supports using an object storage service for holding numerous types of data. +It's recommended over [NFS](#configure-nfs-optional) and in general it's better +in larger setups as object storage is typically much more performant, reliable, +and scalable. + +Object storage options that GitLab has tested, or is aware of customers using include: + +- SaaS/Cloud solutions such as [Amazon S3](https://aws.amazon.com/s3/), [Google cloud storage](https://cloud.google.com/storage). +- On-premises hardware and appliances from various storage vendors. +- MinIO. There is [a guide to deploying this](https://docs.gitlab.com/charts/advanced/external-object-storage/minio.html) within our Helm Chart documentation. + +For configuring GitLab to use Object Storage refer to the following guides +based on what features you intend to use: + +1. Configure [object storage for backups](../../raketasks/backup_restore.md#uploading-backups-to-a-remote-cloud-storage). +1. Configure [object storage for job artifacts](../job_artifacts.md#using-object-storage) + including [incremental logging](../job_logs.md#new-incremental-logging-architecture). +1. Configure [object storage for LFS objects](../lfs/index.md#storing-lfs-objects-in-remote-object-storage). +1. Configure [object storage for uploads](../uploads.md#using-object-storage-core-only). +1. Configure [object storage for merge request diffs](../merge_request_diffs.md#using-object-storage). +1. Configure [object storage for Container Registry](../packages/container_registry.md#use-object-storage) (optional feature). +1. Configure [object storage for Mattermost](https://docs.mattermost.com/administration/config-settings.html#file-storage) (optional feature). +1. Configure [object storage for packages](../packages/index.md#using-object-storage) (optional feature). **(PREMIUM ONLY)** +1. Configure [object storage for Dependency Proxy](../packages/dependency_proxy.md#using-object-storage) (optional feature). **(PREMIUM ONLY)** +1. Configure [object storage for Pseudonymizer](../pseudonymizer.md#configuration) (optional feature). **(ULTIMATE ONLY)** +1. Configure [object storage for autoscale Runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional - for improved performance). +1. Configure [object storage for Terraform state files](../terraform_state.md#using-object-storage-core-only). + +Using separate buckets for each data type is the recommended approach for GitLab. + +A limitation of our configuration is that each use of object storage is separately configured. +[We have an issue for improving this](https://gitlab.com/gitlab-org/gitlab/-/issues/23345) +and easily using one bucket with separate folders is one improvement that this might bring. + +There is at least one specific issue with using the same bucket: +when GitLab is deployed with the Helm chart restore from backup +[will not properly function](https://docs.gitlab.com/charts/advanced/external-object-storage/#lfs-artifacts-uploads-packages-external-diffs-pseudonymizer) +unless separate buckets are used. + +One risk of using a single bucket would be if your organization decided to +migrate GitLab to the Helm deployment in the future. GitLab would run, but the situation with +backups might not be realized until the organization had a critical requirement for the backups to +work. + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure NFS (optional) + +[Object storage](#configure-the-object-storage), along with [Gitaly](#configure-gitaly) +are recommended over NFS wherever possible for improved performance. If you intend +to use GitLab Pages, this currently [requires NFS](troubleshooting.md#gitlab-pages-requires-nfs). + +See how to [configure NFS](../high_availability/nfs.md). + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Troubleshooting + +See the [troubleshooting documentation](troubleshooting.md). + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> diff --git a/doc/administration/reference_architectures/50k_users.md b/doc/administration/reference_architectures/50k_users.md index dd94f5470b4..2540fe68dac 100644 --- a/doc/administration/reference_architectures/50k_users.md +++ b/doc/administration/reference_architectures/50k_users.md @@ -47,7 +47,7 @@ For a full list of reference architectures, see For medium sized installs (3,000 - 5,000) we suggest one Redis cluster for all classes and that Redis Sentinel is hosted alongside Consul. For larger architectures (10,000 users or more) we suggest running a separate - [Redis Cluster](../high_availability/redis.md#running-multiple-redis-clusters) for the Cache class + [Redis Cluster](../redis/replication_and_failover.md#running-multiple-redis-clusters) for the Cache class and another for the Queues and Shared State classes respectively. We also recommend that you run the Redis Sentinel clusters separately for each Redis Cluster. diff --git a/doc/administration/reference_architectures/5k_users.md b/doc/administration/reference_architectures/5k_users.md index 604572b083e..0b4114bca6e 100644 --- a/doc/administration/reference_architectures/5k_users.md +++ b/doc/administration/reference_architectures/5k_users.md @@ -1,73 +1,1767 @@ +--- +reading_time: true +--- + # Reference architecture: up to 5,000 users This page describes GitLab reference architecture for up to 5,000 users. For a full list of reference architectures, see [Available reference architectures](index.md#available-reference-architectures). +NOTE: **Note:** +The 5,000-user reference architecture documented below is +designed to help your organization achieve a highly-available GitLab deployment. +If you do not have the expertise or need to maintain a highly-available +environment, you can have a simpler and less costly-to-operate environment by +following the [2,000-user reference architecture](2k_users.md). + > - **Supported users (approximate):** 5,000 > - **High Availability:** True > - **Test RPS rates:** API: 100 RPS, Web: 10 RPS, Git: 10 RPS -| Service | Nodes | Configuration ([8](#footnotes)) | GCP | AWS | Azure | -|--------------------------------------------------------------|-------|---------------------------------|---------------|-----------------------|----------------| -| GitLab Rails ([1](#footnotes)) | 3 | 16 vCPU, 14.4GB Memory | `n1-highcpu-16` | `c5.4xlarge` | F16s v2 | -| PostgreSQL | 3 | 2 vCPU, 7.5GB Memory | `n1-standard-2` | `m5.large` | D2s v3 | -| PgBouncer | 3 | 2 vCPU, 1.8GB Memory | `n1-highcpu-2` | `c5.large` | F2s v2 | -| Gitaly ([2](#footnotes)) ([5](#footnotes)) ([7](#footnotes)) | X | 8 vCPU, 30GB Memory | `n1-standard-8` | `m5.2xlarge` | D8s v3 | -| Redis ([3](#footnotes)) | 3 | 2 vCPU, 7.5GB Memory | `n1-standard-2` | `m5.large` | D2s v3 | -| Consul + Sentinel ([3](#footnotes)) | 3 | 2 vCPU, 1.8GB Memory | `n1-highcpu-2` | `c5.large` | F2s v2 | -| Sidekiq | 4 | 2 vCPU, 7.5GB Memory | `n1-standard-2` | `m5.large` | D2s v3 | -| Object Storage ([4](#footnotes)) | - | - | - | - | - | -| NFS Server ([5](#footnotes)) ([7](#footnotes)) | 1 | 4 vCPU, 3.6GB Memory | `n1-highcpu-4` | `c5.xlarge` | F4s v2 | -| Monitoring node | 1 | 2 vCPU, 1.8GB Memory | `n1-highcpu-2` | `c5.large` | F2s v2 | -| External load balancing node ([6](#footnotes)) | 1 | 2 vCPU, 1.8GB Memory | `n1-highcpu-2` | `c5.large` | F2s v2 | -| Internal load balancing node ([6](#footnotes)) | 1 | 2 vCPU, 1.8GB Memory | `n1-highcpu-2` | `c5.large` | F2s v2 | - -## Footnotes - -1. In our architectures we run each GitLab Rails node using the Puma webserver - and have its number of workers set to 90% of available CPUs along with four threads. For - nodes that are running Rails with other components the worker value should be reduced - accordingly where we've found 50% achieves a good balance but this is dependent - on workload. - -1. Gitaly node requirements are dependent on customer data, specifically the number of - projects and their sizes. We recommend two nodes as an absolute minimum for HA environments - and at least four nodes should be used when supporting 50,000 or more users. - We also recommend that each Gitaly node should store no more than 5TB of data - and have the number of [`gitaly-ruby` workers](../gitaly/index.md#gitaly-ruby) - set to 20% of available CPUs. Additional nodes should be considered in conjunction - with a review of expected data size and spread based on the recommendations above. - -1. Recommended Redis setup differs depending on the size of the architecture. - For smaller architectures (less than 3,000 users) a single instance should suffice. - For medium sized installs (3,000 - 5,000) we suggest one Redis cluster for all - classes and that Redis Sentinel is hosted alongside Consul. - For larger architectures (10,000 users or more) we suggest running a separate - [Redis Cluster](../high_availability/redis.md#running-multiple-redis-clusters) for the Cache class - and another for the Queues and Shared State classes respectively. We also recommend - that you run the Redis Sentinel clusters separately for each Redis Cluster. - -1. For data objects such as LFS, Uploads, Artifacts, etc. We recommend an [Object Storage service](../object_storage.md) - over NFS where possible, due to better performance and availability. - -1. NFS can be used as an alternative for both repository data (replacing Gitaly) and - object storage but this isn't typically recommended for performance reasons. Note however it is required for - [GitLab Pages](https://gitlab.com/gitlab-org/gitlab-pages/-/issues/196). - -1. Our architectures have been tested and validated with [HAProxy](https://www.haproxy.org/) - as the load balancer. Although other load balancers with similar feature sets - could also be used, those load balancers have not been validated. - -1. We strongly recommend that any Gitaly or NFS nodes be set up with SSD disks over - HDD with a throughput of at least 8,000 IOPS for read operations and 2,000 IOPS for write - as these components have heavy I/O. These IOPS values are recommended only as a starter - as with time they may be adjusted higher or lower depending on the scale of your - environment's workload. If you're running the environment on a Cloud provider - you may need to refer to their documentation on how configure IOPS correctly. - -1. The architectures were built and tested with the [Intel Xeon E5 v3 (Haswell)](https://cloud.google.com/compute/docs/cpu-platforms) - CPU platform on GCP. On different hardware you may find that adjustments, either lower - or higher, are required for your CPU or Node counts accordingly. For more information, a - [Sysbench](https://github.com/akopytov/sysbench) benchmark of the CPU can be found - [here](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Reference-Architectures/GCP-CPU-Benchmarks). +| Service | Nodes | Configuration | GCP | AWS | Azure | +|--------------------------------------------------------------|-------|---------------------------------|-----------------|-----------------------|----------------| +| External load balancing node | 1 | 2 vCPU, 1.8GB Memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| Redis | 3 | 2 vCPU, 7.5GB Memory | `n1-standard-2` | `m5.large` | `D2s v3` | +| Consul + Sentinel | 3 | 2 vCPU, 1.8GB Memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| PostgreSQL | 3 | 2 vCPU, 7.5GB Memory | `n1-standard-2` | `m5.large` | `D2s v3` | +| PgBouncer | 3 | 2 vCPU, 1.8GB Memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| Internal load balancing node | 1 | 2 vCPU, 1.8GB Memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| Gitaly | 2 minimum | 8 vCPU, 30GB Memory | `n1-standard-8` | `m5.2xlarge` | `D8s v3` | +| Sidekiq | 4 | 2 vCPU, 7.5GB Memory | `n1-standard-2` | `m5.large` | `D2s v3` | +| GitLab Rails | 3 | 16 vCPU, 14.4GB Memory | `n1-highcpu-16` | `c5.4xlarge` | `F16s v2` | +| Monitoring node | 1 | 2 vCPU, 1.8GB Memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| Object Storage | n/a | n/a | n/a | n/a | n/a | +| NFS Server (optional, not recommended) | 1 | 4 vCPU, 3.6GB Memory | `n1-highcpu-4` | `c5.xlarge` | `F4s v2` | + +The architectures were built and tested with the [Intel Xeon E5 v3 (Haswell)](https://cloud.google.com/compute/docs/cpu-platforms) +CPU platform on GCP. On different hardware you may find that adjustments, either lower +or higher, are required for your CPU or Node counts accordingly. For more information, a +[Sysbench](https://github.com/akopytov/sysbench) benchmark of the CPU can be found +[here](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Reference-Architectures/GCP-CPU-Benchmarks). + +For data objects such as LFS, Uploads, Artifacts, etc, an [object storage service](#configure-the-object-storage) +is recommended over NFS where possible, due to better performance and availability. +Since this doesn't require a node to be set up, it's marked as not applicable (n/a) +in the table above. + +## Setup components + +To set up GitLab and its components to accommodate up to 5,000 users: + +1. [Configure the external load balancing node](#configure-the-external-load-balancer) + that will handle the load balancing of the two GitLab application services nodes. +1. [Configure Redis](#configure-redis). +1. [Configure Consul and Sentinel](#configure-consul-and-sentinel). +1. [Configure PostgreSQL](#configure-postgresql), the database for GitLab. +1. [Configure PgBouncer](#configure-pgbouncer). +1. [Configure the internal load balancing node](#configure-the-internal-load-balancer) +1. [Configure Gitaly](#configure-gitaly), + which provides access to the Git repositories. +1. [Configure Sidekiq](#configure-sidekiq). +1. [Configure the main GitLab Rails application](#configure-gitlab-rails) + to run Puma/Unicorn, Workhorse, GitLab Shell, and to serve all frontend requests (UI, API, Git + over HTTP/SSH). +1. [Configure Prometheus](#configure-prometheus) to monitor your GitLab environment. +1. [Configure the Object Storage](#configure-the-object-storage) + used for shared data objects. +1. [Configure NFS (Optional)](#configure-nfs-optional) + to have shared disk storage service as an alternative to Gitaly and/or Object Storage (although + not recommended). NFS is required for GitLab Pages, you can skip this step if you're not using + that feature. + +We start with all servers on the same 10.6.0.0/16 private network range, they +can connect to each other freely on those addresses. + +Here is a list and description of each machine and the assigned IP: + +- `10.6.0.10`: External Load Balancer +- `10.6.0.61`: Redis Primary +- `10.6.0.62`: Redis Replica 1 +- `10.6.0.63`: Redis Replica 2 +- `10.6.0.11`: Consul/Sentinel 1 +- `10.6.0.12`: Consul/Sentinel 2 +- `10.6.0.13`: Consul/Sentinel 3 +- `10.6.0.31`: PostgreSQL primary +- `10.6.0.32`: PostgreSQL secondary 1 +- `10.6.0.33`: PostgreSQL secondary 2 +- `10.6.0.21`: PgBouncer 1 +- `10.6.0.22`: PgBouncer 2 +- `10.6.0.23`: PgBouncer 3 +- `10.6.0.20`: Internal Load Balancer +- `10.6.0.51`: Gitaly 1 +- `10.6.0.52`: Gitaly 2 +- `10.6.0.71`: Sidekiq 1 +- `10.6.0.72`: Sidekiq 2 +- `10.6.0.73`: Sidekiq 3 +- `10.6.0.74`: Sidekiq 4 +- `10.6.0.41`: GitLab application 1 +- `10.6.0.42`: GitLab application 2 +- `10.6.0.43`: GitLab application 3 +- `10.6.0.81`: Prometheus + +## Configure the external load balancer + +NOTE: **Note:** +This architecture has been tested and validated with [HAProxy](https://www.haproxy.org/) +as the load balancer. Although other load balancers with similar feature sets +could also be used, those load balancers have not been validated. + +In an active/active GitLab configuration, you will need a load balancer to route +traffic to the application servers. The specifics on which load balancer to use +or the exact configuration is beyond the scope of GitLab documentation. We hope +that if you're managing multi-node systems like GitLab you have a load balancer of +choice already. Some examples including HAProxy (open-source), F5 Big-IP LTM, +and Citrix Net Scaler. This documentation will outline what ports and protocols +you need to use with GitLab. + +The next question is how you will handle SSL in your environment. +There are several different options: + +- [The application node terminates SSL](#application-node-terminates-ssl). +- [The load balancer terminates SSL without backend SSL](#load-balancer-terminates-ssl-without-backend-ssl) + and communication is not secure between the load balancer and the application node. +- [The load balancer terminates SSL with backend SSL](#load-balancer-terminates-ssl-with-backend-ssl) + and communication is *secure* between the load balancer and the application node. + +### Application node terminates SSL + +Configure your load balancer to pass connections on port 443 as `TCP` rather +than `HTTP(S)` protocol. This will pass the connection to the application node's +NGINX service untouched. NGINX will have the SSL certificate and listen on port 443. + +See the [NGINX HTTPS documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https) +for details on managing SSL certificates and configuring NGINX. + +### Load balancer terminates SSL without backend SSL + +Configure your load balancer to use the `HTTP(S)` protocol rather than `TCP`. +The load balancer will then be responsible for managing SSL certificates and +terminating SSL. + +Since communication between the load balancer and GitLab will not be secure, +there is some additional configuration needed. See the +[NGINX proxied SSL documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#supporting-proxied-ssl) +for details. + +### Load balancer terminates SSL with backend SSL + +Configure your load balancer(s) to use the 'HTTP(S)' protocol rather than 'TCP'. +The load balancer(s) will be responsible for managing SSL certificates that +end users will see. + +Traffic will also be secure between the load balancer(s) and NGINX in this +scenario. There is no need to add configuration for proxied SSL since the +connection will be secure all the way. However, configuration will need to be +added to GitLab to configure SSL certificates. See +[NGINX HTTPS documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https) +for details on managing SSL certificates and configuring NGINX. + +### Ports + +The basic ports to be used are shown in the table below. + +| LB Port | Backend Port | Protocol | +| ------- | ------------ | ------------------------ | +| 80 | 80 | HTTP (*1*) | +| 443 | 443 | TCP or HTTPS (*1*) (*2*) | +| 22 | 22 | TCP | + +- (*1*): [Web terminal](../../ci/environments/index.md#web-terminals) support requires + your load balancer to correctly handle WebSocket connections. When using + HTTP or HTTPS proxying, this means your load balancer must be configured + to pass through the `Connection` and `Upgrade` hop-by-hop headers. See the + [web terminal](../integration/terminal.md) integration guide for + more details. +- (*2*): When using HTTPS protocol for port 443, you will need to add an SSL + certificate to the load balancers. If you wish to terminate SSL at the + GitLab application server instead, use TCP protocol. + +If you're using GitLab Pages with custom domain support you will need some +additional port configurations. +GitLab Pages requires a separate virtual IP address. Configure DNS to point the +`pages_external_url` from `/etc/gitlab/gitlab.rb` at the new virtual IP address. See the +[GitLab Pages documentation](../pages/index.md) for more information. + +| LB Port | Backend Port | Protocol | +| ------- | ------------- | --------- | +| 80 | Varies (*1*) | HTTP | +| 443 | Varies (*1*) | TCP (*2*) | + +- (*1*): The backend port for GitLab Pages depends on the + `gitlab_pages['external_http']` and `gitlab_pages['external_https']` + setting. See [GitLab Pages documentation](../pages/index.md) for more details. +- (*2*): Port 443 for GitLab Pages should always use the TCP protocol. Users can + configure custom domains with custom SSL, which would not be possible + if SSL was terminated at the load balancer. + +#### Alternate SSH Port + +Some organizations have policies against opening SSH port 22. In this case, +it may be helpful to configure an alternate SSH hostname that allows users +to use SSH on port 443. An alternate SSH hostname will require a new virtual IP address +compared to the other GitLab HTTP configuration above. + +Configure DNS for an alternate SSH hostname such as `altssh.gitlab.example.com`. + +| LB Port | Backend Port | Protocol | +| ------- | ------------ | -------- | +| 443 | 22 | TCP | + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure Redis + +Using [Redis](https://redis.io/) in scalable environment is possible using a **Primary** x **Replica** +topology with a [Redis Sentinel](https://redis.io/topics/sentinel) service to watch and automatically +start the failover procedure. + +Redis requires authentication if used with Sentinel. See +[Redis Security](https://redis.io/topics/security) documentation for more +information. We recommend using a combination of a Redis password and tight +firewall rules to secure your Redis service. +You are highly encouraged to read the [Redis Sentinel](https://redis.io/topics/sentinel) documentation +before configuring Redis with GitLab to fully understand the topology and +architecture. + +In this section, you'll be guided through configuring an external Redis instance +to be used with GitLab. The following IPs will be used as an example: + +- `10.6.0.61`: Redis Primary +- `10.6.0.62`: Redis Replica 1 +- `10.6.0.63`: Redis Replica 2 + +### Provide your own Redis instance + +Managed Redis from cloud providers such as AWS ElastiCache will work. If these +services support high availability, be sure it is **not** the Redis Cluster type. + +Redis version 5.0 or higher is required, as this is what ships with +Omnibus GitLab packages starting with GitLab 13.0. Older Redis versions +do not support an optional count argument to SPOP which is now required for +[Merge Trains](../../ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md). + +Note the Redis node's IP address or hostname, port, and password (if required). +These will be necessary when configuring the +[GitLab application servers](#configure-gitlab-rails) later. + +### Standalone Redis using Omnibus GitLab + +This is the section where we install and set up the new Redis instances. + +The requirements for a Redis setup are the following: + +1. All Redis nodes must be able to talk to each other and accept incoming + connections over Redis (`6379`) and Sentinel (`26379`) ports (unless you + change the default ones). +1. The server that hosts the GitLab application must be able to access the + Redis nodes. +1. Protect the nodes from access from external networks + ([Internet](https://gitlab.com/gitlab-org/gitlab-foss/uploads/c4cc8cd353604bd80315f9384035ff9e/The_Internet_IT_Crowd.png)), + using a firewall. + +NOTE: **Note:** +Redis nodes (both primary and replica) will need the same password defined in +`redis['password']`. At any time during a failover the Sentinels can +reconfigure a node and change its status from primary to replica and vice versa. + +#### Configuring the primary Redis instance + +1. SSH into the **Primary** Redis server. +1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab + package you want using **steps 1 and 2** from the GitLab downloads page. + - Make sure you select the correct Omnibus package, with the same version + and type (Community, Enterprise editions) of your current install. + - Do not complete any other steps on the download page. + +1. Edit `/etc/gitlab/gitlab.rb` and add the contents: + + ```ruby + # Specify server role as 'redis_master_role' + roles ['redis_master_role'] + + # IP address pointing to a local IP that the other machines can reach to. + # You can also set bind to '0.0.0.0' which listen in all interfaces. + # If you really need to bind to an external accessible IP, make + # sure you add extra firewall rules to prevent unauthorized access. + redis['bind'] = '10.6.0.61' + + # Define a port so Redis can listen for TCP requests which will allow other + # machines to connect to it. + redis['port'] = 6379 + + # Set up password authentication for Redis (use the same password in all nodes). + redis['password'] = 'redis-password-goes-here' + + ## Enable service discovery for Prometheus + consul['enable'] = true + consul['monitoring_service_discovery'] = true + + ## The IPs of the Consul server nodes + ## You can also use FQDNs and intermix them with IPs + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), + } + + # Set the network addresses that the exporters will listen on + node_exporter['listen_address'] = '0.0.0.0:9100' + redis_exporter['listen_address'] = '0.0.0.0:9121' + redis_exporter['flags'] = { + 'redis.addr' => 'redis://10.6.0.61:6379', + 'redis.password' => 'redis-password-goes-here', + } + + # Disable auto migrations + gitlab_rails['auto_migrate'] = false + ``` + +1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + +NOTE: **Note:** +You can specify multiple roles like sentinel and Redis as: +`roles ['redis_sentinel_role', 'redis_master_role']`. +Read more about [roles](https://docs.gitlab.com/omnibus/roles/). + +You can list the current Redis Primary, Replica status via: + +```shell +/opt/gitlab/embedded/bin/redis-cli -h <host> -a 'redis-password-goes-here' info replication +``` + +Show running GitLab services via: + +```shell +gitlab-ctl status +``` + +The output should be similar to the following: + +```plaintext +run: consul: (pid 30043) 76863s; run: log: (pid 29691) 76892s +run: logrotate: (pid 31152) 3070s; run: log: (pid 29595) 76908s +run: node-exporter: (pid 30064) 76862s; run: log: (pid 29624) 76904s +run: redis: (pid 30070) 76861s; run: log: (pid 29573) 76914s +run: redis-exporter: (pid 30075) 76861s; run: log: (pid 29674) 76896s +``` + +#### Configuring the replica Redis instances + +1. SSH into the **replica** Redis server. +1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab + package you want using **steps 1 and 2** from the GitLab downloads page. + - Make sure you select the correct Omnibus package, with the same version + and type (Community, Enterprise editions) of your current install. + - Do not complete any other steps on the download page. + +1. Edit `/etc/gitlab/gitlab.rb` and add the contents: + + ```ruby + # Specify server role as 'redis_replica_role' + roles ['redis_replica_role'] + + # IP address pointing to a local IP that the other machines can reach to. + # You can also set bind to '0.0.0.0' which listen in all interfaces. + # If you really need to bind to an external accessible IP, make + # sure you add extra firewall rules to prevent unauthorized access. + redis['bind'] = '10.6.0.62' + + # Define a port so Redis can listen for TCP requests which will allow other + # machines to connect to it. + redis['port'] = 6379 + + # The same password for Redis authentication you set up for the primary node. + redis['password'] = 'redis-password-goes-here' + + # The IP of the primary Redis node. + redis['master_ip'] = '10.6.0.61' + + # Port of primary Redis server, uncomment to change to non default. Defaults + # to `6379`. + #redis['master_port'] = 6379 + + ## Enable service discovery for Prometheus + consul['enable'] = true + consul['monitoring_service_discovery'] = true + + ## The IPs of the Consul server nodes + ## You can also use FQDNs and intermix them with IPs + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), + } + + # Set the network addresses that the exporters will listen on + node_exporter['listen_address'] = '0.0.0.0:9100' + redis_exporter['listen_address'] = '0.0.0.0:9121' + redis_exporter['flags'] = { + 'redis.addr' => 'redis://10.6.0.62:6379', + 'redis.password' => 'redis-password-goes-here', + } + + # Disable auto migrations + gitlab_rails['auto_migrate'] = false + ``` + +1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. +1. Go through the steps again for all the other replica nodes, and + make sure to set up the IPs correctly. + +NOTE: **Note:** +You can specify multiple roles like sentinel and Redis as: +`roles ['redis_sentinel_role', 'redis_master_role']`. +Read more about [roles](https://docs.gitlab.com/omnibus/roles/). + +These values don't have to be changed again in `/etc/gitlab/gitlab.rb` after +a failover, as the nodes will be managed by the [Sentinels](#configure-consul-and-sentinel), and even after a +`gitlab-ctl reconfigure`, they will get their configuration restored by +the same Sentinels. + +Advanced [configuration options](https://docs.gitlab.com/omnibus/settings/redis.html) +are supported and can be added if needed. + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure Consul and Sentinel + +NOTE: **Note:** +If you are using an external Redis Sentinel instance, be sure +to exclude the `requirepass` parameter from the Sentinel +configuration. This parameter will cause clients to report `NOAUTH +Authentication required.`. [Redis Sentinel 3.2.x does not support +password authentication](https://github.com/antirez/redis/issues/3279). + +Now that the Redis servers are all set up, let's configure the Sentinel +servers. The following IPs will be used as an example: + +- `10.6.0.11`: Consul/Sentinel 1 +- `10.6.0.12`: Consul/Sentinel 2 +- `10.6.0.13`: Consul/Sentinel 3 + +To configure the Sentinel: + +1. SSH into the server that will host Consul/Sentinel. +1. [Download/install](https://about.gitlab.com/install/) the + Omnibus GitLab Enterprise Edition package using **steps 1 and 2** from the + GitLab downloads page. + - Make sure you select the correct Omnibus package, with the same version + the GitLab application is running. + - Do not complete any other steps on the download page. + +1. Edit `/etc/gitlab/gitlab.rb` and add the contents: + + ```ruby + roles ['redis_sentinel_role', 'consul_role'] + + # Must be the same in every sentinel node + redis['master_name'] = 'gitlab-redis' + + # The same password for Redis authentication you set up for the primary node. + redis['master_password'] = 'redis-password-goes-here' + + # The IP of the primary Redis node. + redis['master_ip'] = '10.6.0.61' + + # Define a port so Redis can listen for TCP requests which will allow other + # machines to connect to it. + redis['port'] = 6379 + + # Port of primary Redis server, uncomment to change to non default. Defaults + # to `6379`. + #redis['master_port'] = 6379 + + ## Configure Sentinel + sentinel['bind'] = '10.6.0.11' + + # Port that Sentinel listens on, uncomment to change to non default. Defaults + # to `26379`. + # sentinel['port'] = 26379 + + ## Quorum must reflect the amount of voting sentinels it take to start a failover. + ## Value must NOT be greater then the amount of sentinels. + ## + ## The quorum can be used to tune Sentinel in two ways: + ## 1. If a the quorum is set to a value smaller than the majority of Sentinels + ## we deploy, we are basically making Sentinel more sensible to primary failures, + ## triggering a failover as soon as even just a minority of Sentinels is no longer + ## able to talk with the primary. + ## 1. If a quorum is set to a value greater than the majority of Sentinels, we are + ## making Sentinel able to failover only when there are a very large number (larger + ## than majority) of well connected Sentinels which agree about the primary being down.s + sentinel['quorum'] = 2 + + ## Consider unresponsive server down after x amount of ms. + # sentinel['down_after_milliseconds'] = 10000 + + ## Specifies the failover timeout in milliseconds. It is used in many ways: + ## + ## - The time needed to re-start a failover after a previous failover was + ## already tried against the same primary by a given Sentinel, is two + ## times the failover timeout. + ## + ## - The time needed for a replica replicating to a wrong primary according + ## to a Sentinel current configuration, to be forced to replicate + ## with the right primary, is exactly the failover timeout (counting since + ## the moment a Sentinel detected the misconfiguration). + ## + ## - The time needed to cancel a failover that is already in progress but + ## did not produced any configuration change (REPLICAOF NO ONE yet not + ## acknowledged by the promoted replica). + ## + ## - The maximum time a failover in progress waits for all the replica to be + ## reconfigured as replicas of the new primary. However even after this time + ## the replicas will be reconfigured by the Sentinels anyway, but not with + ## the exact parallel-syncs progression as specified. + # sentinel['failover_timeout'] = 60000 + + ## Enable service discovery for Prometheus + consul['enable'] = true + consul['monitoring_service_discovery'] = true + + ## The IPs of the Consul server nodes + ## You can also use FQDNs and intermix them with IPs + consul['configuration'] = { + server: true, + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), + } + + # Set the network addresses that the exporters will listen on + node_exporter['listen_address'] = '0.0.0.0:9100' + redis_exporter['listen_address'] = '0.0.0.0:9121' + + # Disable auto migrations + gitlab_rails['auto_migrate'] = false + ``` + +1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. +1. Go through the steps again for all the other Consul/Sentinel nodes, and + make sure you set up the correct IPs. + +NOTE: **Note:** +A Consul leader will be elected when the provisioning of the third Consul server is completed. +Viewing the Consul logs `sudo gitlab-ctl tail consul` will display +`...[INFO] consul: New leader elected: ...` + +You can list the current Consul members (server, client): + +```shell +sudo /opt/gitlab/embedded/bin/consul members +``` + +You can verify the GitLab services are running: + +```shell +sudo gitlab-ctl status +``` + +The output should be similar to the following: + +```plaintext +run: consul: (pid 30074) 76834s; run: log: (pid 29740) 76844s +run: logrotate: (pid 30925) 3041s; run: log: (pid 29649) 76861s +run: node-exporter: (pid 30093) 76833s; run: log: (pid 29663) 76855s +run: sentinel: (pid 30098) 76832s; run: log: (pid 29704) 76850s +``` + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure PostgreSQL + +In this section, you'll be guided through configuring an external PostgreSQL database +to be used with GitLab. + +### Provide your own PostgreSQL instance + +If you're hosting GitLab on a cloud provider, you can optionally use a +managed service for PostgreSQL. For example, AWS offers a managed Relational +Database Service (RDS) that runs PostgreSQL. + +If you use a cloud-managed service, or provide your own PostgreSQL: + +1. Set up PostgreSQL according to the + [database requirements document](../../install/requirements.md#database). +1. Set up a `gitlab` username with a password of your choice. The `gitlab` user + needs privileges to create the `gitlabhq_production` database. +1. Configure the GitLab application servers with the appropriate details. + This step is covered in [Configuring the GitLab Rails application](#configure-gitlab-rails). + +### Standalone PostgreSQL using Omnibus GitLab + +The following IPs will be used as an example: + +- `10.6.0.31`: PostgreSQL primary +- `10.6.0.32`: PostgreSQL secondary 1 +- `10.6.0.33`: PostgreSQL secondary 2 + +First, make sure to [install](https://about.gitlab.com/install/) +the Linux GitLab package **on each node**. Following the steps, +install the necessary dependencies from step 1, and add the +GitLab package repository from step 2. When installing GitLab +in the second step, do not supply the `EXTERNAL_URL` value. + +#### PostgreSQL primary node + +1. SSH into the PostgreSQL primary node. +1. Generate a password hash for the PostgreSQL username/password pair. This assumes you will use the default + username of `gitlab` (recommended). The command will request a password + and confirmation. Use the value that is output by this command in the next + step as the value of `<postgresql_password_hash>`: + + ```shell + sudo gitlab-ctl pg-password-md5 gitlab + ``` + +1. Generate a password hash for the PgBouncer username/password pair. This assumes you will use the default + username of `pgbouncer` (recommended). The command will request a password + and confirmation. Use the value that is output by this command in the next + step as the value of `<pgbouncer_password_hash>`: + + ```shell + sudo gitlab-ctl pg-password-md5 pgbouncer + ``` + +1. Generate a password hash for the Consul database username/password pair. This assumes you will use the default + username of `gitlab-consul` (recommended). The command will request a password + and confirmation. Use the value that is output by this command in the next + step as the value of `<consul_password_hash>`: + + ```shell + sudo gitlab-ctl pg-password-md5 gitlab-consul + ``` + +1. On the primary database node, edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: + + ```ruby + # Disable all components except PostgreSQL and Repmgr and Consul + roles ['postgres_role'] + + # PostgreSQL configuration + postgresql['listen_address'] = '0.0.0.0' + postgresql['hot_standby'] = 'on' + postgresql['wal_level'] = 'replica' + postgresql['shared_preload_libraries'] = 'repmgr_funcs' + + # Disable automatic database migrations + gitlab_rails['auto_migrate'] = false + + # Configure the Consul agent + consul['services'] = %w(postgresql) + + # START user configuration + # Please set the real values as explained in Required Information section + # + # Replace PGBOUNCER_PASSWORD_HASH with a generated md5 value + postgresql['pgbouncer_user_password'] = '<pgbouncer_password_hash>' + # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value + postgresql['sql_user_password'] = '<postgresql_password_hash>' + # Set `max_wal_senders` to one more than the number of database nodes in the cluster. + # This is used to prevent replication from using up all of the + # available database connections. + postgresql['max_wal_senders'] = 4 + postgresql['max_replication_slots'] = 4 + + # Replace XXX.XXX.XXX.XXX/YY with Network Address + postgresql['trust_auth_cidr_addresses'] = %w(127.0.0.1/32 10.6.0.0/24) + repmgr['trust_auth_cidr_addresses'] = %w(127.0.0.1/32 10.6.0.0/24) + + ## Enable service discovery for Prometheus + consul['enable'] = true + consul['monitoring_service_discovery'] = true + + # Set the network addresses that the exporters will listen on for monitoring + node_exporter['listen_address'] = '0.0.0.0:9100' + postgres_exporter['listen_address'] = '0.0.0.0:9187' + postgres_exporter['dbname'] = 'gitlabhq_production' + postgres_exporter['password'] = '<postgresql_password_hash>' + + ## The IPs of the Consul server nodes + ## You can also use FQDNs and intermix them with IPs + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), + } + # + # END user configuration + ``` + +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. +1. You can list the current PostgreSQL primary, secondary nodes status via: + + ```shell + sudo /opt/gitlab/bin/gitlab-ctl repmgr cluster show + ``` + +1. Verify the GitLab services are running: + + ```shell + sudo gitlab-ctl status + ``` + + The output should be similar to the following: + + ```plaintext + run: consul: (pid 30593) 77133s; run: log: (pid 29912) 77156s + run: logrotate: (pid 23449) 3341s; run: log: (pid 29794) 77175s + run: node-exporter: (pid 30613) 77133s; run: log: (pid 29824) 77170s + run: postgres-exporter: (pid 30620) 77132s; run: log: (pid 29894) 77163s + run: postgresql: (pid 30630) 77132s; run: log: (pid 29618) 77181s + run: repmgrd: (pid 30639) 77132s; run: log: (pid 29985) 77150s + ``` + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +#### PostgreSQL secondary nodes + +1. On both the secondary nodes, add the same configuration specified above for the primary node + with an additional setting that will inform `gitlab-ctl` that they are standby nodes initially + and there's no need to attempt to register them as a primary node: + + ```ruby + # Disable all components except PostgreSQL and Repmgr and Consul + roles ['postgres_role'] + + # PostgreSQL configuration + postgresql['listen_address'] = '0.0.0.0' + postgresql['hot_standby'] = 'on' + postgresql['wal_level'] = 'replica' + postgresql['shared_preload_libraries'] = 'repmgr_funcs' + + # Disable automatic database migrations + gitlab_rails['auto_migrate'] = false + + # Configure the Consul agent + consul['services'] = %w(postgresql) + + # Specify if a node should attempt to be primary on initialization. + repmgr['master_on_initialization'] = false + + # START user configuration + # Please set the real values as explained in Required Information section + # + # Replace PGBOUNCER_PASSWORD_HASH with a generated md5 value + postgresql['pgbouncer_user_password'] = '<pgbouncer_password_hash>' + # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value + postgresql['sql_user_password'] = '<postgresql_password_hash>' + # Set `max_wal_senders` to one more than the number of database nodes in the cluster. + # This is used to prevent replication from using up all of the + # available database connections. + postgresql['max_wal_senders'] = 4 + postgresql['max_replication_slots'] = 4 + + # Replace XXX.XXX.XXX.XXX/YY with Network Address + postgresql['trust_auth_cidr_addresses'] = %w(127.0.0.1/32 10.6.0.0/24) + repmgr['trust_auth_cidr_addresses'] = %w(127.0.0.1/32 10.6.0.0/24) + + ## Enable service discovery for Prometheus + consul['enable'] = true + consul['monitoring_service_discovery'] = true + + # Set the network addresses that the exporters will listen on for monitoring + node_exporter['listen_address'] = '0.0.0.0:9100' + postgres_exporter['listen_address'] = '0.0.0.0:9187' + postgres_exporter['dbname'] = 'gitlabhq_production' + postgres_exporter['password'] = '<postgresql_password_hash>' + + ## The IPs of the Consul server nodes + ## You can also use FQDNs and intermix them with IPs + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), + } + # END user configuration + ``` + +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + +Advanced [configuration options](https://docs.gitlab.com/omnibus/settings/database.html) +are supported and can be added if needed. + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +#### PostgreSQL post-configuration + +SSH into the **primary node**: + +1. Open a database prompt: + + ```shell + gitlab-psql -d gitlabhq_production + ``` + +1. Enable the `pg_trgm` extension: + + ```shell + CREATE EXTENSION pg_trgm; + ``` + +1. Exit the database prompt by typing `\q` and Enter. + +1. Verify the cluster is initialized with one node: + + ```shell + gitlab-ctl repmgr cluster show + ``` + + The output should be similar to the following: + + ```plaintext + Role | Name | Upstream | Connection String + ----------+----------|----------|---------------------------------------- + * master | HOSTNAME | | host=HOSTNAME user=gitlab_repmgr dbname=gitlab_repmgr + ``` + +1. Note down the hostname or IP address in the connection string: `host=HOSTNAME`. We will + refer to the hostname in the next section as `<primary_node_name>`. If the value + is not an IP address, it will need to be a resolvable name (via DNS or + `/etc/hosts`) + +SSH into the **secondary node**: + +1. Set up the repmgr standby: + + ```shell + gitlab-ctl repmgr standby setup <primary_node_name> + ``` + + Do note that this will remove the existing data on the node. The command + has a wait time. + + The output should be similar to the following: + + ```console + Doing this will delete the entire contents of /var/opt/gitlab/postgresql/data + If this is not what you want, hit Ctrl-C now to exit + To skip waiting, rerun with the -w option + Sleeping for 30 seconds + Stopping the database + Removing the data + Cloning the data + Starting the database + Registering the node with the cluster + ok: run: repmgrd: (pid 19068) 0s + ``` + +Before moving on, make sure the databases are configured correctly. Run the +following command on the **primary** node to verify that replication is working +properly and the secondary nodes appear in the cluster: + +```shell +gitlab-ctl repmgr cluster show +``` + +The output should be similar to the following: + +```plaintext +Role | Name | Upstream | Connection String +----------+---------|-----------|------------------------------------------------ +* master | MASTER | | host=<primary_node_name> user=gitlab_repmgr dbname=gitlab_repmgr + standby | STANDBY | MASTER | host=<secondary_node_name> user=gitlab_repmgr dbname=gitlab_repmgr + standby | STANDBY | MASTER | host=<secondary_node_name> user=gitlab_repmgr dbname=gitlab_repmgr +``` + +If the 'Role' column for any node says "FAILED", check the +[Troubleshooting section](troubleshooting.md) before proceeding. + +Also, check that the `repmgr-check-master` command works successfully on each node: + +```shell +su - gitlab-consul +gitlab-ctl repmgr-check-master || echo 'This node is a standby repmgr node' +``` + +This command relies on exit codes to tell Consul whether a particular node is a master +or secondary. The most important thing here is that this command does not produce errors. +If there are errors it's most likely due to incorrect `gitlab-consul` database user permissions. +Check the [Troubleshooting section](troubleshooting.md) before proceeding. + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure PgBouncer + +Now that the PostgreSQL servers are all set up, let's configure PgBouncer. +The following IPs will be used as an example: + +- `10.6.0.21`: PgBouncer 1 +- `10.6.0.22`: PgBouncer 2 +- `10.6.0.23`: PgBouncer 3 + +1. On each PgBouncer node, edit `/etc/gitlab/gitlab.rb`, and replace + `<consul_password_hash>` and `<pgbouncer_password_hash>` with the + password hashes you [set up previously](#postgresql-primary-node): + + ```ruby + # Disable all components except Pgbouncer and Consul agent + roles ['pgbouncer_role'] + + # Configure PgBouncer + pgbouncer['admin_users'] = %w(pgbouncer gitlab-consul) + + pgbouncer['users'] = { + 'gitlab-consul': { + password: '<consul_password_hash>' + }, + 'pgbouncer': { + password: '<pgbouncer_password_hash>' + } + } + + # Configure Consul agent + consul['watchers'] = %w(postgresql) + consul['enable'] = true + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13) + } + + # Enable service discovery for Prometheus + consul['monitoring_service_discovery'] = true + + # Set the network addresses that the exporters will listen on + node_exporter['listen_address'] = '0.0.0.0:9100' + pgbouncer_exporter['listen_address'] = '0.0.0.0:9188' + ``` + +1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + +1. Create a `.pgpass` file so Consul is able to + reload PgBouncer. Enter the PgBouncer password twice when asked: + + ```shell + gitlab-ctl write-pgpass --host 127.0.0.1 --database pgbouncer --user pgbouncer --hostuser gitlab-consul + ``` + +1. Ensure each node is talking to the current master: + + ```shell + gitlab-ctl pgb-console # You will be prompted for PGBOUNCER_PASSWORD + ``` + + If there is an error `psql: ERROR: Auth failed` after typing in the + password, ensure you previously generated the MD5 password hashes with the correct + format. The correct format is to concatenate the password and the username: + `PASSWORDUSERNAME`. For example, `Sup3rS3cr3tpgbouncer` would be the text + needed to generate an MD5 password hash for the `pgbouncer` user. + +1. Once the console prompt is available, run the following queries: + + ```shell + show databases ; show clients ; + ``` + + The output should be similar to the following: + + ```plaintext + name | host | port | database | force_user | pool_size | reserve_pool | pool_mode | max_connections | current_connections + ---------------------+-------------+------+---------------------+------------+-----------+--------------+-----------+-----------------+--------------------- + gitlabhq_production | MASTER_HOST | 5432 | gitlabhq_production | | 20 | 0 | | 0 | 0 + pgbouncer | | 6432 | pgbouncer | pgbouncer | 2 | 0 | statement | 0 | 0 + (2 rows) + + type | user | database | state | addr | port | local_addr | local_port | connect_time | request_time | ptr | link | remote_pid | tls + ------+-----------+---------------------+---------+----------------+-------+------------+------------+---------------------+---------------------+-----------+------+------------+----- + C | pgbouncer | pgbouncer | active | 127.0.0.1 | 56846 | 127.0.0.1 | 6432 | 2017-08-21 18:09:59 | 2017-08-21 18:10:48 | 0x22b3880 | | 0 | + (2 rows) + ``` + +1. Verify the GitLab services are running: + + ```shell + sudo gitlab-ctl status + ``` + + The output should be similar to the following: + + ```plaintext + run: consul: (pid 31530) 77150s; run: log: (pid 31106) 77182s + run: logrotate: (pid 32613) 3357s; run: log: (pid 30107) 77500s + run: node-exporter: (pid 31550) 77149s; run: log: (pid 30138) 77493s + run: pgbouncer: (pid 32033) 75593s; run: log: (pid 31117) 77175s + run: pgbouncer-exporter: (pid 31558) 77148s; run: log: (pid 31498) 77156s + ``` + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +### Configure the internal load balancer + +If you're running more than one PgBouncer node as recommended, then at this time you'll need to set +up a TCP internal load balancer to serve each correctly. + +The following IP will be used as an example: + +- `10.6.0.20`: Internal Load Balancer + +Here's how you could do it with [HAProxy](https://www.haproxy.org/): + +```plaintext +global + log /dev/log local0 + log localhost local1 notice + log stdout format raw local0 + +defaults + log global + default-server inter 10s fall 3 rise 2 + balance leastconn + +frontend internal-pgbouncer-tcp-in + bind *:6432 + mode tcp + option tcplog + + default_backend pgbouncer + +backend pgbouncer + mode tcp + option tcp-check + + server pgbouncer1 10.6.0.21:6432 check + server pgbouncer2 10.6.0.22:6432 check + server pgbouncer3 10.6.0.23:6432 check +``` + +Refer to your preferred Load Balancer's documentation for further guidance. + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure Gitaly + +Deploying Gitaly in its own server can benefit GitLab installations that are +larger than a single machine. + +The Gitaly node requirements are dependent on customer data, specifically the number of +projects and their repository sizes. Two nodes are recommended as an absolute minimum. +Each Gitaly node should store no more than 5TB of data and have the number of +[`gitaly-ruby` workers](../gitaly/index.md#gitaly-ruby) set to 20% of available CPUs. +Additional nodes should be considered in conjunction with a review of expected +data size and spread based on the recommendations above. + +It is also strongly recommended that all Gitaly nodes be set up with SSD disks with +a throughput of at least 8,000 IOPS for read operations and 2,000 IOPS for write, +as Gitaly has heavy I/O. These IOPS values are recommended only as a starter as with +time they may be adjusted higher or lower depending on the scale of your environment's workload. +If you're running the environment on a Cloud provider, you may need to refer to +their documentation on how to configure IOPS correctly. + +Some things to note: + +- The GitLab Rails application shards repositories into [repository storages](../repository_storage_paths.md). +- A Gitaly server can host one or more storages. +- A GitLab server can use one or more Gitaly servers. +- Gitaly addresses must be specified in such a way that they resolve + correctly for ALL Gitaly clients. +- Gitaly servers must not be exposed to the public internet, as Gitaly's network + traffic is unencrypted by default. The use of a firewall is highly recommended + to restrict access to the Gitaly server. Another option is to + [use TLS](#gitaly-tls-support). + +TIP: **Tip:** +For more information about Gitaly's history and network architecture see the +[standalone Gitaly documentation](../gitaly/index.md). + +Note: **Note:** The token referred to throughout the Gitaly documentation is +just an arbitrary password selected by the administrator. It is unrelated to +tokens created for the GitLab API or other similar web API tokens. + +Below we describe how to configure two Gitaly servers, with IPs and +domain names: + +- `10.6.0.51`: Gitaly 1 (`gitaly1.internal`) +- `10.6.0.52`: Gitaly 2 (`gitaly2.internal`) + +The secret token is assumed to be `gitalysecret` and that +your GitLab installation has three repository storages: + +- `default` on Gitaly 1 +- `storage1` on Gitaly 1 +- `storage2` on Gitaly 2 + +On each node: + +1. [Download/Install](https://about.gitlab.com/install/) the Omnibus GitLab + package you want using **steps 1 and 2** from the GitLab downloads page but + **without** providing the `EXTERNAL_URL` value. +1. Edit `/etc/gitlab/gitlab.rb` to configure storage paths, enable + the network listener and configure the token: + + <!-- + updates to following example must also be made at + https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab + --> + + ```ruby + # /etc/gitlab/gitlab.rb + + # Gitaly and GitLab use two shared secrets for authentication, one to authenticate gRPC requests + # to Gitaly, and a second for authentication callbacks from GitLab-Shell to the GitLab internal API. + # The following two values must be the same as their respective values + # of the GitLab Rails application setup + gitaly['auth_token'] = 'gitlaysecret' + gitlab_shell['secret_token'] = 'shellsecret' + + # Avoid running unnecessary services on the Gitaly server + postgresql['enable'] = false + redis['enable'] = false + nginx['enable'] = false + puma['enable'] = false + unicorn['enable'] = false + sidekiq['enable'] = false + gitlab_workhorse['enable'] = false + grafana['enable'] = false + gitlab_exporter['enable'] = false + + # If you run a seperate monitoring node you can disable these services + alertmanager['enable'] = false + prometheus['enable'] = false + + # Prevent database connections during 'gitlab-ctl reconfigure' + gitlab_rails['rake_cache_clear'] = false + gitlab_rails['auto_migrate'] = false + + # Configure the gitlab-shell API callback URL. Without this, `git push` will + # fail. This can be your 'front door' GitLab URL or an internal load + # balancer. + # Don't forget to copy `/etc/gitlab/gitlab-secrets.json` from web server to Gitaly server. + gitlab_rails['internal_api_url'] = 'https://gitlab.example.com' + + # Make Gitaly accept connections on all network interfaces. You must use + # firewalls to restrict access to this address/port. + # Comment out following line if you only want to support TLS connections + gitaly['listen_addr'] = "0.0.0.0:8075" + + ## Enable service discovery for Prometheus + consul['enable'] = true + consul['monitoring_service_discovery'] = true + + # Set the network addresses that the exporters will listen on for monitoring + gitaly['prometheus_listen_addr'] = "0.0.0.0:9236" + node_exporter['listen_address'] = '0.0.0.0:9100' + gitlab_rails['prometheus_address'] = '10.6.0.81:9090' + + ## The IPs of the Consul server nodes + ## You can also use FQDNs and intermix them with IPs + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), + } + ``` + +1. Append the following to `/etc/gitlab/gitlab.rb` for each respective server: + 1. On `gitaly1.internal`: + + ```ruby + git_data_dirs({ + 'default' => { + 'path' => '/var/opt/gitlab/git-data' + }, + 'storage1' => { + 'path' => '/mnt/gitlab/git-data' + }, + }) + ``` + + 1. On `gitaly2.internal`: + + ```ruby + git_data_dirs({ + 'storage2' => { + 'path' => '/mnt/gitlab/git-data' + }, + }) + ``` + + <!-- + updates to following example must also be made at + https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab + --> + +1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). +1. Confirm that Gitaly can perform callbacks to the internal API: + + ```shell + sudo /opt/gitlab/embedded/service/gitlab-shell/bin/check -config /opt/gitlab/embedded/service/gitlab-shell/config.yml + ``` + +1. Verify the GitLab services are running: + + ```shell + sudo gitlab-ctl status + ``` + + The output should be similar to the following: + + ```plaintext + run: consul: (pid 30339) 77006s; run: log: (pid 29878) 77020s + run: gitaly: (pid 30351) 77005s; run: log: (pid 29660) 77040s + run: logrotate: (pid 7760) 3213s; run: log: (pid 29782) 77032s + run: node-exporter: (pid 30378) 77004s; run: log: (pid 29812) 77026s + ``` + +### Gitaly TLS support + +Gitaly supports TLS encryption. To be able to communicate +with a Gitaly instance that listens for secure connections you will need to use `tls://` URL +scheme in the `gitaly_address` of the corresponding storage entry in the GitLab configuration. + +You will need to bring your own certificates as this isn't provided automatically. +The certificate, or its certificate authority, must be installed on all Gitaly +nodes (including the Gitaly node using the certificate) and on all client nodes +that communicate with it following the procedure described in +[GitLab custom certificate configuration](https://docs.gitlab.com/omnibus/settings/ssl.html#install-custom-public-certificates). + +NOTE: **Note:** +The self-signed certificate must specify the address you use to access the +Gitaly server. If you are addressing the Gitaly server by a hostname, you can +either use the Common Name field for this, or add it as a Subject Alternative +Name. If you are addressing the Gitaly server by its IP address, you must add it +as a Subject Alternative Name to the certificate. +[gRPC does not support using an IP address as Common Name in a certificate](https://github.com/grpc/grpc/issues/2691). + +NOTE: **Note:** +It is possible to configure Gitaly servers with both an +unencrypted listening address `listen_addr` and an encrypted listening +address `tls_listen_addr` at the same time. This allows you to do a +gradual transition from unencrypted to encrypted traffic, if necessary. + +To configure Gitaly with TLS: + +1. Create the `/etc/gitlab/ssl` directory and copy your key and certificate there: + + ```shell + sudo mkdir -p /etc/gitlab/ssl + sudo chmod 755 /etc/gitlab/ssl + sudo cp key.pem cert.pem /etc/gitlab/ssl/ + sudo chmod 644 key.pem cert.pem + ``` + +1. Copy the cert to `/etc/gitlab/trusted-certs` so Gitaly will trust the cert when + calling into itself: + + ```shell + sudo cp /etc/gitlab/ssl/cert.pem /etc/gitlab/trusted-certs/ + ``` + +1. Edit `/etc/gitlab/gitlab.rb` and add: + + <!-- + updates to following example must also be made at + https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab + --> + + ```ruby + gitaly['tls_listen_addr'] = "0.0.0.0:9999" + gitaly['certificate_path'] = "/etc/gitlab/ssl/cert.pem" + gitaly['key_path'] = "/etc/gitlab/ssl/key.pem" + ``` + +1. Delete `gitaly['listen_addr']` to allow only encrypted connections. +1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure Sidekiq + +Sidekiq requires connection to the Redis, PostgreSQL and Gitaly instance. +The following IPs will be used as an example: + +- `10.6.0.71`: Sidekiq 1 +- `10.6.0.72`: Sidekiq 2 +- `10.6.0.73`: Sidekiq 3 +- `10.6.0.74`: Sidekiq 4 + +To configure the Sidekiq nodes, one each one: + +1. SSH into the Sidekiq server. +1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab package +you want using steps 1 and 2 from the GitLab downloads page. +**Do not complete any other steps on the download page.** +1. Open `/etc/gitlab/gitlab.rb` with your editor: + + ```ruby + ######################################## + ##### Services Disabled ### + ######################################## + + nginx['enable'] = false + grafana['enable'] = false + prometheus['enable'] = false + gitlab_rails['auto_migrate'] = false + alertmanager['enable'] = false + gitaly['enable'] = false + gitlab_workhorse['enable'] = false + nginx['enable'] = false + puma['enable'] = false + postgres_exporter['enable'] = false + postgresql['enable'] = false + redis['enable'] = false + redis_exporter['enable'] = false + gitlab_exporter['enable'] = false + + ######################################## + #### Redis ### + ######################################## + + ## Must be the same in every sentinel node + redis['master_name'] = 'gitlab-redis' + + ## The same password for Redis authentication you set up for the master node. + redis['master_password'] = '<redis_primary_password>' + + ## A list of sentinels with `host` and `port` + gitlab_rails['redis_sentinels'] = [ + {'host' => '10.6.0.11', 'port' => 26379}, + {'host' => '10.6.0.12', 'port' => 26379}, + {'host' => '10.6.0.13', 'port' => 26379}, + ] + + ####################################### + ### Gitaly ### + ####################################### + + git_data_dirs({ + 'default' => { 'gitaly_address' => 'tcp://gitaly1.internal:8075' }, + 'storage1' => { 'gitaly_address' => 'tcp://gitaly1.internal:8075' }, + 'storage2' => { 'gitaly_address' => 'tcp://gitaly2.internal:8075' }, + }) + gitlab_rails['gitaly_token'] = 'YOUR_TOKEN' + + ####################################### + ### Postgres ### + ####################################### + gitlab_rails['db_host'] = '10.6.0.20' # internal load balancer IP + gitlab_rails['db_port'] = 6432 + gitlab_rails['db_password'] = '<postgresql_user_password>' + gitlab_rails['db_adapter'] = 'postgresql' + gitlab_rails['db_encoding'] = 'unicode' + gitlab_rails['auto_migrate'] = false + + ####################################### + ### Sidekiq configuration ### + ####################################### + sidekiq['listen_address'] = "0.0.0.0" + + ####################################### + ### Monitoring configuration ### + ####################################### + consul['enable'] = true + consul['monitoring_service_discovery'] = true + + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13) + } + + # Set the network addresses that the exporters will listen on + node_exporter['listen_address'] = '0.0.0.0:9100' + + # Rails Status for prometheus + gitlab_rails['monitoring_whitelist'] = ['10.6.0.81/32', '127.0.0.0/8'] + gitlab_rails['prometheus_address'] = '10.6.0.81:9090' + ``` + +1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). +1. Verify the GitLab services are running: + + ```shell + sudo gitlab-ctl status + ``` + + The output should be similar to the following: + + ```plaintext + run: consul: (pid 30114) 77353s; run: log: (pid 29756) 77367s + run: logrotate: (pid 9898) 3561s; run: log: (pid 29653) 77380s + run: node-exporter: (pid 30134) 77353s; run: log: (pid 29706) 77372s + run: sidekiq: (pid 30142) 77351s; run: log: (pid 29638) 77386s + ``` + +TIP: **Tip:** +You can also run [multiple Sidekiq processes](../operations/extra_sidekiq_processes.md). + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure GitLab Rails + +NOTE: **Note:** +In our architectures we run each GitLab Rails node using the Puma webserver +and have its number of workers set to 90% of available CPUs along with four threads. For +nodes that are running Rails with other components the worker value should be reduced +accordingly where we've found 50% achieves a good balance but this is dependent +on workload. + +This section describes how to configure the GitLab application (Rails) component. +On each node perform the following: + +1. If you're [using NFS](#configure-nfs-optional): + + 1. If necessary, install the NFS client utility packages using the following + commands: + + ```shell + # Ubuntu/Debian + apt-get install nfs-common + + # CentOS/Red Hat + yum install nfs-utils nfs-utils-lib + ``` + + 1. Specify the necessary NFS mounts in `/etc/fstab`. + The exact contents of `/etc/fstab` will depend on how you chose + to configure your NFS server. See the [NFS documentation](../high_availability/nfs.md) + for examples and the various options. + + 1. Create the shared directories. These may be different depending on your NFS + mount locations. + + ```shell + mkdir -p /var/opt/gitlab/.ssh /var/opt/gitlab/gitlab-rails/uploads /var/opt/gitlab/gitlab-rails/shared /var/opt/gitlab/gitlab-ci/builds /var/opt/gitlab/git-data + ``` + +1. Download/install Omnibus GitLab using **steps 1 and 2** from + [GitLab downloads](https://about.gitlab.com/install/). Do not complete other + steps on the download page. +1. Create/edit `/etc/gitlab/gitlab.rb` and use the following configuration. + To maintain uniformity of links across nodes, the `external_url` + on the application server should point to the external URL that users will use + to access GitLab. This would be the URL of the [external load balancer](#configure-the-external-load-balancer) + which will route traffic to the GitLab application server: + + ```ruby + external_url 'https://gitlab.example.com' + + # Gitaly and GitLab use two shared secrets for authentication, one to authenticate gRPC requests + # to Gitaly, and a second for authentication callbacks from GitLab-Shell to the GitLab internal API. + # The following two values must be the same as their respective values + # of the Gitaly setup + gitlab_rails['gitaly_token'] = 'gitalyecret' + gitlab_shell['secret_token'] = 'shellsecret' + + git_data_dirs({ + 'default' => { 'gitaly_address' => 'tcp://gitaly1.internal:8075' }, + 'storage1' => { 'gitaly_address' => 'tcp://gitaly1.internal:8075' }, + 'storage2' => { 'gitaly_address' => 'tcp://gitaly2.internal:8075' }, + }) + + ## Disable components that will not be on the GitLab application server + roles ['application_role'] + gitaly['enable'] = false + nginx['enable'] = true + sidekiq['enable'] = false + + ## PostgreSQL connection details + # Disable PostgreSQL on the application node + postgresql['enable'] = false + gitlab_rails['db_host'] = '10.6.0.20' # internal load balancer IP + gitlab_rails['db_port'] = 6432 + gitlab_rails['db_password'] = '<postgresql_user_password>' + gitlab_rails['auto_migrate'] = false + + ## Redis connection details + ## Must be the same in every sentinel node + redis['master_name'] = 'gitlab-redis' + + ## The same password for Redis authentication you set up for the Redis primary node. + redis['master_password'] = '<redis_primary_password>' + + ## A list of sentinels with `host` and `port` + gitlab_rails['redis_sentinels'] = [ + {'host' => '10.6.0.11', 'port' => 26379}, + {'host' => '10.6.0.12', 'port' => 26379}, + {'host' => '10.6.0.13', 'port' => 26379} + ] + + ## Enable service discovery for Prometheus + consul['enable'] = true + consul['monitoring_service_discovery'] = true + + # Set the network addresses that the exporters used for monitoring will listen on + node_exporter['listen_address'] = '0.0.0.0:9100' + gitlab_workhorse['prometheus_listen_addr'] = '0.0.0.0:9229' + sidekiq['listen_address'] = "0.0.0.0" + puma['listen'] = '0.0.0.0' + + ## The IPs of the Consul server nodes + ## You can also use FQDNs and intermix them with IPs + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), + } + + # Add the monitoring node's IP address to the monitoring whitelist and allow it to + # scrape the NGINX metrics + gitlab_rails['monitoring_whitelist'] = ['10.6.0.81/32', '127.0.0.0/8'] + nginx['status']['options']['allow'] = ['10.6.0.81/32', '127.0.0.0/8'] + gitlab_rails['prometheus_address'] = '10.6.0.81:9090' + + ## Uncomment and edit the following options if you have set up NFS + ## + ## Prevent GitLab from starting if NFS data mounts are not available + ## + #high_availability['mountpoint'] = '/var/opt/gitlab/git-data' + ## + ## Ensure UIDs and GIDs match between servers for permissions via NFS + ## + #user['uid'] = 9000 + #user['gid'] = 9000 + #web_server['uid'] = 9001 + #web_server['gid'] = 9001 + #registry['uid'] = 9002 + #registry['gid'] = 9002 + ``` + +1. If you're using [Gitaly with TLS support](#gitaly-tls-support), make sure the + `git_data_dirs` entry is configured with `tls` instead of `tcp`: + + ```ruby + git_data_dirs({ + 'default' => { 'gitaly_address' => 'tls://gitaly1.internal:9999' }, + 'storage1' => { 'gitaly_address' => 'tls://gitaly1.internal:9999' }, + 'storage2' => { 'gitaly_address' => 'tls://gitaly2.internal:9999' }, + }) + ``` + + 1. Copy the cert into `/etc/gitlab/trusted-certs`: + + ```shell + sudo cp cert.pem /etc/gitlab/trusted-certs/ + ``` + +1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). +1. Run `sudo gitlab-rake gitlab:gitaly:check` to confirm the node can connect to Gitaly. +1. Tail the logs to see the requests: + + ```shell + sudo gitlab-ctl tail gitaly + ``` + +1. Verify the GitLab services are running: + + ```shell + sudo gitlab-ctl status + ``` + + The output should be similar to the following: + + ```plaintext + run: consul: (pid 4890) 8647s; run: log: (pid 29962) 79128s + run: gitlab-exporter: (pid 4902) 8647s; run: log: (pid 29913) 79134s + run: gitlab-workhorse: (pid 4904) 8646s; run: log: (pid 29713) 79155s + run: logrotate: (pid 12425) 1446s; run: log: (pid 29798) 79146s + run: nginx: (pid 4925) 8646s; run: log: (pid 29726) 79152s + run: node-exporter: (pid 4931) 8645s; run: log: (pid 29855) 79140s + run: puma: (pid 4936) 8645s; run: log: (pid 29656) 79161s + ``` + +NOTE: **Note:** +When you specify `https` in the `external_url`, as in the example +above, GitLab assumes you have SSL certificates in `/etc/gitlab/ssl/`. If +certificates are not present, NGINX will fail to start. See the +[NGINX documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https) +for more information. + +### GitLab Rails post-configuration + +1. Ensure that all migrations ran: + + ```shell + gitlab-rake gitlab:db:configure + ``` + + NOTE: **Note:** + If you encounter a `rake aborted!` error stating that PgBouncer is failing to connect to + PostgreSQL it may be that your PgBouncer node's IP address is missing from + PostgreSQL's `trust_auth_cidr_addresses` in `gitlab.rb` on your database nodes. See + [PgBouncer error `ERROR: pgbouncer cannot connect to server`](troubleshooting.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server) + in the Troubleshooting section before proceeding. + +1. [Configure fast lookup of authorized SSH keys in the database](../operations/fast_ssh_key_lookup.md). + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure Prometheus + +The Omnibus GitLab package can be used to configure a standalone Monitoring node +running [Prometheus](../monitoring/prometheus/index.md) and +[Grafana](../monitoring/performance/grafana_configuration.md): + +1. SSH into the Monitoring node. +1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab + package you want using **steps 1 and 2** from the GitLab downloads page. + Do not complete any other steps on the download page. +1. Edit `/etc/gitlab/gitlab.rb` and add the contents: + + ```ruby + external_url 'http://gitlab.example.com' + + # Disable all other services + gitlab_rails['auto_migrate'] = false + alertmanager['enable'] = false + gitaly['enable'] = false + gitlab_exporter['enable'] = false + gitlab_workhorse['enable'] = false + nginx['enable'] = true + postgres_exporter['enable'] = false + postgresql['enable'] = false + redis['enable'] = false + redis_exporter['enable'] = false + sidekiq['enable'] = false + puma['enable'] = false + unicorn['enable'] = false + node_exporter['enable'] = false + gitlab_exporter['enable'] = false + + # Enable Prometheus + prometheus['enable'] = true + prometheus['listen_address'] = '0.0.0.0:9090' + prometheus['monitor_kubernetes'] = false + + # Enable Login form + grafana['disable_login_form'] = false + + # Enable Grafana + grafana['enable'] = true + grafana['admin_password'] = '<grafana_password>' + + # Enable service discovery for Prometheus + consul['enable'] = true + consul['monitoring_service_discovery'] = true + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13) + } + ``` + +1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). +1. In the GitLab UI, set `admin/application_settings/metrics_and_profiling` > Metrics - Grafana to `/-/grafana` to + `http[s]://<MONITOR NODE>/-/grafana`. +1. Verify the GitLab services are running: + + ```shell + sudo gitlab-ctl status + ``` + + The output should be similar to the following: + + ```plaintext + run: consul: (pid 31637) 17337s; run: log: (pid 29748) 78432s + run: grafana: (pid 31644) 17337s; run: log: (pid 29719) 78438s + run: logrotate: (pid 31809) 2936s; run: log: (pid 29581) 78462s + run: nginx: (pid 31665) 17335s; run: log: (pid 29556) 78468s + run: prometheus: (pid 31672) 17335s; run: log: (pid 29633) 78456s + ``` + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure the object storage + +GitLab supports using an object storage service for holding numerous types of data. +It's recommended over [NFS](#configure-nfs-optional) and in general it's better +in larger setups as object storage is typically much more performant, reliable, +and scalable. + +Object storage options that GitLab has tested, or is aware of customers using include: + +- SaaS/Cloud solutions such as [Amazon S3](https://aws.amazon.com/s3/), [Google cloud storage](https://cloud.google.com/storage). +- On-premises hardware and appliances from various storage vendors. +- MinIO. There is [a guide to deploying this](https://docs.gitlab.com/charts/advanced/external-object-storage/minio.html) within our Helm Chart documentation. + +For configuring GitLab to use Object Storage refer to the following guides +based on what features you intend to use: + +1. Configure [object storage for backups](../../raketasks/backup_restore.md#uploading-backups-to-a-remote-cloud-storage). +1. Configure [object storage for job artifacts](../job_artifacts.md#using-object-storage) + including [incremental logging](../job_logs.md#new-incremental-logging-architecture). +1. Configure [object storage for LFS objects](../lfs/index.md#storing-lfs-objects-in-remote-object-storage). +1. Configure [object storage for uploads](../uploads.md#using-object-storage-core-only). +1. Configure [object storage for merge request diffs](../merge_request_diffs.md#using-object-storage). +1. Configure [object storage for Container Registry](../packages/container_registry.md#use-object-storage) (optional feature). +1. Configure [object storage for Mattermost](https://docs.mattermost.com/administration/config-settings.html#file-storage) (optional feature). +1. Configure [object storage for packages](../packages/index.md#using-object-storage) (optional feature). **(PREMIUM ONLY)** +1. Configure [object storage for Dependency Proxy](../packages/dependency_proxy.md#using-object-storage) (optional feature). **(PREMIUM ONLY)** +1. Configure [object storage for Pseudonymizer](../pseudonymizer.md#configuration) (optional feature). **(ULTIMATE ONLY)** +1. Configure [object storage for autoscale Runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional - for improved performance). +1. Configure [object storage for Terraform state files](../terraform_state.md#using-object-storage-core-only). + +Using separate buckets for each data type is the recommended approach for GitLab. + +A limitation of our configuration is that each use of object storage is separately configured. +[We have an issue for improving this](https://gitlab.com/gitlab-org/gitlab/-/issues/23345) +and easily using one bucket with separate folders is one improvement that this might bring. + +There is at least one specific issue with using the same bucket: +when GitLab is deployed with the Helm chart restore from backup +[will not properly function](https://docs.gitlab.com/charts/advanced/external-object-storage/#lfs-artifacts-uploads-packages-external-diffs-pseudonymizer) +unless separate buckets are used. + +One risk of using a single bucket would be if your organization decided to +migrate GitLab to the Helm deployment in the future. GitLab would run, but the situation with +backups might not be realized until the organization had a critical requirement for the backups to +work. + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure NFS (optional) + +[Object storage](#configure-the-object-storage), along with [Gitaly](#configure-gitaly) +are recommended over NFS wherever possible for improved performance. If you intend +to use GitLab Pages, this currently [requires NFS](troubleshooting.md#gitlab-pages-requires-nfs). + +See how to [configure NFS](../high_availability/nfs.md). + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Troubleshooting + +See the [troubleshooting documentation](troubleshooting.md). + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> diff --git a/doc/administration/reference_architectures/index.md b/doc/administration/reference_architectures/index.md index 623d7f3f776..8fde71a66bf 100644 --- a/doc/administration/reference_architectures/index.md +++ b/doc/administration/reference_architectures/index.md @@ -38,7 +38,8 @@ When scaling GitLab, there are several factors to consider: - A load balancer is added in front to distribute traffic across the application nodes. - The application nodes connects to a shared file server and PostgreSQL and Redis services on the backend. -NOTE: **Note:** Depending on your workflow, the following recommended +NOTE: **Note:** +Depending on your workflow, the following recommended reference architectures may need to be adapted accordingly. Your workload is influenced by factors including how active your users are, how much automation you use, mirroring, and repository/change size. Additionally the @@ -120,13 +121,13 @@ As long as at least one of each component is online and capable of handling the ### Automated database failover **(PREMIUM ONLY)** > - Level of complexity: **High** -> - Required domain knowledge: PgBouncer, Repmgr, shared storage, distributed systems +> - Required domain knowledge: PgBouncer, Repmgr or Patroni, shared storage, distributed systems > - Supported tiers: [GitLab Premium and Ultimate](https://about.gitlab.com/pricing/) By adding automatic failover for database systems, you can enable higher uptime with additional database nodes. This extends the default database with cluster management and failover policies. -[PgBouncer in conjunction with Repmgr](../postgresql/replication_and_failover.md) +[PgBouncer in conjunction with Repmgr or Patroni](../postgresql/replication_and_failover.md) is recommended. ### Instance level replication with GitLab Geo **(PREMIUM ONLY)** @@ -164,6 +165,7 @@ column. | [PostgreSQL](../../development/architecture.md#postgresql) | Database | [PostgreSQL configuration](https://docs.gitlab.com/omnibus/settings/database.html) | Yes | | [PgBouncer](../../development/architecture.md#pgbouncer) | Database connection pooler | [PgBouncer configuration](../high_availability/pgbouncer.md#running-pgbouncer-as-part-of-a-non-ha-gitlab-installation) **(PREMIUM ONLY)** | Yes | | Repmgr | PostgreSQL cluster management and failover | [PostgreSQL and Repmgr configuration](../postgresql/replication_and_failover.md) | Yes | +| Patroni | An alternative PostgreSQL cluster management and failover | [PostgreSQL and Patroni configuration](../postgresql/replication_and_failover.md#patroni) | Yes | | [Redis](../../development/architecture.md#redis) ([3](#footnotes)) | Key/value store for fast data lookup and caching | [Redis configuration](../high_availability/redis.md) | Yes | | Redis Sentinel | Redis | [Redis Sentinel configuration](../high_availability/redis.md) | Yes | | [Gitaly](../../development/architecture.md#gitaly) ([2](#footnotes)) ([7](#footnotes)) | Provides access to Git repositories | [Gitaly configuration](../gitaly/index.md#run-gitaly-on-its-own-server) | Yes | @@ -209,7 +211,7 @@ cluster with the Rails nodes broken down into a number of smaller Pods across th For medium sized installs (3,000 - 5,000) we suggest one Redis cluster for all classes and that Redis Sentinel is hosted alongside Consul. For larger architectures (10,000 users or more) we suggest running a separate - [Redis Cluster](../high_availability/redis.md#running-multiple-redis-clusters) for the Cache class + [Redis Cluster](../redis/replication_and_failover.md#running-multiple-redis-clusters) for the Cache class and another for the Queues and Shared State classes respectively. We also recommend that you run the Redis Sentinel clusters separately for each Redis Cluster. diff --git a/doc/administration/reference_architectures/troubleshooting.md b/doc/administration/reference_architectures/troubleshooting.md index 15e377fe183..35bdb65c810 100644 --- a/doc/administration/reference_architectures/troubleshooting.md +++ b/doc/administration/reference_architectures/troubleshooting.md @@ -1,4 +1,4 @@ -# Troubleshooting a reference architecture set up +# Troubleshooting a reference architecture setup This page serves as the troubleshooting documentation if you followed one of the [reference architectures](index.md#reference-architectures). @@ -17,7 +17,7 @@ with the Fog library that GitLab uses. Symptoms include: ### GitLab Pages requires NFS If you intend to use [GitLab Pages](../../user/project/pages/index.md), this currently requires -[NFS](../high_availability/nfs.md). There is [work in progress](https://gitlab.com/gitlab-org/gitlab-pages/issues/196) +[NFS](../high_availability/nfs.md). There is [work in progress](https://gitlab.com/gitlab-org/gitlab-pages/-/issues/196) to remove this dependency. In the future, GitLab Pages may use [object storage](https://gitlab.com/gitlab-org/gitlab/-/issues/208135). @@ -86,8 +86,117 @@ a workaround, in the mean time, to ## Troubleshooting Redis -If the application node cannot connect to the Redis node, check your firewall rules and -make sure Redis can accept TCP connections under port `6379`. +There are a lot of moving parts that needs to be taken care carefully +in order for the HA setup to work as expected. + +Before proceeding with the troubleshooting below, check your firewall rules: + +- Redis machines + - Accept TCP connection in `6379` + - Connect to the other Redis machines via TCP in `6379` +- Sentinel machines + - Accept TCP connection in `26379` + - Connect to other Sentinel machines via TCP in `26379` + - Connect to the Redis machines via TCP in `6379` + +### Troubleshooting Redis replication + +You can check if everything is correct by connecting to each server using +`redis-cli` application, and sending the `info replication` command as below. + +```shell +/opt/gitlab/embedded/bin/redis-cli -h <redis-host-or-ip> -a '<redis-password>' info replication +``` + +When connected to a `Primary` Redis, you will see the number of connected +`replicas`, and a list of each with connection details: + +```plaintext +# Replication +role:master +connected_replicas:1 +replica0:ip=10.133.5.21,port=6379,state=online,offset=208037514,lag=1 +master_repl_offset:208037658 +repl_backlog_active:1 +repl_backlog_size:1048576 +repl_backlog_first_byte_offset:206989083 +repl_backlog_histlen:1048576 +``` + +When it's a `replica`, you will see details of the primary connection and if +its `up` or `down`: + +```plaintext +# Replication +role:replica +master_host:10.133.1.58 +master_port:6379 +master_link_status:up +master_last_io_seconds_ago:1 +master_sync_in_progress:0 +replica_repl_offset:208096498 +replica_priority:100 +replica_read_only:1 +connected_replicas:0 +master_repl_offset:0 +repl_backlog_active:0 +repl_backlog_size:1048576 +repl_backlog_first_byte_offset:0 +repl_backlog_histlen:0 +``` + +### Troubleshooting Sentinel + +If you get an error like: `Redis::CannotConnectError: No sentinels available.`, +there may be something wrong with your configuration files or it can be related +to [this issue](https://github.com/redis/redis-rb/issues/531). + +You must make sure you are defining the same value in `redis['master_name']` +and `redis['master_pasword']` as you defined for your sentinel node. + +The way the Redis connector `redis-rb` works with sentinel is a bit +non-intuitive. We try to hide the complexity in omnibus, but it still requires +a few extra configurations. + +--- + +To make sure your configuration is correct: + +1. SSH into your GitLab application server +1. Enter the Rails console: + + ```shell + # For Omnibus installations + sudo gitlab-rails console + + # For source installations + sudo -u git rails console -e production + ``` + +1. Run in the console: + + ```ruby + redis = Redis.new(Gitlab::Redis::SharedState.params) + redis.info + ``` + + Keep this screen open and try to simulate a failover below. + +1. To simulate a failover on primary Redis, SSH into the Redis server and run: + + ```shell + # port must match your primary redis port, and the sleep time must be a few seconds bigger than defined one + redis-cli -h localhost -p 6379 DEBUG sleep 20 + ``` + +1. Then back in the Rails console from the first step, run: + + ```ruby + redis.info + ``` + + You should see a different port after a few seconds delay + (the failover/reconnect time). ## Troubleshooting Gitaly @@ -291,7 +400,7 @@ unset https_proxy When updating the `gitaly['listen_addr']` or `gitaly['prometheus_listen_addr']` values, Gitaly may continue to listen on the old address after a `sudo gitlab-ctl reconfigure`. -When this occurs, performing a `sudo gitlab-ctl restart` will resolve the issue. This will no longer be necessary after [this issue](https://gitlab.com/gitlab-org/gitaly/issues/2521) is resolved. +When this occurs, performing a `sudo gitlab-ctl restart` will resolve the issue. This will no longer be necessary after [this issue](https://gitlab.com/gitlab-org/gitaly/-/issues/2521) is resolved. ### Permission denied errors appearing in Gitaly logs when accessing repositories from a standalone Gitaly node @@ -327,3 +436,135 @@ or ```shell curl http[s]://localhost:<EXPORTER LISTENING PORT>/-/metric ``` + +## Troubleshooting PgBouncer + +In case you are experiencing any issues connecting through PgBouncer, the first place to check is always the logs: + +```shell +sudo gitlab-ctl tail pgbouncer +``` + +Additionally, you can check the output from `show databases` in the [administrative console](#pgbouncer-administrative-console). In the output, you would expect to see values in the `host` field for the `gitlabhq_production` database. Additionally, `current_connections` should be greater than 1. + +### PgBouncer administrative console + +As part of Omnibus GitLab, the `gitlab-ctl pgb-console` command is provided to automatically connect to the PgBouncer administrative console. See the [PgBouncer documentation](https://www.pgbouncer.org/usage.html#admin-console) for detailed instructions on how to interact with the console. + +To start a session: + +```shell +sudo gitlab-ctl pgb-console +``` + +The password you will be prompted for is the `pgbouncer_user_password` + +To get some basic information about the instance, run + +```shell +pgbouncer=# show databases; show clients; show servers; + name | host | port | database | force_user | pool_size | reserve_pool | pool_mode | max_connections | current_connections +---------------------+-----------+------+---------------------+------------+-----------+--------------+-----------+-----------------+--------------------- + gitlabhq_production | 127.0.0.1 | 5432 | gitlabhq_production | | 100 | 5 | | 0 | 1 + pgbouncer | | 6432 | pgbouncer | pgbouncer | 2 | 0 | statement | 0 | 0 +(2 rows) + + type | user | database | state | addr | port | local_addr | local_port | connect_time | request_time | ptr | link +| remote_pid | tls +------+-----------+---------------------+--------+-----------+-------+------------+------------+---------------------+---------------------+-----------+------ ++------------+----- + C | gitlab | gitlabhq_production | active | 127.0.0.1 | 44590 | 127.0.0.1 | 6432 | 2018-04-24 22:13:10 | 2018-04-24 22:17:10 | 0x12444c0 | +| 0 | + C | gitlab | gitlabhq_production | active | 127.0.0.1 | 44592 | 127.0.0.1 | 6432 | 2018-04-24 22:13:10 | 2018-04-24 22:17:10 | 0x12447c0 | +| 0 | + C | gitlab | gitlabhq_production | active | 127.0.0.1 | 44594 | 127.0.0.1 | 6432 | 2018-04-24 22:13:10 | 2018-04-24 22:17:10 | 0x1244940 | +| 0 | + C | gitlab | gitlabhq_production | active | 127.0.0.1 | 44706 | 127.0.0.1 | 6432 | 2018-04-24 22:14:22 | 2018-04-24 22:16:31 | 0x1244ac0 | +| 0 | + C | gitlab | gitlabhq_production | active | 127.0.0.1 | 44708 | 127.0.0.1 | 6432 | 2018-04-24 22:14:22 | 2018-04-24 22:15:15 | 0x1244c40 | +| 0 | + C | gitlab | gitlabhq_production | active | 127.0.0.1 | 44794 | 127.0.0.1 | 6432 | 2018-04-24 22:15:15 | 2018-04-24 22:15:15 | 0x1244dc0 | +| 0 | + C | gitlab | gitlabhq_production | active | 127.0.0.1 | 44798 | 127.0.0.1 | 6432 | 2018-04-24 22:15:15 | 2018-04-24 22:16:31 | 0x1244f40 | +| 0 | + C | pgbouncer | pgbouncer | active | 127.0.0.1 | 44660 | 127.0.0.1 | 6432 | 2018-04-24 22:13:51 | 2018-04-24 22:17:12 | 0x1244640 | +| 0 | +(8 rows) + + type | user | database | state | addr | port | local_addr | local_port | connect_time | request_time | ptr | link | rem +ote_pid | tls +------+--------+---------------------+-------+-----------+------+------------+------------+---------------------+---------------------+-----------+------+---- +--------+----- + S | gitlab | gitlabhq_production | idle | 127.0.0.1 | 5432 | 127.0.0.1 | 35646 | 2018-04-24 22:15:15 | 2018-04-24 22:17:10 | 0x124dca0 | | + 19980 | +(1 row) +``` + +### Message: `LOG: invalid CIDR mask in address` + +See the suggested fix [in Geo documentation](../geo/replication/troubleshooting.md#message-log--invalid-cidr-mask-in-address). + +### Message: `LOG: invalid IP mask "md5": Name or service not known` + +See the suggested fix [in Geo documentation](../geo/replication/troubleshooting.md#message-log--invalid-ip-mask-md5-name-or-service-not-known). + +## Troubleshooting PostgreSQL + +In case you are experiencing any issues connecting through PgBouncer, the first place to check is always the logs: + +```shell +sudo gitlab-ctl tail postgresql +``` + +### Consul and PostgreSQL changes not taking effect + +Due to the potential impacts, `gitlab-ctl reconfigure` only reloads Consul and PostgreSQL, it will not restart the services. However, not all changes can be activated by reloading. + +To restart either service, run `gitlab-ctl restart SERVICE` + +For PostgreSQL, it is usually safe to restart the master node by default. Automatic failover defaults to a 1 minute timeout. Provided the database returns before then, nothing else needs to be done. To be safe, you can stop `repmgrd` on the standby nodes first with `gitlab-ctl stop repmgrd`, then start afterwards with `gitlab-ctl start repmgrd`. + +On the Consul server nodes, it is important to restart the Consul service in a controlled fashion. Read our [Consul documentation](../high_availability/consul.md#restarting-the-server-cluster) for instructions on how to restart the service. + +### `gitlab-ctl repmgr-check-master` command produces errors + +If this command displays errors about database permissions it is likely that something failed during +install, resulting in the `gitlab-consul` database user getting incorrect permissions. Follow these +steps to fix the problem: + +1. On the master database node, connect to the database prompt - `gitlab-psql -d template1` +1. Delete the `gitlab-consul` user - `DROP USER "gitlab-consul";` +1. Exit the database prompt - `\q` +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) and the user will be re-added with the proper permissions. +1. Change to the `gitlab-consul` user - `su - gitlab-consul` +1. Try the check command again - `gitlab-ctl repmgr-check-master`. + +Now there should not be errors. If errors still occur then there is another problem. + +### PgBouncer error `ERROR: pgbouncer cannot connect to server` + +You may get this error when running `gitlab-rake gitlab:db:configure` or you +may see the error in the PgBouncer log file. + +```plaintext +PG::ConnectionBad: ERROR: pgbouncer cannot connect to server +``` + +The problem may be that your PgBouncer node's IP address is not included in the +`trust_auth_cidr_addresses` setting in `/etc/gitlab/gitlab.rb` on the database nodes. + +You can confirm that this is the issue by checking the PostgreSQL log on the master +database node. If you see the following error then `trust_auth_cidr_addresses` +is the problem. + +```plaintext +2018-03-29_13:59:12.11776 FATAL: no pg_hba.conf entry for host "123.123.123.123", user "pgbouncer", database "gitlabhq_production", SSL off +``` + +To fix the problem, add the IP address to `/etc/gitlab/gitlab.rb`. + +```ruby +postgresql['trust_auth_cidr_addresses'] = %w(123.123.123.123/32 <other_cidrs>) +``` + +[Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. diff --git a/doc/administration/repository_storage_paths.md b/doc/administration/repository_storage_paths.md index 87f901becf5..5520eeace0a 100644 --- a/doc/administration/repository_storage_paths.md +++ b/doc/administration/repository_storage_paths.md @@ -1,3 +1,10 @@ +--- +stage: Create +group: Gitaly +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: reference, howto +--- + # Repository storage paths > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/4578) in GitLab 8.10. @@ -60,7 +67,8 @@ files and add the full paths of the alternative repository storage paths. In the example below, we add two more mount points that are named `nfs_1` and `nfs_2` respectively. -NOTE: **Note:** This example uses NFS. We do not recommend using EFS for storage as it may impact GitLab's performance. See the [relevant documentation](high_availability/nfs.md#avoid-using-awss-elastic-file-system-efs) for more details. +NOTE: **Note:** +This example uses NFS. We do not recommend using EFS for storage as it may impact GitLab's performance. See the [relevant documentation](high_availability/nfs.md#avoid-using-awss-elastic-file-system-efs) for more details. **For installations from source** @@ -110,7 +118,10 @@ Once you set the multiple storage paths, you can choose where new repositories will be stored under **Admin Area > Settings > Repository > Repository storage > Storage nodes for new repositories**. - +Each storage can be assigned a weight from 0-100. When a new project is created, these +weights are used to determine the storage location the repository will be created on. + + Beginning with GitLab 8.13.4, multiple paths can be chosen. New repositories will be randomly placed on one of the selected paths. diff --git a/doc/administration/repository_storage_types.md b/doc/administration/repository_storage_types.md index 825da08b66e..1c036cfe2ac 100644 --- a/doc/administration/repository_storage_types.md +++ b/doc/administration/repository_storage_types.md @@ -1,3 +1,10 @@ +--- +stage: Create +group: Gitaly +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: reference, howto +--- + # Repository storage types **(CORE ONLY)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/28283) in GitLab 10.0. diff --git a/doc/administration/server_hooks.md b/doc/administration/server_hooks.md index 6df0f187a42..2fea000b799 100644 --- a/doc/administration/server_hooks.md +++ b/doc/administration/server_hooks.md @@ -8,130 +8,183 @@ disqus_identifier: 'https://docs.gitlab.com/ee/administration/custom_hooks.html' # Server hooks **(CORE ONLY)** -> **Notes:** -> -> - Server hooks were [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/196051) in GitLab 12.8 replacing Custom Hooks. -> - Server hooks must be configured on the filesystem of the GitLab server. Only GitLab server administrators will be able to complete these tasks. Please explore [webhooks](../user/project/integrations/webhooks.md) and [GitLab CI/CD](../ci/README.md) as an option if you do not have filesystem access. For a user-configurable Git hook interface, see [Push Rules](../push_rules/push_rules.md), available in GitLab Starter **(STARTER)**. -> - Server hooks won't be replicated to secondary nodes if you use [GitLab Geo](geo/replication/index.md). - -Git natively supports hooks that are executed on different actions. These hooks run -on the server and can be used to enforce specific commit policies or perform other -tasks based on the state of the repository. - -Examples of server-side Git hooks include `pre-receive`, `post-receive`, and `update`. -See [Git SCM Server-Side Hooks](https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks#Server-Side-Hooks) +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/196051) in GitLab 12.8 replacing Custom Hooks. + +Git supports hooks that are executed on different actions. These hooks run on the server and can be +used to enforce specific commit policies or perform other tasks based on the state of the +repository. + +Git supports the following hooks: + +- `pre-receive` +- `post-receive` +- `update` + +See [the Git documentation](https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks#_server_side_hooks) for more information about each hook type. -## Create a server hook for a repository +Server-side Git hooks can be configured for: -Server-side Git hooks are typically placed in the repository's `hooks` -subdirectory. In GitLab, hook directories are symlinked to the GitLab Shell -`hooks` directory for ease of maintenance between GitLab Shell upgrades. -Server hooks are implemented differently, but the behavior is exactly the same -once the hook is created. +- [A single repository](#create-a-server-hook-for-a-repository). +- [All repositories](#create-a-global-server-hook-for-all-repositories). + +Note the following about server hooks: + +- Server hooks must be configured on the file system of the GitLab server. Only GitLab server + administrators are able to complete these tasks. If you don't have file system access, see + possible alternatives such as: + - [Webhooks](../user/project/integrations/webhooks.md). + - [GitLab CI/CD](../ci/README.md). + - [Push Rules](../push_rules/push_rules.md), for a user-configurable Git hook + interface. **(STARTER)** +- Server hooks aren't replicated to [Geo](geo/replication/index.md) secondary nodes. + +## Create a server hook for a repository -NOTE: **Note:** If you are not using [hashed storage](repository_storage_types.md#hashed-storage), the project's -repository directory might not exactly match the instructions below. In that case, -for an installation from source the path is usually `/home/git/repositories/<group>/<project>.git`. -For Omnibus installs the path is usually `/var/opt/gitlab/git-data/repositories/<group>/<project>.git`. - -Follow the steps below to set up a server hook for a -repository: - -1. Find that project's path on the GitLab server, by navigating to the - **Admin area > Projects**. From there, select the project for which you - would like to add a hook. You can find the path to the project's repository - under **Gitaly relative path** on that page. -1. Create a new directory in this location called `custom_hooks`. -1. Inside the new `custom_hooks` directory, create a file with a name matching - the hook type. For a pre-receive hook the file name should be `pre-receive` - with no extension. -1. Make the hook file executable and make sure it's owned by Git. -1. Write the code to make the server hook function as expected. Hooks can be - in any language. Ensure the 'shebang' at the top properly reflects the language - type. For example, if the script is in Ruby the shebang will probably be +repository directory might not exactly match the instructions below. In that case: + +- For an installation from source, the path is usually + `/home/git/repositories/<group>/<project>.git`. +- For Omnibus GitLab installs, the path is usually + `/var/opt/gitlab/git-data/repositories/<group>/<project>.git`. + +Follow the steps below to set up a server-side hook for a repository: + +1. Navigate to **Admin area > Projects** and click on the project you want to add a server hook to. +1. Locate the **Gitaly relative path** on the page that appears. This is where the server hook + must be implemented. For information on interpreting the relative path, see + [Translating hashed storage paths](repository_storage_types.md#translating-hashed-storage-paths). +1. On the file system, create a new directory in this location called `custom_hooks`. +1. Inside the new `custom_hooks` directory, create a file with a name matching the hook type. For + example, for a pre-receive hook the filename should be `pre-receive` with no extension. +1. Make the hook file executable and ensure that it's owned by the Git user. +1. Write the code to make the server hook function as expected. Hooks can be in any language. Ensure + the ["shebang"](https://en.wikipedia.org/wiki/Shebang_(Unix)) at the top properly reflects the + language type. For example, if the script is in Ruby the shebang is probably `#!/usr/bin/env ruby`. -Assuming the hook code is properly implemented the hook will run as appropriate. - -## Set a global server hook for all repositories - -To create a Git hook that applies to all of your repositories in -your instance, set a global server hook. Since GitLab will look inside the GitLab Shell -`hooks` directory for global hooks, adding any hook there will apply it to all repositories. -Follow the steps below to properly set up a server hook for all repositories: - -1. On the GitLab server, navigate to the configured custom hook directory. The - default is in the GitLab Shell directory. The GitLab Shell `hook` directory - for an installation from source the path is usually - `/home/git/gitlab-shell/hooks`. For Omnibus installs the path is usually - `/opt/gitlab/embedded/service/gitlab-shell/hooks`. - To look in a different directory for the global custom hooks, - set `custom_hooks_dir` in the Gitaly config. For Omnibus installations, this is set - in `gitlab.rb`. For source installations, the configuration location depends on the - GitLab version. For: - - - GitLab 13.0 and earlier, this is set in `gitlab-shell/config.yml`. - - GitLab 13.1 and later, this is set in `gitaly/config.toml` under the `[hooks]` - section. - - NOTE: **Note:** - The `custom_hooks_dir` value in `gitlab-shell/config.yml` is still honored in GitLab - 13.1 and later if the value in `gitaly/config.toml` is blank or non-existent. - -1. Create a new directory in this location. Depending on your hook, it will be - either a `pre-receive.d`, `post-receive.d`, or `update.d` directory. -1. Inside this new directory, add your hook. Hooks can be - in any language. Ensure the 'shebang' at the top properly reflects the language - type. For example, if the script is in Ruby the shebang will probably be +Assuming the hook code is properly implemented, the hook code is executed as appropriate. + +## Create a global server hook for all repositories + +To create a Git hook that applies to all of the repositories in your instance, set a global server +hook. The default global server hook directory is in the GitLab Shell directory. Any +hook added there applies to all repositories. + +The default directory: + +- For an installation from source is usually `/home/git/gitlab-shell/hooks`. +- For Omnibus GitLab installs is usually `/opt/gitlab/embedded/service/gitlab-shell/hooks`. + +To use a different directory for global server hooks, set `custom_hooks_dir` in Gitaly +configuration: + +- For Omnibus installations, this is set in `gitlab.rb`. +- For source installations, the configuration location depends on the GitLab version. For: + - GitLab 13.0 and earlier, this is set in `gitlab-shell/config.yml`. + - GitLab 13.1 and later, this is set in `gitaly/config.toml` under the `[hooks]` section. + +NOTE: **Note:** +The `custom_hooks_dir` value in `gitlab-shell/config.yml` is still honored in GitLab 13.1 and later +if the value in `gitaly/config.toml` is blank or non-existent. + +Follow the steps below to set up a global server hook for all repositories: + +1. On the GitLab server, navigate to the configured global server hook directory. +1. Create a new directory in this location. Depending on the type of hook, it can be either a + `pre-receive.d`, `post-receive.d`, or `update.d` directory. +1. Inside this new directory, add your hook. Hooks can be in any language. Ensure the + ["shebang"](https://en.wikipedia.org/wiki/Shebang_(Unix)) at the top properly reflects the + language type. For example, if the script is in Ruby the shebang is probably `#!/usr/bin/env ruby`. -1. Make the hook file executable and make sure it's owned by Git. +1. Make the hook file executable and ensure that it's owned by the Git user. Now test the hook to check whether it is functioning properly. -## Chained hooks support +## Chained hooks > [Introduced](https://gitlab.com/gitlab-org/gitlab-shell/-/merge_requests/93) in GitLab Shell 4.1.0 and GitLab 8.15. -Hooks can be also global or be set per project directories and support a chained -execution of the hooks. +Server hooks set [per project](#create-a-server-hook-for-a-repository) or +[globally](#create-a-global-server-hook-for-all-repositories) can be executed in a chain. -NOTE: **Note:** -`<hook_name>.d` would need to be either `pre-receive.d`, -`post-receive.d`, or `update.d` to work properly. Any other names will be ignored. +Server hooks are searched for and executed in the following order of priority: + +- Built-in GitLab server hooks. These are not user-customizable. +- `<project>.git/custom_hooks/<hook_name>`: Per-project hooks. This was kept for backwards + compatibility. +- `<project>.git/custom_hooks/<hook_name>.d/*`: Location for per-project hooks. +- `<custom_hooks_dir>/<hook_name>.d/*`: Location for all executable global hook files + except editor backup files. + +Within a directory, server hooks: + +- Are executed in alphabetical order. +- Stop executing when a hook exits with a non-zero value. + +Note: + +- `<hook_name>.d` must be either `pre-receive.d`, `post-receive.d`, or `update.d` to work properly. + Any other names are ignored. +- Files in `.d` directories must be executable and not match the backup file pattern (`*~`). +- For `<project>.git` you need to [translate](repository_storage_types.md#translating-hashed-storage-paths) + your project name into the hashed storage format that GitLab uses. + +## Environment Variables + +The following set of environment variables are available to server hooks. + +| Environment variable | Description | +|:---------------------|:----------------------------------------------------------------------------| +| `GL_ID` | GitLab identifier of user that initiated the push. For example, `user-2234` | +| `GL_PROJECT_PATH` | (GitLab 13.2 and later) GitLab project path | +| `GL_PROTOCOL` | (GitLab 13.2 and later) Protocol used with push | +| `GL_REPOSITORY` | `project-<id>` where `id` is the ID of the project | +| `GL_USERNAME` | GitLab username of the user that initiated the push | + +Pre-receive and post-receive server hooks can also access the following Git environment variables. + +| Environment variable | Description | +|:-----------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `GIT_ALTERNATE_OBJECT_DIRECTORIES` | Alternate object directories in the quarantine environment. See [Git `receive-pack` documentation](https://git-scm.com/docs/git-receive-pack#_quarantine_environment). | +| `GIT_OBJECT_DIRECTORY` | GitLab project path in the quarantine environment. See [Git `receive-pack` documentation](https://git-scm.com/docs/git-receive-pack#_quarantine_environment). | +| `GIT_PUSH_OPTION_COUNT` | Number of push options. See [Git `pre-receive` documentation](https://git-scm.com/docs/githooks#pre-receive). | +| `GIT_PUSH_OPTION_<i>` | Value of push options where `i` is from `0` to `GIT_PUSH_OPTION_COUNT - 1`. See [Git `pre-receive` documentation](https://git-scm.com/docs/githooks#pre-receive). | NOTE: **Note:** -Files in `.d` directories need to be executable and not match the backup file -pattern (`*~`). +While other environment variables can be passed to server hooks, your application should not rely on +them as they can change. -The hooks are searched and executed in this order: +## Transition to Go -1. Built-in GitLab server hooks (not user-customizable). -1. `<project>.git/custom_hooks/<hook_name>` - per-project hook (this was kept as the already existing behavior). -1. `<project>.git/custom_hooks/<hook_name>.d/*` - per-project hooks. -1. `<custom_hooks_dir>/<hook_name>.d/*` - global hooks: all executable files (except editor backup files). +> Introduced in GitLab 13.2 using feature flags. -The hooks of the same type are executed in order and execution stops on the -first script exiting with a non-zero value. +The following server hooks have been re-implemented in Go: -For `<project>.git` you'll need to -[translate your project name into the hashed storage format](repository_storage_types.md#translating-hashed-storage-paths) -that GitLab uses. +- `pre-receive`, with the Go implementation used by default. To use the Ruby implementation instead, + [disable](../operations/feature_flags.md#enable-or-disable-feature-flag-strategies) the + `:gitaly_go_preceive_hook` feature flag. +- `update`, with the Go implementation used by default. To use the Ruby implementation instead, + [disable](../operations/feature_flags.md#enable-or-disable-feature-flag-strategies) the + `:gitaly_go_update_hook` feature flag. +- `post-receive`, however the Ruby implementation is used by default. To use the Go implementation + instead, [enable](../operations/feature_flags.md#enable-or-disable-feature-flag-strategies) the + `:gitaly_go_postreceive_hook` feature flag. ## Custom error messages > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/5073) in GitLab 8.10. -To have custom error messages appear in GitLab's UI when the commit is -declined or an error occurs during the Git hook, your script should: +To have custom error messages appear in GitLab's UI when a commit is declined or an error occurs +during the Git hook, your script should: - Send the custom error messages to either the script's `stdout` or `stderr`. - Prefix each message with `GL-HOOK-ERR:` with no characters appearing before the prefix. ### Example custom error message -This hook script written in bash will generate the following message in GitLab's UI: +This hook script written in Bash generates the following message in GitLab's UI: ```shell #!/bin/sh diff --git a/doc/administration/smime_signing_email.md b/doc/administration/smime_signing_email.md index bab7c5c260d..304b65917c1 100644 --- a/doc/administration/smime_signing_email.md +++ b/doc/administration/smime_signing_email.md @@ -21,7 +21,8 @@ files must be provided: Optionally, you can also provide a bundle of CA certs (PEM-encoded) to be included on each signature. This will typically be an intermediate CA. -NOTE: **Note:** Be mindful of the access levels for your private keys and visibility to +NOTE: **Note:** +Be mindful of the access levels for your private keys and visibility to third parties. **For Omnibus installations:** @@ -38,7 +39,8 @@ third parties. 1. Save the file and [reconfigure GitLab](restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. -NOTE: **Note:** The key needs to be readable by the GitLab system user (`git` by default). +NOTE: **Note:** +The key needs to be readable by the GitLab system user (`git` by default). **For installations from source:** @@ -61,7 +63,8 @@ NOTE: **Note:** The key needs to be readable by the GitLab system user (`git` by 1. Save the file and [restart GitLab](restart_gitlab.md#installations-from-source) for the changes to take effect. -NOTE: **Note:** The key needs to be readable by the GitLab system user (`git` by default). +NOTE: **Note:** +The key needs to be readable by the GitLab system user (`git` by default). ### How to convert S/MIME PKCS#12 / PFX format to PEM encoding diff --git a/doc/administration/terraform_state.md b/doc/administration/terraform_state.md index 77d5495418e..8d3ddc4e306 100644 --- a/doc/administration/terraform_state.md +++ b/doc/administration/terraform_state.md @@ -68,20 +68,7 @@ The following settings are: ### S3-compatible connection settings -The connection settings match those provided by [Fog](https://github.com/fog), and are as follows: - -| Setting | Description | Default | -|---------|-------------|---------| -| `provider` | Always `AWS` for compatible hosts | `AWS` | -| `aws_access_key_id` | Credentials for AWS or compatible provider | | -| `aws_secret_access_key` | Credentials for AWS or compatible provider | | -| `aws_signature_version` | AWS signature version to use. 2 or 4 are valid options. Digital Ocean Spaces and other providers may need 2. | 4 | -| `enable_signature_v4_streaming` | Set to true to enable HTTP chunked transfers with [AWS v4 signatures](https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html). Oracle Cloud S3 needs this to be false | `true` | -| `region` | AWS region | us-east-1 | -| `host` | S3-compatible host when not using AWS. For example, `localhost` or `storage.example.com` | `s3.amazonaws.com` | -| `endpoint` | Can be used when configuring an S3-compatible service such as [MinIO](https://min.io), by entering a URL such as `http://127.0.0.1:9000` | (optional) | -| `path_style` | Set to true to use `host/bucket_name/object` style paths instead of `bucket_name.host/object`. Leave as false for AWS S3 | `false` | -| `use_iam_profile` | For AWS S3, set to true to use an IAM profile instead of access keys | `false` | +See [the available connection settings for different providers](object_storage.md#connection-settings). **In Omnibus installations:** @@ -91,7 +78,7 @@ The connection settings match those provided by [Fog](https://github.com/fog), a ```ruby gitlab_rails['terraform_state_enabled'] = true gitlab_rails['terraform_state_object_store_enabled'] = true - gitlab_rails['terraform_state_object_store_remote_directory'] = "terraform_state" + gitlab_rails['terraform_state_object_store_remote_directory'] = "terraform" gitlab_rails['terraform_state_object_store_connection'] = { 'provider' => 'AWS', 'region' => 'eu-central-1', @@ -123,7 +110,7 @@ The connection settings match those provided by [Fog](https://github.com/fog), a enabled: true object_store: enabled: true - remote_directory: "terraform_state" # The bucket name + remote_directory: "terraform" # The bucket name connection: provider: AWS # Only AWS supported at the moment aws_access_key_id: AWS_ACESS_KEY_ID diff --git a/doc/administration/troubleshooting/elasticsearch.md b/doc/administration/troubleshooting/elasticsearch.md index 12b82e4bc48..e13261e3074 100644 --- a/doc/administration/troubleshooting/elasticsearch.md +++ b/doc/administration/troubleshooting/elasticsearch.md @@ -261,6 +261,9 @@ Beyond that, you will want to review the error. If it is: - Specifically from the indexer, this could be a bug/issue and should be escalated to GitLab support. - An OS issue, you will want to reach out to your systems administrator. +- A `Faraday::TimeoutError (execution expired)` error **and** you're using a proxy, + [set a custom `gitlab_rails['env']` environment variable, called `no_proxy`](https://docs.gitlab.com/omnibus/settings/environment-variables.html) + with the IP address of your Elasticsearch host. ### Troubleshooting performance diff --git a/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md b/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md index c911c617210..8e9749fb239 100644 --- a/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md +++ b/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md @@ -11,13 +11,13 @@ having an issue with GitLab, it is highly recommended that you check your [support options](https://about.gitlab.com/support/) first, before attempting to use this information. -CAUTION: **CAUTION:** +CAUTION: **Caution:** Please note that some of these scripts could be damaging if not run correctly, or under the right conditions. We highly recommend running them under the guidance of a Support Engineer, or running them in a test environment with a backup of the instance ready to be restored, just in case. -CAUTION: **CAUTION:** +CAUTION: **Caution:** Please also note that as GitLab changes, changes to the code are inevitable, and so some scripts may not work as they once used to. These are not kept up-to-date as these scripts/commands were added as they were found/needed. As @@ -201,6 +201,20 @@ project.repository_read_only = true; project.save project.update!(repository_read_only: true) ``` +### Transfer project from one namespace to another + +```ruby + p= Project.find_by_full_path('') + + # To set the owner of the project + current_user= p.creator + +# Namespace where you want this to be moved. +namespace = Namespace.find_by_full_path("") + +::Projects::TransferService.new(p, current_user).execute(namespace) +``` + ### Bulk update service integration password for _all_ projects For example, change the Jira user's password for all projects that have the Jira @@ -447,7 +461,7 @@ group = Group.find_by_path_or_name("groupname") # Count users from subgroup and up (inherited) group.members_with_parents.count -# Count users from parent group and down (specific grants) +# Count users from the parent group and down (specific grants) parent.members_with_descendants.count ``` diff --git a/doc/administration/troubleshooting/linux_cheat_sheet.md b/doc/administration/troubleshooting/linux_cheat_sheet.md index 7af4219caa3..06c49d67f40 100644 --- a/doc/administration/troubleshooting/linux_cheat_sheet.md +++ b/doc/administration/troubleshooting/linux_cheat_sheet.md @@ -10,7 +10,7 @@ and it may be useful for users with experience with Linux. If you are currently having an issue with GitLab, you may want to check your [support options](https://about.gitlab.com/support/) first, before attempting to use this information. -CAUTION: **CAUTION:** +CAUTION: **Caution:** If you are administering GitLab you are expected to know these commands for your distribution of choice. If you are a GitLab Support Engineer, consider this a cross-reference to translate `yum` -> `apt-get` and the like. diff --git a/doc/administration/troubleshooting/navigating_gitlab_via_rails_console.md b/doc/administration/troubleshooting/navigating_gitlab_via_rails_console.md index 69af7ea6801..571973c12d9 100644 --- a/doc/administration/troubleshooting/navigating_gitlab_via_rails_console.md +++ b/doc/administration/troubleshooting/navigating_gitlab_via_rails_console.md @@ -6,7 +6,7 @@ Thanks to this, we also get access to the amazing tools built right into Rails. In this guide, we'll introduce the [Rails console](debug.md#starting-a-rails-console-session) and the basics of interacting with your GitLab instance from the command line. -CAUTION: **CAUTION:** +CAUTION: **Caution:** The Rails console interacts directly with your GitLab instance. In many cases, there are no handrails to prevent you from permanently modifying, corrupting or destroying production data. If you would like to explore the Rails console @@ -186,7 +186,7 @@ user.save Which would return: ```ruby -Enqueued ActionMailer::DeliveryJob (Job ID: 05915c4e-c849-4e14-80bb-696d5ae22065) to Sidekiq(mailers) with arguments: "DeviseMailer", "password_change", "deliver_now", #<GlobalID:0x00007f42d8ccebe8 @uri=#<URI::GID gid://gitlab/User/1>> +Enqueued ActionMailer::MailDeliveryJob (Job ID: 05915c4e-c849-4e14-80bb-696d5ae22065) to Sidekiq(mailers) with arguments: "DeviseMailer", "password_change", "deliver_now", #<GlobalID:0x00007f42d8ccebe8 @uri=#<URI::GID gid://gitlab/User/1>> => true ``` diff --git a/doc/administration/troubleshooting/postgresql.md b/doc/administration/troubleshooting/postgresql.md index e5a4dffb3cc..36c1f20818a 100644 --- a/doc/administration/troubleshooting/postgresql.md +++ b/doc/administration/troubleshooting/postgresql.md @@ -8,7 +8,8 @@ This page is useful information about PostgreSQL that the GitLab Support Team sometimes uses while troubleshooting. GitLab is making this public, so that anyone can make use of the Support team's collected knowledge. -CAUTION: **Caution:** Some procedures documented here may break your GitLab instance. Use at your own risk. +CAUTION: **Caution:** +Some procedures documented here may break your GitLab instance. Use at your own risk. If you are on a [paid tier](https://about.gitlab.com/pricing/) and are not sure how to use these commands, it is best to [contact Support](https://about.gitlab.com/support/) @@ -46,7 +47,7 @@ This section is for links to information elsewhere in the GitLab documentation. - Managing Omnibus PostgreSQL versions [from the development docs](https://docs.gitlab.com/omnibus/development/managing-postgresql-versions.html) - [PostgreSQL scaling](../postgresql/replication_and_failover.md) - - including [troubleshooting](../postgresql/replication_and_failover.md#troubleshooting) `gitlab-ctl repmgr-check-master` and PgBouncer errors + - including [troubleshooting](../postgresql/replication_and_failover.md#troubleshooting) `gitlab-ctl repmgr-check-master` (or `gitlab-ctl patroni check-leader` if you are using Patroni) and PgBouncer errors - [Developer database documentation](../../development/README.md#database-guides) - some of which is absolutely not for production use. Including: - understanding EXPLAIN plans @@ -115,7 +116,8 @@ Quoting from issue [#1](https://gitlab.com/gitlab-org/gitlab/-/issues/30528): > "If a deadlock is hit, and we resolve it through aborting the transaction after a short period, then the retry mechanisms we already have will make the deadlocked piece of work try again, and it's unlikely we'll deadlock multiple times in a row." -TIP: **Tip:** In support, our general approach to reconfiguring timeouts (applies also to the HTTP stack as well) is that it's acceptable to do it temporarily as a workaround. If it makes GitLab usable for the customer, then it buys time to understand the problem more completely, implement a hot fix, or make some other change that addresses the root cause. Generally, the timeouts should be put back to reasonable defaults once the root cause is resolved. +TIP: **Tip:** +In support, our general approach to reconfiguring timeouts (applies also to the HTTP stack as well) is that it's acceptable to do it temporarily as a workaround. If it makes GitLab usable for the customer, then it buys time to understand the problem more completely, implement a hot fix, or make some other change that addresses the root cause. Generally, the timeouts should be put back to reasonable defaults once the root cause is resolved. In this case, the guidance we had from development was to drop deadlock_timeout and/or statement_timeout but to leave the third setting at 60s. Setting idle_in_transaction protects the database from sessions potentially hanging for days. There's more discussion in [the issue relating to introducing this timeout on GitLab.com](https://gitlab.com/gitlab-com/gl-infra/production/-/issues/1053). diff --git a/doc/administration/troubleshooting/sidekiq.md b/doc/administration/troubleshooting/sidekiq.md index 5109a3baff2..566e2686735 100644 --- a/doc/administration/troubleshooting/sidekiq.md +++ b/doc/administration/troubleshooting/sidekiq.md @@ -38,7 +38,7 @@ Example log output: ```json {"severity":"INFO","time":"2020-06-08T14:37:37.892Z","class":"AdminEmailsWorker","args":["[FILTERED]","[FILTERED]","[FILTERED]"],"retry":3,"queue":"admin_emails","backtrace":true,"jid":"9e35e2674ac7b12d123e13cc","created_at":"2020-06-08T14:37:37.373Z","meta.user":"root","meta.caller_id":"Admin::EmailsController#create","correlation_id":"37D3lArJmT1","uber-trace-id":"2d942cc98cc1b561:6dc94409cfdd4d77:9fbe19bdee865293:1","enqueued_at":"2020-06-08T14:37:37.410Z","pid":65011,"message":"AdminEmailsWorker JID-9e35e2674ac7b12d123e13cc: done: 0.48085 sec","job_status":"done","scheduling_latency_s":0.001012,"redis_calls":9,"redis_duration_s":0.004608,"redis_read_bytes":696,"redis_write_bytes":6141,"duration_s":0.48085,"cpu_s":0.308849,"completed_at":"2020-06-08T14:37:37.892Z","db_duration_s":0.010742} -{"severity":"INFO","time":"2020-06-08T14:37:37.894Z","class":"ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper","wrapped":"ActionMailer::DeliveryJob","queue":"mailers","args":["[FILTERED]"],"retry":3,"backtrace":true,"jid":"e47a4f6793d475378432e3c8","created_at":"2020-06-08T14:37:37.884Z","meta.user":"root","meta.caller_id":"AdminEmailsWorker","correlation_id":"37D3lArJmT1","uber-trace-id":"2d942cc98cc1b561:29344de0f966446d:5c3b0e0e1bef987b:1","enqueued_at":"2020-06-08T14:37:37.885Z","pid":65011,"message":"ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper JID-e47a4f6793d475378432e3c8: start","job_status":"start","scheduling_latency_s":0.009473} +{"severity":"INFO","time":"2020-06-08T14:37:37.894Z","class":"ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper","wrapped":"ActionMailer::MailDeliveryJob","queue":"mailers","args":["[FILTERED]"],"retry":3,"backtrace":true,"jid":"e47a4f6793d475378432e3c8","created_at":"2020-06-08T14:37:37.884Z","meta.user":"root","meta.caller_id":"AdminEmailsWorker","correlation_id":"37D3lArJmT1","uber-trace-id":"2d942cc98cc1b561:29344de0f966446d:5c3b0e0e1bef987b:1","enqueued_at":"2020-06-08T14:37:37.885Z","pid":65011,"message":"ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper JID-e47a4f6793d475378432e3c8: start","job_status":"start","scheduling_latency_s":0.009473} {"severity":"INFO","time":"2020-06-08T14:39:50.648Z","class":"NewIssueWorker","args":["455","1"],"retry":3,"queue":"new_issue","backtrace":true,"jid":"a24af71f96fd129ec47f5d1e","created_at":"2020-06-08T14:39:50.643Z","meta.user":"root","meta.project":"h5bp/html5-boilerplate","meta.root_namespace":"h5bp","meta.caller_id":"Projects::IssuesController#create","correlation_id":"f9UCZHqhuP7","uber-trace-id":"28f65730f99f55a3:a5d2b62dec38dffc:48ddd092707fa1b7:1","enqueued_at":"2020-06-08T14:39:50.646Z","pid":65011,"message":"NewIssueWorker JID-a24af71f96fd129ec47f5d1e: start","job_status":"start","scheduling_latency_s":0.001144} ``` @@ -283,7 +283,8 @@ queue = Sidekiq::Queue.new('<queue name>') queue.each { |job| job.delete if <condition>} ``` -NOTE: **Note:** This will remove jobs that are queued but not started, running jobs will not be killed. Have a look at the section below for cancelling running jobs. +NOTE: **Note:** +This will remove jobs that are queued but not started, running jobs will not be killed. Have a look at the section below for cancelling running jobs. In the method above, `<queue-name>` is the name of the queue that contains the job(s) you want to delete and `<condition>` will decide which jobs get deleted. diff --git a/doc/administration/uploads.md b/doc/administration/uploads.md index 620f349912c..d9902208e93 100644 --- a/doc/administration/uploads.md +++ b/doc/administration/uploads.md @@ -57,6 +57,9 @@ This configuration relies on valid AWS credentials to be configured already. [Read more about using object storage with GitLab](object_storage.md). +NOTE: **Note:** +We recommend using the [consolidated object storage settings](object_storage.md#consolidated-object-storage-configuration). The following instructions apply to the original config format. + ## Object Storage Settings For source installations the following settings are nested under `uploads:` and then `object_store:`. On Omnibus GitLab installs they are prefixed by `uploads_object_store_`. @@ -70,22 +73,9 @@ For source installations the following settings are nested under `uploads:` and | `proxy_download` | Set to true to enable proxying all files served. Option allows to reduce egress traffic as this allows clients to download directly from remote storage instead of proxying all data | `false` | | `connection` | Various connection options described below | | -### S3 compatible connection settings +### Connection settings -The connection settings match those provided by [Fog](https://github.com/fog), and are as follows: - -| Setting | Description | Default | -|---------|-------------|---------| -| `provider` | Always `AWS` for compatible hosts | `AWS` | -| `aws_access_key_id` | AWS credentials, or compatible | | -| `aws_secret_access_key` | AWS credentials, or compatible | | -| `aws_signature_version` | AWS signature version to use. `2` or `4` are valid options. Digital Ocean Spaces and other providers may need `2`. | `4` | -| `enable_signature_v4_streaming` | Set to true to enable HTTP chunked transfers with [AWS v4 signatures](https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html). Oracle Cloud S3 needs this to be `false`. | `true` | -| `region` | AWS region | us-east-1 | -| `host` | S3 compatible host for when not using AWS, e.g. `localhost` or `storage.example.com` | `s3.amazonaws.com` | -| `endpoint` | Can be used when configuring an S3 compatible service such as [MinIO](https://min.io), by entering a URL such as `http://127.0.0.1:9000` | (optional) | -| `path_style` | Set to `true` to use `host/bucket_name/object` style paths instead of `bucket_name.host/object`. Leave as `false` for AWS S3. | `false` | -| `use_iam_profile` | Set to `true` to use IAM profile instead of access keys | false +See [the available connection settings for different providers](object_storage.md#connection-settings). **In Omnibus installations:** @@ -143,35 +133,7 @@ _The uploads are stored by default in 1. Save the file and [restart GitLab](restart_gitlab.md#installations-from-source) for the changes to take effect. 1. Migrate any existing local uploads to the object storage using [`gitlab:uploads:migrate` Rake task](raketasks/uploads/migrate.md). -### Oracle Cloud S3 connection settings - -Note that Oracle Cloud S3 must be sure to use the following settings: - -| Setting | Value | -|---------|-------| -| `enable_signature_v4_streaming` | `false` | -| `path_style` | `true` | - -If `enable_signature_v4_streaming` is set to `true`, you may see the -following error: - -```plaintext -STREAMING-AWS4-HMAC-SHA256-PAYLOAD is not supported -``` - -### OpenStack compatible connection settings - -The connection settings match those provided by [Fog](https://github.com/fog), and are as follows: - -| Setting | Description | Default | -|---------|-------------|---------| -| `provider` | Always `OpenStack` for compatible hosts | `OpenStack` | -| `openstack_username` | OpenStack username | | -| `openstack_api_key` | OpenStack API key | | -| `openstack_temp_url_key` | OpenStack key for generating temporary URLs | | -| `openstack_auth_url` | OpenStack authentication endpoint | | -| `openstack_region` | OpenStack region | | -| `openstack_tenant` | OpenStack tenant ID | +### OpenStack example **In Omnibus installations:** diff --git a/doc/api/README.md b/doc/api/README.md index 6cbb99a76cb..b07f14b5a7a 100644 --- a/doc/api/README.md +++ b/doc/api/README.md @@ -423,7 +423,7 @@ Status: 200 OK ``` CAUTION: **Deprecation:** -The `Links` Header will be removed in GitLab 14.0 to be aligned with the [W3C specification](https://www.w3.org/wiki/LinkHeader) +The `Links` Header will be removed in GitLab 14.0 to be aligned with the [W3C `Link` specification](https://www.w3.org/wiki/LinkHeader) The link to the next page contains an additional filter `id_after=42` which excludes records we have retrieved already. Note the type of filter depends on the `order_by` option used and we may have more than one additional filter. @@ -491,6 +491,28 @@ GET /api/v4/projects/1/branches/my%2Fbranch/commits GET /api/v4/projects/1/repository/tags/my%2Ftag ``` +## Request Payload + +API Requests can use parameters sent as [query strings](https://en.wikipedia.org/wiki/Query_string) +or as a [payload body](https://tools.ietf.org/html/draft-ietf-httpbis-p3-payload-14#section-3.2). +GET requests usually send a query string, while PUT/POST requests usually send the payload body: + +- Query string: + + ```shell + curl --request POST "https://gitlab/api/v4/projects?name=<example-name>&description=<example-description>" + ``` + +- Request payload (JSON): + + ```shell + curl --request POST --header "Content-Type: application/json" --data '{"name":"<example-name>", "description":"<example-description"}' "https://gitlab/api/v4/projects" + ``` + +URL encoded query strings have a length limitation. Requests that are too large will +result in a `414 Request-URI Too Large` error message. This can be resolved by using +a payload body instead. + ## Encoding API parameters of `array` and `hash` types We can call the API with `array` and `hash` types parameters as shown below: diff --git a/doc/api/api_resources.md b/doc/api/api_resources.md index 2adf06a8e95..e93dfed3b1f 100644 --- a/doc/api/api_resources.md +++ b/doc/api/api_resources.md @@ -129,6 +129,7 @@ The following API resources are available outside of project and group contexts | [Geo Nodes](geo_nodes.md) **(PREMIUM ONLY)** | `/geo_nodes` | | [Group Activity Analytics](group_activity_analytics.md) **(STARTER)** | `/analytics/group_activity/{issues_count | merge_requests_count | new_members_count }` | | [Import repository from GitHub](import.md) | `/import/github` | +| [Instance clusters](instance_clusters.md) | `/admin/clusters` | | [Issues](issues.md) | `/issues` (also available for groups and projects) | | [Issues Statistics](issues_statistics.md) | `/issues_statistics` (also available for groups and projects) | | [Keys](keys.md) | `/keys` | @@ -140,7 +141,7 @@ The following API resources are available outside of project and group contexts | [Notification settings](notification_settings.md) | `/notification_settings` (also available for groups and projects) | | [Pages domains](pages_domains.md) | `/pages/domains` (also available for projects) | | [Projects](projects.md) | `/users/:id/projects` (also available for projects) | -| [Project Repository Storage Moves](project_repository_storage_moves.md) | `/project_repository_storage_moves` | +| [Project repository storage moves](project_repository_storage_moves.md) **(CORE ONLY)** | `/project_repository_storage_moves` | | [Runners](runners.md) | `/runners` (also available for projects) | | [Search](search.md) | `/search` (also available for groups and projects) | | [Settings](settings.md) **(CORE ONLY)** | `/application/settings` | diff --git a/doc/api/boards.md b/doc/api/boards.md index 155a876e76a..a370205aa01 100644 --- a/doc/api/boards.md +++ b/doc/api/boards.md @@ -452,7 +452,7 @@ POST /projects/:id/boards/:board_id/lists | `assignee_id` **(PREMIUM)** | integer | no | The ID of a user | | `milestone_id` **(PREMIUM)** | integer | no | The ID of a milestone | -NOTE: **Note**: +NOTE: **Note:** Label, assignee and milestone arguments are mutually exclusive, that is, only one of them are accepted in a request. Check the [Issue Board docs](../user/project/issue_board.md#summary-of-features-per-tier) diff --git a/doc/api/container_registry.md b/doc/api/container_registry.md index d4a4fc1a733..e4687994017 100644 --- a/doc/api/container_registry.md +++ b/doc/api/container_registry.md @@ -1,3 +1,9 @@ +--- +stage: Package +group: Package +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Container Registry API > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/55978) in GitLab 11.8. diff --git a/doc/api/deploy_keys.md b/doc/api/deploy_keys.md index 1634d07768a..97b7614f932 100644 --- a/doc/api/deploy_keys.md +++ b/doc/api/deploy_keys.md @@ -1,3 +1,9 @@ +--- +stage: Release +group: Progressive Delivery +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Deploy Keys API ## List all deploy keys diff --git a/doc/api/deploy_tokens.md b/doc/api/deploy_tokens.md index 16fc57eaa58..f11f88ab5c9 100644 --- a/doc/api/deploy_tokens.md +++ b/doc/api/deploy_tokens.md @@ -1,3 +1,9 @@ +--- +stage: Release +group: Progressive Delivery +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Deploy Tokens API ## List all deploy tokens diff --git a/doc/api/deployments.md b/doc/api/deployments.md index 8c952ba07b1..426b3e10ecf 100644 --- a/doc/api/deployments.md +++ b/doc/api/deployments.md @@ -1,3 +1,10 @@ +--- +stage: Release +group: Release Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: concepts, howto +--- + # Deployments API ## List project deployments diff --git a/doc/api/discussions.md b/doc/api/discussions.md index 1f509a7aadc..aa1a691a8f8 100644 --- a/doc/api/discussions.md +++ b/doc/api/discussions.md @@ -782,10 +782,14 @@ Diff comments also contain position: "old_line": 27, "new_line": 27, "line_range": { - "start_line_code": "588440f66559714280628a4f9799f0c4eb880a4a_10_10", - "start_line_type": "new", - "end_line_code": "588440f66559714280628a4f9799f0c4eb880a4a_11_11", - "end_line_type": "old" + "start": { + "line_code": "588440f66559714280628a4f9799f0c4eb880a4a_10_10", + "type": "new", + }, + "end": { + "line_code": "588440f66559714280628a4f9799f0c4eb880a4a_11_11", + "type": "old" + }, } }, "resolved": false, @@ -832,30 +836,32 @@ POST /projects/:id/merge_requests/:merge_request_iid/discussions Parameters: -| Attribute | Type | Required | Description | -| --------------------------------------- | -------------- | -------- | ----------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) | -| `merge_request_iid` | integer | yes | The IID of a merge request | -| `body` | string | yes | The content of the thread | -| `created_at` | string | no | Date time string, ISO 8601 formatted, e.g. 2016-03-11T03:45:40Z (requires admin or project/group owner rights) | -| `position` | hash | no | Position when creating a diff note | -| `position[base_sha]` | string | yes | Base commit SHA in the source branch | -| `position[start_sha]` | string | yes | SHA referencing commit in target branch | -| `position[head_sha]` | string | yes | SHA referencing HEAD of this merge request | -| `position[position_type]` | string | yes | Type of the position reference', allowed values: 'text' or 'image' | -| `position[new_path]` | string | no | File path after change | -| `position[new_line]` | integer | no | Line number after change (for 'text' diff notes) | -| `position[old_path]` | string | no | File path before change | -| `position[old_line]` | integer | no | Line number before change (for 'text' diff notes) | -| `position[line_range]` | hash | no | Line range for a multi-line diff note | -| `position[line_range][start_line_code]` | string | yes | Line code for the start line | -| `position[line_range][end_line_code]` | string | yes | Line code for the end line | -| `position[line_range][start_line_type]` | string | yes | Line type for the start line | -| `position[line_range][end_line_type]` | string | yes | Line type for the end line | -| `position[width]` | integer | no | Width of the image (for 'image' diff notes) | -| `position[height]` | integer | no | Height of the image (for 'image' diff notes) | -| `position[x]` | integer | no | X coordinate (for 'image' diff notes) | -| `position[y]` | integer | no | Y coordinate (for 'image' diff notes) | +| Attribute | Type | Required | Description | +| ---------------------------------------- | -------------- | -------- | ----------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) | +| `merge_request_iid` | integer | yes | The IID of a merge request | +| `body` | string | yes | The content of the thread | +| `created_at` | string | no | Date time string, ISO 8601 formatted, e.g. 2016-03-11T03:45:40Z (requires admin or project/group owner rights) | +| `position` | hash | no | Position when creating a diff note | +| `position[base_sha]` | string | yes | Base commit SHA in the source branch | +| `position[start_sha]` | string | yes | SHA referencing commit in target branch | +| `position[head_sha]` | string | yes | SHA referencing HEAD of this merge request | +| `position[position_type]` | string | yes | Type of the position reference', allowed values: 'text' or 'image' | +| `position[new_path]` | string | no | File path after change | +| `position[new_line]` | integer | no | Line number after change (for 'text' diff notes) | +| `position[old_path]` | string | no | File path before change | +| `position[old_line]` | integer | no | Line number before change (for 'text' diff notes) | +| `position[line_range]` | hash | no | Line range for a multi-line diff note | +| `position[line_range][start]` | hash | no | Multiline note starting line | +| `position[line_range][start][line_code]` | string | yes | Line code for the start line | +| `position[line_range][start][type]` | string | yes | Line type for the start line | +| `position[line_range][end]` | hash | no | Multiline note ending line | +| `position[line_range][end][line_code]` | string | yes | Line code for the end line | +| `position[line_range][end][type]` | string | yes | Line type for the end line | +| `position[width]` | integer | no | Width of the image (for 'image' diff notes) | +| `position[height]` | integer | no | Height of the image (for 'image' diff notes) | +| `position[x]` | integer | no | X coordinate (for 'image' diff notes) | +| `position[y]` | integer | no | Y coordinate (for 'image' diff notes) | ```shell curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/merge_requests/11/discussions?body=comment" diff --git a/doc/api/environments.md b/doc/api/environments.md index 5f6bdc251ba..2287ec9aad2 100644 --- a/doc/api/environments.md +++ b/doc/api/environments.md @@ -1,3 +1,10 @@ +--- +stage: Release +group: Release Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: concepts, howto +--- + # Environments API ## List environments @@ -13,6 +20,7 @@ GET /projects/:id/environments | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | | `name` | string | no | Return the environment with this name. Mutually exclusive with `search` | | `search` | string | no | Return list of environments matching the search criteria. Mutually exclusive with `name` | +| `states` | string | no | List all environments that match a specific state. Accepted values: `available` or `stopped`. If no state value given, returns all environments. | ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/environments?name=review%2Ffix-foo" diff --git a/doc/api/epic_issues.md b/doc/api/epic_issues.md index 86f88b0322f..4ab505f3627 100644 --- a/doc/api/epic_issues.md +++ b/doc/api/epic_issues.md @@ -4,13 +4,15 @@ group: Portfolio Management info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# Epic Issues API **(ULTIMATE)** +# Epic Issues API **(PREMIUM)** Every API call to epic_issues must be authenticated. -If a user is not a member of a group and the group is private, a `GET` request on that group will result to a `404` status code. +If a user is not a member of a group and the group is private, a `GET` request on that group will +result in a `404` status code. -Epics are available only in Ultimate. If epics feature is not available a `403` status code will be returned. +Epics are available only in GitLab [Premium and higher](https://about.gitlab.com/pricing/). +If the Epics feature is not available, a `403` status code will be returned. ## List issues for an epic diff --git a/doc/api/epic_links.md b/doc/api/epic_links.md index 756a41a0680..1d54bfe01e3 100644 --- a/doc/api/epic_links.md +++ b/doc/api/epic_links.md @@ -9,7 +9,8 @@ Every API call to `epic_links` must be authenticated. If a user is not a member of a group and the group is private, a `GET` request on that group will result to a `404` status code. -Epics are available only in the [Ultimate/Gold tier](https://about.gitlab.com/pricing/). If the epics feature is not available, a `403` status code will be returned. +Multi-level Epics are available only in GitLab [Ultimate/Gold](https://about.gitlab.com/pricing/). +If the Multi-level Epics feature is not available, a `403` status code will be returned. ## List epics related to a given epic diff --git a/doc/api/epics.md b/doc/api/epics.md index a420ef4cd15..fcdbb8cea71 100644 --- a/doc/api/epics.md +++ b/doc/api/epics.md @@ -38,11 +38,11 @@ are paginated. Read more on [pagination](README.md#pagination). -CAUTION: **Deprecation** +CAUTION: **Deprecation:** > `reference` attribute in response is deprecated in favour of `references`. > Introduced [GitLab 12.6](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/20354) -NOTE: **Note** +NOTE: **Note:** > `references.relative` is relative to the group that the epic is being requested. When epic is fetched from its origin group > `relative` format would be the same as `short` format and when requested cross groups it is expected to be the same as `full` format. diff --git a/doc/api/error_tracking.md b/doc/api/error_tracking.md index e18fbaf25c3..658480ce6fa 100644 --- a/doc/api/error_tracking.md +++ b/doc/api/error_tracking.md @@ -1,3 +1,9 @@ +--- +stage: Monitor +group: Health +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Error Tracking settings API > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/34940) in GitLab 12.7. diff --git a/doc/api/feature_flag_specs.md b/doc/api/feature_flag_specs.md index 52a4864fdc5..eaa6ea36c23 100644 --- a/doc/api/feature_flag_specs.md +++ b/doc/api/feature_flag_specs.md @@ -8,12 +8,12 @@ info: To determine the technical writer assigned to the Stage/Group associated w > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9566) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.5. -CAUTION: **Deprecation** +CAUTION: **Deprecation:** This API is deprecated and [scheduled for removal in GitLab 14.0](https://gitlab.com/gitlab-org/gitlab/-/issues/213369). The API for creating, updating, reading and deleting Feature Flag Specs. Automation engineers benefit from this API by being able to modify Feature Flag Specs without accessing user interface. -To manage the [Feature Flag](../user/project/operations/feature_flags.md) resources via public API, please refer to the [Feature Flags API](feature_flags.md) document. +To manage the [Feature Flag](../operations/feature_flags.md) resources via public API, please refer to the [Feature Flags API](feature_flags.md) document. Users with Developer or higher [permissions](../user/permissions.md) can access Feature Flag Specs API. @@ -166,7 +166,7 @@ POST /projects/:id/feature_flags/:name/scopes | `name` | string | yes | The name of the feature flag. | | `environment_scope` | string | yes | The [environment spec](../ci/environments/index.md#scoping-environments-with-specs) of the feature flag. | | `active` | boolean | yes | Whether the spec is active. | -| `strategies` | json | yes | The [strategies](../user/project/operations/feature_flags.md#feature-flag-strategies) of the feature flag spec. | +| `strategies` | JSON | yes | The [strategies](../operations/feature_flags.md#feature-flag-strategies) of the feature flag spec. | ```shell curl "https://gitlab.example.com/api/v4/projects/1/feature_flags/new_live_trace/scopes" \ @@ -249,7 +249,7 @@ PUT /projects/:id/feature_flags/:name/scopes/:environment_scope | `name` | string | yes | The name of the feature flag. | | `environment_scope` | string | yes | The URL-encoded [environment spec](../ci/environments/index.md#scoping-environments-with-specs) of the feature flag. | | `active` | boolean | yes | Whether the spec is active. | -| `strategies` | json | yes | The [strategies](../user/project/operations/feature_flags.md#feature-flag-strategies) of the feature flag spec. | +| `strategies` | JSON | yes | The [strategies](../operations/feature_flags.md#feature-flag-strategies) of the feature flag spec. | ```shell curl "https://gitlab.example.com/api/v4/projects/1/feature_flags/new_live_trace/scopes/production" \ diff --git a/doc/api/feature_flags.md b/doc/api/feature_flags.md index f3af662c972..99303e23c37 100644 --- a/doc/api/feature_flags.md +++ b/doc/api/feature_flags.md @@ -8,10 +8,11 @@ info: To determine the technical writer assigned to the Stage/Group associated w > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9566) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.5. -NOTE: **Note** -This API is behind a [feature flag](../user/project/operations/feature_flags.md#feature-flag-behavior-change-in-130). If this flag is not enabled in your environment, you can use the [legacy feature flags API](feature_flags_legacy.md). +NOTE: **Note:** +This API is behind a [feature flag](../operations/feature_flags.md#enable-or-disable-feature-flag-strategies). +If this flag is not enabled in your environment, you can use the [legacy feature flags API](feature_flags_legacy.md). -API for accessing resources of [GitLab Feature Flags](../user/project/operations/feature_flags.md). +API for accessing resources of [GitLab Feature Flags](../operations/feature_flags.md). Users with Developer or higher [permissions](../user/permissions.md) can access Feature Flag API. @@ -145,7 +146,7 @@ POST /projects/:id/feature_flags | `name` | string | yes | The name of the feature flag. | | `version` | string | yes | The version of the feature flag. Must be `new_version_flag`. Omit or set to `legacy_flag` to create a [Legacy Feature Flag](feature_flags_legacy.md). | | `description` | string | no | The description of the feature flag. | -| `strategies` | JSON | no | The feature flag [strategies](../user/project/operations/feature_flags.md#feature-flag-strategies). | +| `strategies` | JSON | no | The feature flag [strategies](../operations/feature_flags.md#feature-flag-strategies). | | `strategies:name` | JSON | no | The strategy name. | | `strategies:parameters` | JSON | no | The strategy parameters. | | `strategies:scopes` | JSON | no | The scopes for the strategy. | @@ -203,7 +204,7 @@ PUT /projects/:id/feature_flags/:name | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | | `name` | string | yes | The name of the feature flag. | | `description` | string | no | The description of the feature flag. | -| `strategies` | JSON | no | The feature flag [strategies](../user/project/operations/feature_flags.md#feature-flag-strategies). | +| `strategies` | JSON | no | The feature flag [strategies](../operations/feature_flags.md#feature-flag-strategies). | | `strategies:id` | JSON | no | The feature flag strategy id. | | `strategies:name` | JSON | no | The strategy name. | | `strategies:parameters` | JSON | no | The strategy parameters. | diff --git a/doc/api/feature_flags_legacy.md b/doc/api/feature_flags_legacy.md index 30bae9c5eeb..7e4fc47a1de 100644 --- a/doc/api/feature_flags_legacy.md +++ b/doc/api/feature_flags_legacy.md @@ -8,10 +8,10 @@ info: To determine the technical writer assigned to the Stage/Group associated w > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9566) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.5. -CAUTION: **Deprecation** +CAUTION: **Deprecation:** This API is deprecated and [scheduled for removal in GitLab 14.0](https://gitlab.com/gitlab-org/gitlab/-/issues/213369). Use [this API](feature_flags.md) instead. -API for accessing resources of [GitLab Feature Flags](../user/project/operations/feature_flags.md). +API for accessing resources of [GitLab Feature Flags](../operations/feature_flags.md). Users with Developer or higher [permissions](../user/permissions.md) can access Feature Flag API. @@ -166,7 +166,7 @@ POST /projects/:id/feature_flags | `scopes` | JSON | no | The feature flag specs of the feature flag. | | `scopes:environment_scope` | string | no | The environment spec. | | `scopes:active` | boolean | no | Whether the spec is active. | -| `scopes:strategies` | JSON | no | The [strategies](../user/project/operations/feature_flags.md#feature-flag-strategies) of the feature flag spec. | +| `scopes:strategies` | JSON | no | The [strategies](../operations/feature_flags.md#feature-flag-strategies) of the feature flag spec. | ```shell curl https://gitlab.example.com/api/v4/projects/1/feature_flags \ diff --git a/doc/api/freeze_periods.md b/doc/api/freeze_periods.md index ee5e657c945..e6a5e69497f 100644 --- a/doc/api/freeze_periods.md +++ b/doc/api/freeze_periods.md @@ -1,8 +1,15 @@ +--- +stage: Release +group: Release Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: concepts, howto +--- + # Freeze Periods API > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/29382) in GitLab 13.0. -You can use the Freeze Periods API to manipulate GitLab's [Freeze Period](../user/project/releases/index.md#set-a-deploy-freeze) entries. +You can use the Freeze Periods API to manipulate GitLab's [Freeze Period](../user/project/releases/index.md#prevent-unintentional-releases-by-setting-a-deploy-freeze) entries. ## Permissions and security diff --git a/doc/api/geo_nodes.md b/doc/api/geo_nodes.md index 12f785a3e3d..402170fba37 100644 --- a/doc/api/geo_nodes.md +++ b/doc/api/geo_nodes.md @@ -367,8 +367,9 @@ Example response: "package_files_count": 10, "package_files_checksummed_count": 10, "package_files_checksum_failed_count": 0, - "package_files_synced_count": 10, - "package_files_failed_count": 5 + "package_files_registry_count": 10, + "package_files_synced_count": 6, + "package_files_failed_count": 3 }, { "geo_node_id": 2, @@ -440,8 +441,9 @@ Example response: "package_files_count": 10, "package_files_checksummed_count": 10, "package_files_checksum_failed_count": 0, - "package_files_synced_count": 10, - "package_files_failed_count": 5 + "package_files_registry_count": 10, + "package_files_synced_count": 6, + "package_files_failed_count": 3 } ] ``` diff --git a/doc/api/graphql/getting_started.md b/doc/api/graphql/getting_started.md index 901bf85ff40..bf8a2120734 100644 --- a/doc/api/graphql/getting_started.md +++ b/doc/api/graphql/getting_started.md @@ -147,7 +147,7 @@ Example: Let's have some tea - add a `:tea:` reaction emoji to an issue. ```graphql mutation { - addAwardEmoji(input: { awardableId: "gid://gitlab/Issue/27039960", + awardEmojiAdd(input: { awardableId: "gid://gitlab/Issue/27039960", name: "tea" }) { awardEmoji { diff --git a/doc/api/graphql/reference/gitlab_schema.graphql b/doc/api/graphql/reference/gitlab_schema.graphql index c4bbe7d969d..2ed6bec104d 100644 --- a/doc/api/graphql/reference/gitlab_schema.graphql +++ b/doc/api/graphql/reference/gitlab_schema.graphql @@ -260,6 +260,11 @@ type AlertManagementAlert implements Noteable { issueIid: ID """ + URL for metrics embed for the alert + """ + metricsDashboardUrl: String + + """ Monitoring tool the alert came from """ monitoringTool: String @@ -390,12 +395,12 @@ enum AlertManagementAlertSort { EVENT_COUNT_DESC """ - Severity by ascending order + Severity from less critical to more critical """ SEVERITY_ASC """ - Severity by descending order + Severity from more critical to less critical """ SEVERITY_DESC @@ -410,12 +415,12 @@ enum AlertManagementAlertSort { STARTED_AT_DESC """ - Status by ascending order + Status by order: Ignored > Resolved > Acknowledged > Triggered """ STATUS_ASC """ - Status by descending order + Status by order: Triggered > Acknowledged > Resolved > Ignored """ STATUS_DESC @@ -598,6 +603,61 @@ type AlertSetAssigneesPayload { The issue created after mutation """ issue: Issue + + """ + The todo after mutation + """ + todo: Todo +} + +""" +Autogenerated input type of AlertTodoCreate +""" +input AlertTodoCreateInput { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + The iid of the alert to mutate + """ + iid: String! + + """ + The project the alert to mutate is in + """ + projectPath: ID! +} + +""" +Autogenerated return type of AlertTodoCreate +""" +type AlertTodoCreatePayload { + """ + The alert after mutation + """ + alert: AlertManagementAlert + + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Errors encountered during execution of the mutation. + """ + errors: [String!]! + + """ + The issue created after mutation + """ + issue: Issue + + """ + The todo after mutation + """ + todo: Todo } """ @@ -635,6 +695,131 @@ type AwardEmoji { user: User! } +""" +Autogenerated input type of AwardEmojiAdd +""" +input AwardEmojiAddInput { + """ + The global id of the awardable resource + """ + awardableId: ID! + + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + The emoji name + """ + name: String! +} + +""" +Autogenerated return type of AwardEmojiAdd +""" +type AwardEmojiAddPayload { + """ + The award emoji after mutation + """ + awardEmoji: AwardEmoji + + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Errors encountered during execution of the mutation. + """ + errors: [String!]! +} + +""" +Autogenerated input type of AwardEmojiRemove +""" +input AwardEmojiRemoveInput { + """ + The global id of the awardable resource + """ + awardableId: ID! + + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + The emoji name + """ + name: String! +} + +""" +Autogenerated return type of AwardEmojiRemove +""" +type AwardEmojiRemovePayload { + """ + The award emoji after mutation + """ + awardEmoji: AwardEmoji + + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Errors encountered during execution of the mutation. + """ + errors: [String!]! +} + +""" +Autogenerated input type of AwardEmojiToggle +""" +input AwardEmojiToggleInput { + """ + The global id of the awardable resource + """ + awardableId: ID! + + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + The emoji name + """ + name: String! +} + +""" +Autogenerated return type of AwardEmojiToggle +""" +type AwardEmojiTogglePayload { + """ + The award emoji after mutation + """ + awardEmoji: AwardEmoji + + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Errors encountered during execution of the mutation. + """ + errors: [String!]! + + """ + Indicates the status of the emoji. True if the toggle awarded the emoji, and false if the toggle removed the emoji. + """ + toggledOn: Boolean! +} + type BaseService implements Service { """ Indicates if the service is active @@ -664,6 +849,11 @@ type Blob implements Entry { lfsOid: String """ + Blob mode in numeric format + """ + mode: String + + """ Name of the entry """ name: String! @@ -1222,6 +1412,51 @@ enum CommitEncoding { } """ +Represents a ComplianceFramework associated with a Project +""" +type ComplianceFramework { + """ + Name of the compliance framework + """ + name: ProjectSettingEnum! +} + +""" +The connection type for ComplianceFramework. +""" +type ComplianceFrameworkConnection { + """ + A list of edges. + """ + edges: [ComplianceFrameworkEdge] + + """ + A list of nodes. + """ + nodes: [ComplianceFramework] + + """ + Information to aid in pagination. + """ + pageInfo: PageInfo! +} + +""" +An edge in a connection. +""" +type ComplianceFrameworkEdge { + """ + A cursor for use in pagination. + """ + cursor: String! + + """ + The item at the end of the edge. + """ + node: ComplianceFramework +} + +""" A tag expiration policy designed to keep only the images that matter most """ type ContainerExpirationPolicy { @@ -1248,12 +1483,12 @@ type ContainerExpirationPolicy { """ Tags with names matching this regex pattern will expire """ - nameRegex: String + nameRegex: UntrustedRegexp """ Tags with names matching this regex pattern will be preserved """ - nameRegexKeep: String + nameRegexKeep: UntrustedRegexp """ Next time that this container expiration policy will get executed @@ -1395,6 +1630,11 @@ type CreateAlertIssuePayload { The issue created after mutation """ issue: Issue + + """ + The todo after mutation + """ + todo: Todo } """ @@ -1517,6 +1757,11 @@ input CreateDiffNoteInput { clientMutationId: String """ + The confidentiality flag of a note. Default is false. + """ + confidential: Boolean + + """ The global id of the resource to add a note to """ noteableId: ID! @@ -1642,6 +1887,11 @@ input CreateImageDiffNoteInput { clientMutationId: String """ + The confidentiality flag of a note. Default is false. + """ + confidential: Boolean + + """ The global id of the resource to add a note to """ noteableId: ID! @@ -1742,6 +1992,11 @@ input CreateNoteInput { clientMutationId: String """ + The confidentiality flag of a note. Default is false. + """ + confidential: Boolean + + """ The global id of the discussion this note is in reply to """ discussionId: ID @@ -1824,7 +2079,7 @@ input CreateSnippetInput { """ Content of the snippet """ - content: String! + content: String """ Description of the snippet @@ -1837,6 +2092,11 @@ input CreateSnippetInput { fileName: String """ + The snippet files to create + """ + files: [SnippetFileInputType!] + + """ The project full path the snippet is associated with """ projectPath: ID @@ -1885,6 +2145,51 @@ enum DastScanTypeEnum { } """ +Autogenerated input type of DastSiteProfileCreate +""" +input DastSiteProfileCreateInput { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + The project the site profile belongs to. + """ + fullPath: ID! + + """ + The name of the site profile. + """ + profileName: String! + + """ + The URL of the target to be scanned. + """ + targetUrl: String +} + +""" +Autogenerated return type of DastSiteProfileCreate +""" +type DastSiteProfileCreatePayload { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Errors encountered during execution of the mutation. + """ + errors: [String!]! + + """ + ID of the site profile. + """ + id: ID +} + +""" Autogenerated input type of DeleteAnnotation """ input DeleteAnnotationInput { @@ -2946,6 +3251,51 @@ type DiffRefs { startSha: String! } +""" +Changes to a single file +""" +type DiffStats { + """ + Number of lines added to this file + """ + additions: Int! + + """ + Number of lines deleted from this file + """ + deletions: Int! + + """ + File path, relative to repository root + """ + path: String! +} + +""" +Aggregated summary of changes +""" +type DiffStatsSummary { + """ + Number of lines added + """ + additions: Int! + + """ + Number of lines changed + """ + changes: Int! + + """ + Number of lines deleted + """ + deletions: Int! + + """ + Number of files changed + """ + fileCount: Int! +} + type Discussion implements ResolvableInterface { """ Timestamp of the discussion's creation @@ -3294,7 +3644,7 @@ type Epic implements Noteable { last: Int """ - Filter epics by title and description + Search query for epic title or description """ search: String @@ -4081,6 +4431,11 @@ The connection type for EpicIssue. """ type EpicIssueConnection { """ + Total count of collection + """ + count: Int! + + """ A list of edges. """ edges: [EpicIssueEdge] @@ -4571,7 +4926,7 @@ type Group { labelName: [String!] """ - Filter epics by title and description + Search query for epic title or description """ search: String @@ -4648,7 +5003,7 @@ type Group { last: Int """ - Filter epics by title and description + Search query for epic title or description """ search: String @@ -4754,6 +5109,11 @@ type Group { iids: [String!] """ + Iterations applied to the issue + """ + iterationId: [ID] + + """ Labels applied to this issue """ labelName: [String] @@ -4769,7 +5129,7 @@ type Group { milestoneTitle: [String] """ - Search query for finding issues by title or description + Search query for issue title or description """ search: String @@ -4820,6 +5180,21 @@ type Group { first: Int """ + The ID of the Iteration to look up + """ + id: ID + + """ + The internal ID of the Iteration to look up + """ + iid: ID + + """ + Whether to include ancestor iterations. Defaults to true + """ + includeAncestors: Boolean + + """ Returns the last _n_ elements from the list. """ last: Int @@ -5014,11 +5389,21 @@ type Group { shareWithGroupLock: Boolean """ + Total storage limit of the root namespace in bytes + """ + storageSizeLimit: Float + + """ The permission level required to create subgroups within the group """ subgroupCreationLevel: String """ + Date until the temporary storage increase is active + """ + temporaryStorageIncreaseEndsOn: Time + + """ Time logged in issues by group members """ timelogs( @@ -5113,6 +5498,11 @@ type Group { reportType: [VulnerabilityReportType!] """ + Filter vulnerabilities by scanner + """ + scanner: [String!] + + """ Filter vulnerabilities by severity """ severity: [VulnerabilitySeverity!] @@ -5159,6 +5549,31 @@ type Group { ): VulnerabilitiesCountByDayAndSeverityConnection """ + Vulnerability scanners reported on the project vulnerabilties of the group and its subgroups + """ + vulnerabilityScanners( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + ): VulnerabilityScannerConnection + + """ Web URL of the group """ webUrl: String! @@ -5285,6 +5700,31 @@ type InstanceSecurityDashboard { """ last: Int ): ProjectConnection! + + """ + Vulnerability scanners reported on the vulnerabilties from projects selected in Instance Security Dashboard + """ + vulnerabilityScanners( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + ): VulnerabilityScannerConnection } """ @@ -5413,6 +5853,11 @@ type Issue implements Noteable { healthStatus: HealthStatus """ + ID of the issue + """ + id: ID! + + """ Internal ID of the issue """ iid: ID! @@ -5593,6 +6038,11 @@ The connection type for Issue. """ type IssueConnection { """ + Total count of collection + """ + count: Int! + + """ A list of edges. """ edges: [IssueEdge] @@ -5804,6 +6254,51 @@ type IssueSetIterationPayload { } """ +Autogenerated input type of IssueSetLocked +""" +input IssueSetLockedInput { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + The iid of the issue to mutate + """ + iid: String! + + """ + Whether or not to lock discussion on the issue + """ + locked: Boolean! + + """ + The project the issue to mutate is in + """ + projectPath: ID! +} + +""" +Autogenerated return type of IssueSetLocked +""" +type IssueSetLockedPayload { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Errors encountered during execution of the mutation. + """ + errors: [String!]! + + """ + The issue after mutation + """ + issue: Issue +} + +""" Autogenerated input type of IssueSetWeight """ input IssueSetWeightInput { @@ -5962,6 +6457,11 @@ type Iteration { id: ID! """ + Internal ID of the iteration + """ + iid: ID! + + """ Timestamp of the iteration start date """ startDate: Time @@ -6138,6 +6638,11 @@ input JiraImportStartInput { The project to import the Jira project into """ projectPath: ID! + + """ + The mapping of Jira to GitLab users + """ + usersMapping: [JiraUsersMappingInputType!] } """ @@ -6259,7 +6764,7 @@ type JiraService implements Service { active: Boolean """ - List of Jira projects fetched through Jira REST API + List of all Jira projects fetched through Jira REST API """ projects( """ @@ -6296,11 +6801,21 @@ type JiraService implements Service { type JiraUser { """ - Id of the matched GitLab user + ID of the matched GitLab user """ gitlabId: Int """ + Name of the matched GitLab user + """ + gitlabName: String + + """ + Username of the matched GitLab user + """ + gitlabUsername: String + + """ Account id of the Jira user """ jiraAccountId: String! @@ -6316,6 +6831,18 @@ type JiraUser { jiraEmail: String } +input JiraUsersMappingInputType { + """ + Id of the GitLab user + """ + gitlabId: Int + + """ + Jira account id of the user + """ + jiraAccountId: String! +} + type Label { """ Background color of the label @@ -6521,6 +7048,21 @@ type MergeRequest implements Noteable { diffRefs: DiffRefs """ + Details about which files were changed in this merge request + """ + diffStats( + """ + A specific file-path + """ + path: String + ): [DiffStats!] + + """ + Summary of which files were changed in this merge request + """ + diffStatsSummary: DiffStatsSummary + + """ Indicates if comments on the merge request are locked to members only """ discussionLocked: Boolean! @@ -7311,6 +7853,61 @@ enum MergeRequestState { opened } +""" +Autogenerated input type of MergeRequestUpdate +""" +input MergeRequestUpdateInput { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Description of the merge request (Markdown rendered as HTML for caching) + """ + description: String + + """ + The iid of the merge request to mutate + """ + iid: String! + + """ + The project the merge request to mutate is in + """ + projectPath: ID! + + """ + Target branch of the merge request + """ + targetBranch: String + + """ + Title of the merge request + """ + title: String +} + +""" +Autogenerated return type of MergeRequestUpdate +""" +type MergeRequestUpdatePayload { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Errors encountered during execution of the mutation. + """ + errors: [String!]! + + """ + The merge request after mutation + """ + mergeRequest: MergeRequest +} + type Metadata { """ Revision @@ -7477,6 +8074,11 @@ type Milestone { state: MilestoneStateEnum! """ + Milestone statistics + """ + stats: MilestoneStats + + """ Indicates if milestone is at subgroup level """ subgroupMilestone: Boolean! @@ -7538,6 +8140,21 @@ enum MilestoneStateEnum { } """ +Contains statistics about a milestone +""" +type MilestoneStats { + """ + Number of closed issues associated with the milestone + """ + closedIssuesCount: Int + + """ + Total number of issues associated with the milestone + """ + totalIssuesCount: Int +} + +""" The position to which the adjacent object should be moved """ enum MoveType { @@ -7553,10 +8170,14 @@ enum MoveType { } type Mutation { - addAwardEmoji(input: AddAwardEmojiInput!): AddAwardEmojiPayload + addAwardEmoji(input: AddAwardEmojiInput!): AddAwardEmojiPayload @deprecated(reason: "Use awardEmojiAdd. Deprecated in 13.2") addProjectToSecurityDashboard(input: AddProjectToSecurityDashboardInput!): AddProjectToSecurityDashboardPayload adminSidekiqQueuesDeleteJobs(input: AdminSidekiqQueuesDeleteJobsInput!): AdminSidekiqQueuesDeleteJobsPayload alertSetAssignees(input: AlertSetAssigneesInput!): AlertSetAssigneesPayload + alertTodoCreate(input: AlertTodoCreateInput!): AlertTodoCreatePayload + awardEmojiAdd(input: AwardEmojiAddInput!): AwardEmojiAddPayload + awardEmojiRemove(input: AwardEmojiRemoveInput!): AwardEmojiRemovePayload + awardEmojiToggle(input: AwardEmojiToggleInput!): AwardEmojiTogglePayload boardListUpdateLimitMetrics(input: BoardListUpdateLimitMetricsInput!): BoardListUpdateLimitMetricsPayload commitCreate(input: CommitCreateInput!): CommitCreatePayload createAlertIssue(input: CreateAlertIssueInput!): CreateAlertIssuePayload @@ -7569,6 +8190,7 @@ type Mutation { createNote(input: CreateNoteInput!): CreateNotePayload createRequirement(input: CreateRequirementInput!): CreateRequirementPayload createSnippet(input: CreateSnippetInput!): CreateSnippetPayload + dastSiteProfileCreate(input: DastSiteProfileCreateInput!): DastSiteProfileCreatePayload deleteAnnotation(input: DeleteAnnotationInput!): DeleteAnnotationPayload designManagementDelete(input: DesignManagementDeleteInput!): DesignManagementDeletePayload designManagementUpload(input: DesignManagementUploadInput!): DesignManagementUploadPayload @@ -7586,6 +8208,7 @@ type Mutation { issueSetConfidential(input: IssueSetConfidentialInput!): IssueSetConfidentialPayload issueSetDueDate(input: IssueSetDueDateInput!): IssueSetDueDatePayload issueSetIteration(input: IssueSetIterationInput!): IssueSetIterationPayload + issueSetLocked(input: IssueSetLockedInput!): IssueSetLockedPayload issueSetWeight(input: IssueSetWeightInput!): IssueSetWeightPayload jiraImportStart(input: JiraImportStartInput!): JiraImportStartPayload jiraImportUsers(input: JiraImportUsersInput!): JiraImportUsersPayload @@ -7597,14 +8220,19 @@ type Mutation { mergeRequestSetMilestone(input: MergeRequestSetMilestoneInput!): MergeRequestSetMilestonePayload mergeRequestSetSubscription(input: MergeRequestSetSubscriptionInput!): MergeRequestSetSubscriptionPayload mergeRequestSetWip(input: MergeRequestSetWipInput!): MergeRequestSetWipPayload - removeAwardEmoji(input: RemoveAwardEmojiInput!): RemoveAwardEmojiPayload + + """ + Update attributes of a merge request + """ + mergeRequestUpdate(input: MergeRequestUpdateInput!): MergeRequestUpdatePayload + removeAwardEmoji(input: RemoveAwardEmojiInput!): RemoveAwardEmojiPayload @deprecated(reason: "Use awardEmojiRemove. Deprecated in 13.2") removeProjectFromSecurityDashboard(input: RemoveProjectFromSecurityDashboardInput!): RemoveProjectFromSecurityDashboardPayload runDastScan(input: RunDASTScanInput!): RunDASTScanPayload todoMarkDone(input: TodoMarkDoneInput!): TodoMarkDonePayload todoRestore(input: TodoRestoreInput!): TodoRestorePayload todoRestoreMany(input: TodoRestoreManyInput!): TodoRestoreManyPayload todosMarkAllDone(input: TodosMarkAllDoneInput!): TodosMarkAllDonePayload - toggleAwardEmoji(input: ToggleAwardEmojiInput!): ToggleAwardEmojiPayload + toggleAwardEmoji(input: ToggleAwardEmojiInput!): ToggleAwardEmojiPayload @deprecated(reason: "Use awardEmojiToggle. Deprecated in 13.2") updateAlertStatus(input: UpdateAlertStatusInput!): UpdateAlertStatusPayload updateContainerExpirationPolicy(input: UpdateContainerExpirationPolicyInput!): UpdateContainerExpirationPolicyPayload updateEpic(input: UpdateEpicInput!): UpdateEpicPayload @@ -7616,6 +8244,7 @@ type Mutation { """ updateImageDiffNote(input: UpdateImageDiffNoteInput!): UpdateImageDiffNotePayload updateIssue(input: UpdateIssueInput!): UpdateIssuePayload + updateIteration(input: UpdateIterationInput!): UpdateIterationPayload """ Updates a Note. If the body of the Note contains only quick actions, the Note @@ -7733,6 +8362,16 @@ type Namespace { rootStorageStatistics: RootStorageStatistics """ + Total storage limit of the root namespace in bytes + """ + storageSizeLimit: Float + + """ + Date until the temporary storage increase is active + """ + temporaryStorageIncreaseEndsOn: Time + + """ Visibility of the namespace """ visibility: String @@ -7845,6 +8484,11 @@ type Note implements ResolvableInterface { system: Boolean! """ + Name of the icon corresponding to a system note + """ + systemNoteIconName: String + + """ Timestamp of the note's last activity """ updatedAt: Time! @@ -8463,6 +9107,31 @@ type Project { ): BoardConnection """ + Compliance frameworks associated with the project + """ + complianceFrameworks( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + ): ComplianceFrameworkConnection + + """ The container expiration policy of the project """ containerExpirationPolicy: ContainerExpirationPolicy @@ -8517,7 +9186,7 @@ type Project { name: String """ - Search query + Search query for environment name """ search: String @@ -8607,6 +9276,11 @@ type Project { iids: [String!] """ + Iterations applied to the issue + """ + iterationId: [ID] + + """ Labels applied to this issue """ labelName: [String] @@ -8617,7 +9291,7 @@ type Project { milestoneTitle: [String] """ - Search query for finding issues by title or description + Search query for issue title or description """ search: String @@ -8702,6 +9376,11 @@ type Project { iids: [String!] """ + Iterations applied to the issue + """ + iterationId: [ID] + + """ Labels applied to this issue """ labelName: [String] @@ -8717,7 +9396,7 @@ type Project { milestoneTitle: [String] """ - Search query for finding issues by title or description + Search query for issue title or description """ search: String @@ -9056,7 +9735,7 @@ type Project { publicJobs: Boolean """ - A single release of the project. Available only when feature flag `graphql_release_data` is enabled + A single release of the project """ release( """ @@ -9066,7 +9745,7 @@ type Project { ): Release """ - Releases of the project. Available only when feature flag `graphql_release_data` is enabled + Releases of the project """ releases( """ @@ -9125,7 +9804,7 @@ type Project { iids: [ID!] """ - Filter requirements by title search + Search query for requirement title """ search: String @@ -9185,7 +9864,7 @@ type Project { last: Int """ - Filter requirements by title search + Search query for requirement title """ search: String @@ -9201,6 +9880,16 @@ type Project { ): RequirementConnection """ + SAST CI configuration for the project + """ + sastCiConfiguration: SastCiConfiguration + + """ + Information about security analyzers used in the project + """ + securityScanners: SecurityScanners + + """ Detailed version of a Sentry error on the project """ sentryDetailedError( @@ -9375,6 +10064,11 @@ type Project { reportType: [VulnerabilityReportType!] """ + Filter vulnerabilities by scanner + """ + scanner: [String!] + + """ Filter vulnerabilities by severity """ severity: [VulnerabilitySeverity!] @@ -9386,6 +10080,31 @@ type Project { ): VulnerabilityConnection """ + Vulnerability scanners reported on the project vulnerabilties + """ + vulnerabilityScanners( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + ): VulnerabilityScannerConnection + + """ Counts for each severity of vulnerability of the project """ vulnerabilitySeveritiesCount: VulnerabilitySeveritiesCount @@ -9733,6 +10452,17 @@ type ProjectPermissions { uploadFile: Boolean! } +""" +Names of compliance frameworks that can be assigned to a Project +""" +enum ProjectSettingEnum { + gdpr + hipaa + pci_dss + soc_2 + sox +} + type ProjectStatistics { """ Build artifacts size of the project @@ -9760,6 +10490,11 @@ type ProjectStatistics { repositorySize: Float! """ + Snippets size of the project + """ + snippetsSize: Float + + """ Storage size of the project """ storageSize: Float! @@ -9871,7 +10606,7 @@ type Query { membership: Boolean """ - Search criteria + Search query for project name, path, or description """ search: String ): ProjectConnection @@ -9932,7 +10667,7 @@ type Query { ): SnippetConnection """ - Find a user on this instance + Find a user """ user( """ @@ -10021,6 +10756,11 @@ type Query { reportType: [VulnerabilityReportType!] """ + Filter vulnerabilities by scanner + """ + scanner: [String!] + + """ Filter vulnerabilities by severity """ severity: [VulnerabilitySeverity!] @@ -10092,6 +10832,9 @@ enum RegistryState { SYNCED } +""" +Represents a release +""" type Release { """ Assets of the release @@ -10149,6 +10892,11 @@ type Release { ): ReleaseEvidenceConnection """ + Links of the release + """ + links: ReleaseLinks + + """ Milestones associated to the release """ milestones( @@ -10186,7 +10934,7 @@ type Release { """ Name of the tag associated with the release """ - tagName: String! + tagName: String """ Relative web path to the tag associated with the release @@ -10194,11 +10942,104 @@ type Release { tagPath: String } +""" +Represents an asset link associated with a release +""" +type ReleaseAssetLink { + """ + Indicates the link points to an external resource + """ + external: Boolean + + """ + ID of the link + """ + id: ID! + + """ + Type of the link: `other`, `runbook`, `image`, `package`; defaults to `other` + """ + linkType: ReleaseAssetLinkType + + """ + Name of the link + """ + name: String + + """ + URL of the link + """ + url: String +} + +""" +The connection type for ReleaseAssetLink. +""" +type ReleaseAssetLinkConnection { + """ + A list of edges. + """ + edges: [ReleaseAssetLinkEdge] + + """ + A list of nodes. + """ + nodes: [ReleaseAssetLink] + + """ + Information to aid in pagination. + """ + pageInfo: PageInfo! +} + +""" +An edge in a connection. +""" +type ReleaseAssetLinkEdge { + """ + A cursor for use in pagination. + """ + cursor: String! + + """ + The item at the end of the edge. + """ + node: ReleaseAssetLink +} + +""" +Type of the link: `other`, `runbook`, `image`, `package`; defaults to `other` +""" +enum ReleaseAssetLinkType { + """ + Image link type + """ + IMAGE + + """ + Other link type + """ + OTHER + + """ + Package link type + """ + PACKAGE + + """ + Runbook link type + """ + RUNBOOK +} + +""" +A container for all assets associated with a release +""" type ReleaseAssets { """ Number of assets of the release """ - assetsCount: Int + count: Int """ Asset links of the release @@ -10223,7 +11064,7 @@ type ReleaseAssets { Returns the last _n_ elements from the list. """ last: Int - ): ReleaseLinkConnection + ): ReleaseAssetLinkConnection """ Sources of the release @@ -10346,93 +11187,31 @@ type ReleaseEvidenceEdge { node: ReleaseEvidence } -type ReleaseLink { +type ReleaseLinks { """ - Indicates the link points to an external resource + HTTP URL of the release's edit page """ - external: Boolean + editUrl: String """ - ID of the link + HTTP URL of the issues page filtered by this release """ - id: ID! + issuesUrl: String """ - Type of the link: `other`, `runbook`, `image`, `package`; defaults to `other` + HTTP URL of the merge request page filtered by this release """ - linkType: ReleaseLinkType + mergeRequestsUrl: String """ - Name of the link + HTTP URL of the release """ - name: String - - """ - URL of the link - """ - url: String + selfUrl: String } """ -The connection type for ReleaseLink. -""" -type ReleaseLinkConnection { - """ - A list of edges. - """ - edges: [ReleaseLinkEdge] - - """ - A list of nodes. - """ - nodes: [ReleaseLink] - - """ - Information to aid in pagination. - """ - pageInfo: PageInfo! -} - -""" -An edge in a connection. +Represents the source code attached to a release in a particular format """ -type ReleaseLinkEdge { - """ - A cursor for use in pagination. - """ - cursor: String! - - """ - The item at the end of the edge. - """ - node: ReleaseLink -} - -""" -Type of the link: `other`, `runbook`, `image`, `package`; defaults to `other` -""" -enum ReleaseLinkType { - """ - Image link type - """ - IMAGE - - """ - Other link type - """ - OTHER - - """ - Package link type - """ - PACKAGE - - """ - Runbook link type - """ - RUNBOOK -} - type ReleaseSource { """ Format of the source @@ -10799,6 +11578,11 @@ type RootStorageStatistics { repositorySize: Float! """ + The snippets size in bytes + """ + snippetsSize: Float! + + """ The total storage in bytes """ storageSize: Float! @@ -10860,6 +11644,341 @@ type RunDASTScanPayload { } """ +Represents a CI configuration of SAST +""" +type SastCiConfiguration { + """ + List of analyzers entities attached to SAST configuration. + """ + analyzers( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + ): SastCiConfigurationAnalyzersEntityConnection + + """ + List of global entities related to SAST configuration. + """ + global( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + ): SastCiConfigurationEntityConnection + + """ + List of pipeline entities related to SAST configuration. + """ + pipeline( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + ): SastCiConfigurationEntityConnection +} + +""" +Represents an analyzer entity in SAST CI configuration +""" +type SastCiConfigurationAnalyzersEntity { + """ + Analyzer description that is displayed on the form. + """ + description: String + + """ + Indicates whether an analyzer is enabled. + """ + enabled: Boolean + + """ + Analyzer label used in the config UI. + """ + label: String + + """ + Name of the analyzer. + """ + name: String +} + +""" +The connection type for SastCiConfigurationAnalyzersEntity. +""" +type SastCiConfigurationAnalyzersEntityConnection { + """ + A list of edges. + """ + edges: [SastCiConfigurationAnalyzersEntityEdge] + + """ + A list of nodes. + """ + nodes: [SastCiConfigurationAnalyzersEntity] + + """ + Information to aid in pagination. + """ + pageInfo: PageInfo! +} + +""" +An edge in a connection. +""" +type SastCiConfigurationAnalyzersEntityEdge { + """ + A cursor for use in pagination. + """ + cursor: String! + + """ + The item at the end of the edge. + """ + node: SastCiConfigurationAnalyzersEntity +} + +""" +Represents an entity in SAST CI configuration +""" +type SastCiConfigurationEntity { + """ + Default value that is used if value is empty. + """ + defaultValue: String + + """ + Entity description that is displayed on the form. + """ + description: String + + """ + CI keyword of entity. + """ + field: String + + """ + Label for entity used in the form. + """ + label: String + + """ + Different possible values of the field. + """ + options( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + ): SastCiConfigurationOptionsEntityConnection + + """ + Type of the field value. + """ + type: String + + """ + Current value of the entity. + """ + value: String +} + +""" +The connection type for SastCiConfigurationEntity. +""" +type SastCiConfigurationEntityConnection { + """ + A list of edges. + """ + edges: [SastCiConfigurationEntityEdge] + + """ + A list of nodes. + """ + nodes: [SastCiConfigurationEntity] + + """ + Information to aid in pagination. + """ + pageInfo: PageInfo! +} + +""" +An edge in a connection. +""" +type SastCiConfigurationEntityEdge { + """ + A cursor for use in pagination. + """ + cursor: String! + + """ + The item at the end of the edge. + """ + node: SastCiConfigurationEntity +} + +""" +Represents an entity for options in SAST CI configuration +""" +type SastCiConfigurationOptionsEntity { + """ + Label of option entity. + """ + label: String + + """ + Value of option entity. + """ + value: String +} + +""" +The connection type for SastCiConfigurationOptionsEntity. +""" +type SastCiConfigurationOptionsEntityConnection { + """ + A list of edges. + """ + edges: [SastCiConfigurationOptionsEntityEdge] + + """ + A list of nodes. + """ + nodes: [SastCiConfigurationOptionsEntity] + + """ + Information to aid in pagination. + """ + pageInfo: PageInfo! +} + +""" +An edge in a connection. +""" +type SastCiConfigurationOptionsEntityEdge { + """ + A cursor for use in pagination. + """ + cursor: String! + + """ + The item at the end of the edge. + """ + node: SastCiConfigurationOptionsEntity +} + +""" +Represents a resource scanned by a security scan +""" +type ScannedResource { + """ + The HTTP request method used to access the URL + """ + requestMethod: String + + """ + The URL scanned by the scanner + """ + url: String +} + +""" +The connection type for ScannedResource. +""" +type ScannedResourceConnection { + """ + A list of edges. + """ + edges: [ScannedResourceEdge] + + """ + A list of nodes. + """ + nodes: [ScannedResource] + + """ + Information to aid in pagination. + """ + pageInfo: PageInfo! +} + +""" +An edge in a connection. +""" +type ScannedResourceEdge { + """ + A cursor for use in pagination. + """ + cursor: String! + + """ + The item at the end of the edge. + """ + node: ScannedResource +} + +""" Represents summary of a security report """ type SecurityReportSummary { @@ -10869,6 +11988,11 @@ type SecurityReportSummary { containerScanning: SecurityReportSummarySection """ + Aggregated counts for the coverage_fuzzing scan + """ + coverageFuzzing: SecurityReportSummarySection + + """ Aggregated counts for the dast scan """ dast: SecurityReportSummarySection @@ -10894,17 +12018,78 @@ Represents a section of a summary of a security report """ type SecurityReportSummarySection { """ + A list of the first 20 scanned resources + """ + scannedResources( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + ): ScannedResourceConnection + + """ Total number of scanned resources """ scannedResourcesCount: Int """ + Path to download all the scanned resources in CSV format + """ + scannedResourcesCsvPath: String + + """ Total number of vulnerabilities """ vulnerabilitiesCount: Int } """ +The type of the security scanner. +""" +enum SecurityScannerType { + CONTAINER_SCANNING + DAST + DEPENDENCY_SCANNING + SAST + SECRET_DETECTION +} + +""" +Represents a list of security scanners +""" +type SecurityScanners { + """ + List of analyzers which are available for the project. + """ + available: [SecurityScannerType!] + + """ + List of analyzers which are enabled for the project. + """ + enabled: [SecurityScannerType!] + + """ + List of analyzers which ran successfully in the latest pipeline. + """ + pipelineRun: [SecurityScannerType!] +} + +""" A Sentry error. """ type SentryDetailedError { @@ -10934,11 +12119,16 @@ type SentryDetailedError { firstReleaseLastCommit: String """ - Release version the error was first seen + Release short version the error was first seen """ firstReleaseShortVersion: String """ + Release version the error was first seen + """ + firstReleaseVersion: String + + """ Timestamp when the error was first seen """ firstSeen: Time! @@ -10974,11 +12164,16 @@ type SentryDetailedError { lastReleaseLastCommit: String """ - Release version the error was last seen + Release short version the error was last seen """ lastReleaseShortVersion: String """ + Release version the error was last seen + """ + lastReleaseVersion: String + + """ Timestamp when the error was last seen """ lastSeen: Time! @@ -11178,7 +12373,7 @@ type SentryErrorCollection { last: Int """ - Search term for the Sentry error. + Search query for the Sentry error details """ searchTerm: String @@ -11401,6 +12596,7 @@ enum ServiceType { BUGZILLA_SERVICE BUILDKITE_SERVICE CAMPFIRE_SERVICE + CONFLUENCE_SERVICE CUSTOM_ISSUE_TRACKER_SERVICE DISCORD_SERVICE DRONE_CI_SERVICE @@ -11710,6 +12906,41 @@ type SnippetEdge { node: Snippet } +""" +Type of a snippet file input action +""" +enum SnippetFileInputActionEnum { + create + delete + move + update +} + +""" +Represents an action to perform over a snippet file +""" +input SnippetFileInputType { + """ + Type of input action + """ + action: SnippetFileInputActionEnum! + + """ + Snippet file content + """ + content: String + + """ + Path of the snippet file + """ + filePath: String! + + """ + Previous path of the snippet file + """ + previousPath: String +} + type SnippetPermissions { """ Indicates the user can perform `admin_snippet` on this resource @@ -11923,6 +13154,7 @@ type TestReportEdge { State of a test report """ enum TestReportState { + FAILED PASSED } @@ -12168,9 +13400,14 @@ type TodoRestoreManyPayload { errors: [String!]! """ - The ids of the updated todo items + Updated todos + """ + todos: [Todo!]! + + """ + The ids of the updated todo items. Deprecated in 13.2: Use todos """ - updatedIds: [ID!]! + updatedIds: [ID!]! @deprecated(reason: "Use todos. Deprecated in 13.2") } """ @@ -12200,6 +13437,11 @@ enum TodoStateEnum { enum TodoTargetEnum { """ + An Alert + """ + ALERT + + """ A Commit """ COMMIT @@ -12250,9 +13492,14 @@ type TodosMarkAllDonePayload { errors: [String!]! """ - Ids of the updated todos + Updated todos + """ + todos: [Todo!]! + + """ + Ids of the updated todos. Deprecated in 13.2: Use todos """ - updatedIds: [ID!]! + updatedIds: [ID!]! @deprecated(reason: "Use todos. Deprecated in 13.2") } """ @@ -12463,6 +13710,11 @@ enum TypeEnum { } """ +A regexp containing patterns sourced from user input +""" +scalar UntrustedRegexp + +""" Autogenerated input type of UpdateAlertStatus """ input UpdateAlertStatusInput { @@ -12510,6 +13762,11 @@ type UpdateAlertStatusPayload { The issue created after mutation """ issue: Issue + + """ + The todo after mutation + """ + todo: Todo } """ @@ -12537,6 +13794,16 @@ input UpdateContainerExpirationPolicyInput { keepN: ContainerExpirationPolicyKeepEnum """ + Tags with names matching this regex pattern will expire + """ + nameRegex: UntrustedRegexp + + """ + Tags with names matching this regex pattern will be preserved + """ + nameRegexKeep: UntrustedRegexp + + """ Tags older that this will expire """ olderThan: ContainerExpirationPolicyOlderThanEnum @@ -12790,6 +14057,66 @@ type UpdateIssuePayload { } """ +Autogenerated input type of UpdateIteration +""" +input UpdateIterationInput { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + The description of the iteration + """ + description: String + + """ + The end date of the iteration + """ + dueDate: String + + """ + The group of the iteration + """ + groupPath: ID! + + """ + The id of the iteration + """ + id: ID! + + """ + The start date of the iteration + """ + startDate: String + + """ + The title of the iteration + """ + title: String +} + +""" +Autogenerated return type of UpdateIteration +""" +type UpdateIterationPayload { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Errors encountered during execution of the mutation. + """ + errors: [String!]! + + """ + The updated iteration + """ + iteration: Iteration +} + +""" Autogenerated input type of UpdateNote """ input UpdateNoteInput { @@ -12904,6 +14231,11 @@ input UpdateSnippetInput { fileName: String """ + The snippet files to update + """ + files: [SnippetFileInputType!] + + """ The global id of the snippet to update """ id: ID! @@ -13387,6 +14719,11 @@ type Vulnerability { id: ID! """ + Identifiers of the vulnerability. + """ + identifiers: [VulnerabilityIdentifier!]! + + """ List of issue links related to the vulnerability """ issueLinks( @@ -13422,17 +14759,28 @@ type Vulnerability { location: VulnerabilityLocation """ + Primary identifier of the vulnerability. + """ + primaryIdentifier: VulnerabilityIdentifier + + """ The project on which the vulnerability was found """ project: Project """ Type of the security report that found the vulnerability (SAST, - DEPENDENCY_SCANNING, CONTAINER_SCANNING, DAST, SECRET_DETECTION) + DEPENDENCY_SCANNING, CONTAINER_SCANNING, DAST, SECRET_DETECTION, + COVERAGE_FUZZING) """ reportType: VulnerabilityReportType """ + Scanner metadata for the vulnerability. + """ + scanner: VulnerabilityScanner + + """ Severity of the vulnerability (INFO, UNKNOWN, LOW, MEDIUM, HIGH, CRITICAL) """ severity: VulnerabilitySeverity @@ -13499,6 +14847,31 @@ type VulnerabilityEdge { } """ +Represents a vulnerability identifier. +""" +type VulnerabilityIdentifier { + """ + External ID of the vulnerability identifier + """ + externalId: String + + """ + External type of the vulnerability identifier + """ + externalType: String + + """ + Name of the vulnerability identifier + """ + name: String + + """ + URL of the vulnerability identifier + """ + url: String +} + +""" Represents an issue link of a vulnerability. """ type VulnerabilityIssueLink { @@ -13736,6 +15109,7 @@ The type of the security scan that found the vulnerability. """ enum VulnerabilityReportType { CONTAINER_SCANNING + COVERAGE_FUZZING DAST DEPENDENCY_SCANNING SAST @@ -13743,6 +15117,66 @@ enum VulnerabilityReportType { } """ +Represents a vulnerability scanner. +""" +type VulnerabilityScanner { + """ + External ID of the vulnerability scanner + """ + externalId: String + + """ + Name of the vulnerability scanner + """ + name: String + + """ + Type of the vulnerability report + """ + reportType: VulnerabilityReportType + + """ + Vendor of the vulnerability scanner + """ + vendor: String +} + +""" +The connection type for VulnerabilityScanner. +""" +type VulnerabilityScannerConnection { + """ + A list of edges. + """ + edges: [VulnerabilityScannerEdge] + + """ + A list of nodes. + """ + nodes: [VulnerabilityScanner] + + """ + Information to aid in pagination. + """ + pageInfo: PageInfo! +} + +""" +An edge in a connection. +""" +type VulnerabilityScannerEdge { + """ + A cursor for use in pagination. + """ + cursor: String! + + """ + The item at the end of the edge. + """ + node: VulnerabilityScanner +} + +""" Represents vulnerability counts by severity """ type VulnerabilitySeveritiesCount { diff --git a/doc/api/graphql/reference/gitlab_schema.json b/doc/api/graphql/reference/gitlab_schema.json index d2bc599ff9d..80aaa4aa949 100644 --- a/doc/api/graphql/reference/gitlab_schema.json +++ b/doc/api/graphql/reference/gitlab_schema.json @@ -717,6 +717,20 @@ "deprecationReason": null }, { + "name": "metricsDashboardUrl", + "description": "URL for metrics embed for the alert", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "monitoringTool", "description": "Monitoring tool the alert came from", "args": [ @@ -1089,25 +1103,25 @@ }, { "name": "SEVERITY_ASC", - "description": "Severity by ascending order", + "description": "Severity from less critical to more critical", "isDeprecated": false, "deprecationReason": null }, { "name": "SEVERITY_DESC", - "description": "Severity by descending order", + "description": "Severity from more critical to less critical", "isDeprecated": false, "deprecationReason": null }, { "name": "STATUS_ASC", - "description": "Status by ascending order", + "description": "Status by order: Ignored > Resolved > Acknowledged > Triggered", "isDeprecated": false, "deprecationReason": null }, { "name": "STATUS_DESC", - "description": "Status by descending order", + "description": "Status by order: Triggered > Acknowledged > Resolved > Ignored", "isDeprecated": false, "deprecationReason": null } @@ -1446,6 +1460,164 @@ }, "isDeprecated": false, "deprecationReason": null + }, + { + "name": "todo", + "description": "The todo after mutation", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "Todo", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "INPUT_OBJECT", + "name": "AlertTodoCreateInput", + "description": "Autogenerated input type of AlertTodoCreate", + "fields": null, + "inputFields": [ + { + "name": "projectPath", + "description": "The project the alert to mutate is in", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "iid", + "description": "The iid of the alert to mutate", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "AlertTodoCreatePayload", + "description": "Autogenerated return type of AlertTodoCreate", + "fields": [ + { + "name": "alert", + "description": "The alert after mutation", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "AlertManagementAlert", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "errors", + "description": "Errors encountered during execution of the mutation.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "issue", + "description": "The issue created after mutation", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "Issue", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "todo", + "description": "The todo after mutation", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "Todo", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null } ], "inputFields": null, @@ -1577,6 +1749,372 @@ "possibleTypes": null }, { + "kind": "INPUT_OBJECT", + "name": "AwardEmojiAddInput", + "description": "Autogenerated input type of AwardEmojiAdd", + "fields": null, + "inputFields": [ + { + "name": "awardableId", + "description": "The global id of the awardable resource", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "name", + "description": "The emoji name", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "AwardEmojiAddPayload", + "description": "Autogenerated return type of AwardEmojiAdd", + "fields": [ + { + "name": "awardEmoji", + "description": "The award emoji after mutation", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "AwardEmoji", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "errors", + "description": "Errors encountered during execution of the mutation.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "INPUT_OBJECT", + "name": "AwardEmojiRemoveInput", + "description": "Autogenerated input type of AwardEmojiRemove", + "fields": null, + "inputFields": [ + { + "name": "awardableId", + "description": "The global id of the awardable resource", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "name", + "description": "The emoji name", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "AwardEmojiRemovePayload", + "description": "Autogenerated return type of AwardEmojiRemove", + "fields": [ + { + "name": "awardEmoji", + "description": "The award emoji after mutation", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "AwardEmoji", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "errors", + "description": "Errors encountered during execution of the mutation.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "INPUT_OBJECT", + "name": "AwardEmojiToggleInput", + "description": "Autogenerated input type of AwardEmojiToggle", + "fields": null, + "inputFields": [ + { + "name": "awardableId", + "description": "The global id of the awardable resource", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "name", + "description": "The emoji name", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "AwardEmojiTogglePayload", + "description": "Autogenerated return type of AwardEmojiToggle", + "fields": [ + { + "name": "awardEmoji", + "description": "The award emoji after mutation", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "AwardEmoji", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "errors", + "description": "Errors encountered during execution of the mutation.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "toggledOn", + "description": "Indicates the status of the emoji. True if the toggle awarded the emoji, and false if the toggle removed the emoji.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { "kind": "OBJECT", "name": "BaseService", "description": null, @@ -1677,6 +2215,20 @@ "deprecationReason": null }, { + "name": "mode", + "description": "Blob mode in numeric format", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "name", "description": "Name of the entry", "args": [ @@ -3243,6 +3795,149 @@ }, { "kind": "OBJECT", + "name": "ComplianceFramework", + "description": "Represents a ComplianceFramework associated with a Project", + "fields": [ + { + "name": "name", + "description": "Name of the compliance framework", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "ENUM", + "name": "ProjectSettingEnum", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "ComplianceFrameworkConnection", + "description": "The connection type for ComplianceFramework.", + "fields": [ + { + "name": "edges", + "description": "A list of edges.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "ComplianceFrameworkEdge", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "nodes", + "description": "A list of nodes.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "ComplianceFramework", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "pageInfo", + "description": "Information to aid in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "PageInfo", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "ComplianceFrameworkEdge", + "description": "An edge in a connection.", + "fields": [ + { + "name": "cursor", + "description": "A cursor for use in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "node", + "description": "The item at the end of the edge.", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "ComplianceFramework", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", "name": "ContainerExpirationPolicy", "description": "A tag expiration policy designed to keep only the images that matter most", "fields": [ @@ -3322,7 +4017,7 @@ ], "type": { "kind": "SCALAR", - "name": "String", + "name": "UntrustedRegexp", "ofType": null }, "isDeprecated": false, @@ -3336,7 +4031,7 @@ ], "type": { "kind": "SCALAR", - "name": "String", + "name": "UntrustedRegexp", "ofType": null }, "isDeprecated": false, @@ -3640,6 +4335,20 @@ }, "isDeprecated": false, "deprecationReason": null + }, + { + "name": "todo", + "description": "The todo after mutation", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "Todo", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null } ], "inputFields": null, @@ -3974,6 +4683,16 @@ "defaultValue": null }, { + "name": "confidential", + "description": "The confidentiality flag of a note. Default is false.", + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "defaultValue": null + }, + { "name": "position", "description": "The position of this note on a diff", "type": { @@ -4312,6 +5031,16 @@ "defaultValue": null }, { + "name": "confidential", + "description": "The confidentiality flag of a note. Default is false.", + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "defaultValue": null + }, + { "name": "position", "description": "The position of this note on a diff", "type": { @@ -4584,6 +5313,16 @@ "defaultValue": null }, { + "name": "confidential", + "description": "The confidentiality flag of a note. Default is false.", + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "defaultValue": null + }, + { "name": "discussionId", "description": "The global id of the discussion this note is in reply to", "type": { @@ -4825,13 +5564,9 @@ "name": "content", "description": "Content of the snippet", "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } + "kind": "SCALAR", + "name": "String", + "ofType": null }, "defaultValue": null }, @@ -4888,6 +5623,24 @@ "defaultValue": null }, { + "name": "files", + "description": "The snippet files to create", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "SnippetFileInputType", + "ofType": null + } + } + }, + "defaultValue": null + }, + { "name": "clientMutationId", "description": "A unique identifier for the client performing the mutation.", "type": { @@ -4988,6 +5741,132 @@ }, { "kind": "INPUT_OBJECT", + "name": "DastSiteProfileCreateInput", + "description": "Autogenerated input type of DastSiteProfileCreate", + "fields": null, + "inputFields": [ + { + "name": "fullPath", + "description": "The project the site profile belongs to.", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "profileName", + "description": "The name of the site profile.", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "targetUrl", + "description": "The URL of the target to be scanned.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "DastSiteProfileCreatePayload", + "description": "Autogenerated return type of DastSiteProfileCreate", + "fields": [ + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "errors", + "description": "Errors encountered during execution of the mutation.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "id", + "description": "ID of the site profile.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "INPUT_OBJECT", "name": "DeleteAnnotationInput", "description": "Autogenerated input type of DeleteAnnotation", "fields": null, @@ -8130,6 +9009,158 @@ }, { "kind": "OBJECT", + "name": "DiffStats", + "description": "Changes to a single file", + "fields": [ + { + "name": "additions", + "description": "Number of lines added to this file", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "deletions", + "description": "Number of lines deleted from this file", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "path", + "description": "File path, relative to repository root", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "DiffStatsSummary", + "description": "Aggregated summary of changes", + "fields": [ + { + "name": "additions", + "description": "Number of lines added", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "changes", + "description": "Number of lines changed", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "deletions", + "description": "Number of lines deleted", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "fileCount", + "description": "Number of files changed", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", "name": "Discussion", "description": null, "fields": [ @@ -9117,7 +10148,7 @@ }, { "name": "search", - "description": "Filter epics by title and description", + "description": "Search query for epic title or description", "type": { "kind": "SCALAR", "name": "String", @@ -11358,6 +12389,24 @@ "description": "The connection type for EpicIssue.", "fields": [ { + "name": "count", + "description": "Total count of collection", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "edges", "description": "A list of edges.", "args": [ @@ -12696,7 +13745,7 @@ }, { "name": "search", - "description": "Filter epics by title and description", + "description": "Search query for epic title or description", "type": { "kind": "SCALAR", "name": "String", @@ -12825,7 +13874,7 @@ }, { "name": "search", - "description": "Filter epics by title and description", + "description": "Search query for epic title or description", "type": { "kind": "SCALAR", "name": "String", @@ -13164,7 +14213,7 @@ }, { "name": "search", - "description": "Search query for finding issues by title or description", + "description": "Search query for issue title or description", "type": { "kind": "SCALAR", "name": "String", @@ -13183,6 +14232,20 @@ "defaultValue": "created_desc" }, { + "name": "iterationId", + "description": "Iterations applied to the issue", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "defaultValue": null + }, + { "name": "after", "description": "Returns the elements in the list that come after the specified cursor.", "type": { @@ -13276,6 +14339,36 @@ "defaultValue": null }, { + "name": "id", + "description": "The ID of the Iteration to look up", + "type": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "iid", + "description": "The internal ID of the Iteration to look up", + "type": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "includeAncestors", + "description": "Whether to include ancestor iterations. Defaults to true", + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "defaultValue": null + }, + { "name": "after", "description": "Returns the elements in the list that come after the specified cursor.", "type": { @@ -13733,6 +14826,20 @@ "deprecationReason": null }, { + "name": "storageSizeLimit", + "description": "Total storage limit of the root namespace in bytes", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Float", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "subgroupCreationLevel", "description": "The permission level required to create subgroups within the group", "args": [ @@ -13747,6 +14854,20 @@ "deprecationReason": null }, { + "name": "temporaryStorageIncreaseEndsOn", + "description": "Date until the temporary storage increase is active", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "timelogs", "description": "Time logged in issues by group members", "args": [ @@ -13966,6 +15087,24 @@ "defaultValue": null }, { + "name": "scanner", + "description": "Filter vulnerabilities by scanner", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + }, + "defaultValue": null + }, + { "name": "after", "description": "Returns the elements in the list that come after the specified cursor.", "type": { @@ -14096,6 +15235,59 @@ "deprecationReason": null }, { + "name": "vulnerabilityScanners", + "description": "Vulnerability scanners reported on the project vulnerabilties of the group and its subgroups", + "args": [ + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "VulnerabilityScannerConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "webUrl", "description": "Web URL of the group", "args": [ @@ -14493,6 +15685,59 @@ }, "isDeprecated": false, "deprecationReason": null + }, + { + "name": "vulnerabilityScanners", + "description": "Vulnerability scanners reported on the vulnerabilties from projects selected in Instance Security Dashboard", + "args": [ + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "VulnerabilityScannerConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null } ], "inputFields": null, @@ -14859,6 +16104,24 @@ "deprecationReason": null }, { + "name": "id", + "description": "ID of the issue", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "iid", "description": "Internal ID of the issue", "args": [ @@ -15370,6 +16633,24 @@ "description": "The connection type for Issue.", "fields": [ { + "name": "count", + "description": "Total count of collection", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "edges", "description": "A list of edges.", "args": [ @@ -16021,6 +17302,136 @@ }, { "kind": "INPUT_OBJECT", + "name": "IssueSetLockedInput", + "description": "Autogenerated input type of IssueSetLocked", + "fields": null, + "inputFields": [ + { + "name": "projectPath", + "description": "The project the issue to mutate is in", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "iid", + "description": "The iid of the issue to mutate", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "locked", + "description": "Whether or not to lock discussion on the issue", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "IssueSetLockedPayload", + "description": "Autogenerated return type of IssueSetLocked", + "fields": [ + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "errors", + "description": "Errors encountered during execution of the mutation.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "issue", + "description": "The issue after mutation", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "Issue", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "INPUT_OBJECT", "name": "IssueSetWeightInput", "description": "Autogenerated input type of IssueSetWeight", "fields": null, @@ -16349,6 +17760,24 @@ "deprecationReason": null }, { + "name": "iid", + "description": "Internal ID of the iteration", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "startDate", "description": "Timestamp of the iteration start date", "args": [ @@ -16907,6 +18336,24 @@ "defaultValue": null }, { + "name": "usersMapping", + "description": "The mapping of Jira to GitLab users", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "JiraUsersMappingInputType", + "ofType": null + } + } + }, + "defaultValue": null + }, + { "name": "clientMutationId", "description": "A unique identifier for the client performing the mutation.", "type": { @@ -17304,7 +18751,7 @@ }, { "name": "projects", - "description": "List of Jira projects fetched through Jira REST API", + "description": "List of all Jira projects fetched through Jira REST API", "args": [ { "name": "name", @@ -17398,7 +18845,7 @@ "fields": [ { "name": "gitlabId", - "description": "Id of the matched GitLab user", + "description": "ID of the matched GitLab user", "args": [ ], @@ -17411,6 +18858,34 @@ "deprecationReason": null }, { + "name": "gitlabName", + "description": "Name of the matched GitLab user", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "gitlabUsername", + "description": "Username of the matched GitLab user", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "jiraAccountId", "description": "Account id of the Jira user", "args": [ @@ -17469,6 +18944,41 @@ "possibleTypes": null }, { + "kind": "INPUT_OBJECT", + "name": "JiraUsersMappingInputType", + "description": null, + "fields": null, + "inputFields": [ + { + "name": "jiraAccountId", + "description": "Jira account id of the user", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "gitlabId", + "description": "Id of the GitLab user", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { "kind": "OBJECT", "name": "Label", "description": null, @@ -18091,6 +19601,51 @@ "deprecationReason": null }, { + "name": "diffStats", + "description": "Details about which files were changed in this merge request", + "args": [ + { + "name": "path", + "description": "A specific file-path", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "DiffStats", + "ofType": null + } + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "diffStatsSummary", + "description": "Summary of which files were changed in this merge request", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "DiffStatsSummary", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "discussionLocked", "description": "Indicates if comments on the merge request are locked to members only", "args": [ @@ -20409,6 +21964,152 @@ "possibleTypes": null }, { + "kind": "INPUT_OBJECT", + "name": "MergeRequestUpdateInput", + "description": "Autogenerated input type of MergeRequestUpdate", + "fields": null, + "inputFields": [ + { + "name": "projectPath", + "description": "The project the merge request to mutate is in", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "iid", + "description": "The iid of the merge request to mutate", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "title", + "description": "Title of the merge request", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "targetBranch", + "description": "Target branch of the merge request", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "description", + "description": "Description of the merge request (Markdown rendered as HTML for caching)", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "MergeRequestUpdatePayload", + "description": "Autogenerated return type of MergeRequestUpdate", + "fields": [ + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "errors", + "description": "Errors encountered during execution of the mutation.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "mergeRequest", + "description": "The merge request after mutation", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "MergeRequest", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { "kind": "OBJECT", "name": "Metadata", "description": null, @@ -20920,6 +22621,20 @@ "deprecationReason": null }, { + "name": "stats", + "description": "Milestone statistics", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "MilestoneStats", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "subgroupMilestone", "description": "Indicates if milestone is at subgroup level", "args": [ @@ -21135,6 +22850,47 @@ "possibleTypes": null }, { + "kind": "OBJECT", + "name": "MilestoneStats", + "description": "Contains statistics about a milestone", + "fields": [ + { + "name": "closedIssuesCount", + "description": "Number of closed issues associated with the milestone", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "totalIssuesCount", + "description": "Total number of issues associated with the milestone", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { "kind": "ENUM", "name": "MoveType", "description": "The position to which the adjacent object should be moved", @@ -21186,8 +22942,8 @@ "name": "AddAwardEmojiPayload", "ofType": null }, - "isDeprecated": false, - "deprecationReason": null + "isDeprecated": true, + "deprecationReason": "Use awardEmojiAdd. Deprecated in 13.2" }, { "name": "addProjectToSecurityDashboard", @@ -21271,6 +23027,114 @@ "deprecationReason": null }, { + "name": "alertTodoCreate", + "description": null, + "args": [ + { + "name": "input", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "AlertTodoCreateInput", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "AlertTodoCreatePayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "awardEmojiAdd", + "description": null, + "args": [ + { + "name": "input", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "AwardEmojiAddInput", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "AwardEmojiAddPayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "awardEmojiRemove", + "description": null, + "args": [ + { + "name": "input", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "AwardEmojiRemoveInput", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "AwardEmojiRemovePayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "awardEmojiToggle", + "description": null, + "args": [ + { + "name": "input", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "AwardEmojiToggleInput", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "AwardEmojiTogglePayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "boardListUpdateLimitMetrics", "description": null, "args": [ @@ -21595,6 +23459,33 @@ "deprecationReason": null }, { + "name": "dastSiteProfileCreate", + "description": null, + "args": [ + { + "name": "input", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "DastSiteProfileCreateInput", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "DastSiteProfileCreatePayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "deleteAnnotation", "description": null, "args": [ @@ -21946,6 +23837,33 @@ "deprecationReason": null }, { + "name": "issueSetLocked", + "description": null, + "args": [ + { + "name": "input", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "IssueSetLockedInput", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "IssueSetLockedPayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "issueSetWeight", "description": null, "args": [ @@ -22243,6 +24161,33 @@ "deprecationReason": null }, { + "name": "mergeRequestUpdate", + "description": "Update attributes of a merge request", + "args": [ + { + "name": "input", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "MergeRequestUpdateInput", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "MergeRequestUpdatePayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "removeAwardEmoji", "description": null, "args": [ @@ -22266,8 +24211,8 @@ "name": "RemoveAwardEmojiPayload", "ofType": null }, - "isDeprecated": false, - "deprecationReason": null + "isDeprecated": true, + "deprecationReason": "Use awardEmojiRemove. Deprecated in 13.2" }, { "name": "removeProjectFromSecurityDashboard", @@ -22455,8 +24400,8 @@ "name": "ToggleAwardEmojiPayload", "ofType": null }, - "isDeprecated": false, - "deprecationReason": null + "isDeprecated": true, + "deprecationReason": "Use awardEmojiToggle. Deprecated in 13.2" }, { "name": "updateAlertStatus", @@ -22594,6 +24539,33 @@ "deprecationReason": null }, { + "name": "updateIteration", + "description": null, + "args": [ + { + "name": "input", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "UpdateIterationInput", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "UpdateIterationPayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "updateNote", "description": "Updates a Note. If the body of the Note contains only quick actions, the Note will be destroyed during the update, and no Note will be returned", "args": [ @@ -22954,6 +24926,34 @@ "deprecationReason": null }, { + "name": "storageSizeLimit", + "description": "Total storage limit of the root namespace in bytes", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Float", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "temporaryStorageIncreaseEndsOn", + "description": "Date until the temporary storage increase is active", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "visibility", "description": "Visibility of the namespace", "args": [ @@ -23317,6 +25317,20 @@ "deprecationReason": null }, { + "name": "systemNoteIconName", + "description": "Name of the icon corresponding to a system note", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "updatedAt", "description": "Timestamp of the note's last activity", "args": [ @@ -25182,6 +27196,59 @@ "deprecationReason": null }, { + "name": "complianceFrameworks", + "description": "Compliance frameworks associated with the project", + "args": [ + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "ComplianceFrameworkConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "containerExpirationPolicy", "description": "The container expiration policy of the project", "args": [ @@ -25267,7 +27334,7 @@ }, { "name": "search", - "description": "Search query", + "description": "Search query for environment name", "type": { "kind": "SCALAR", "name": "String", @@ -25604,7 +27671,7 @@ }, { "name": "search", - "description": "Search query for finding issues by title or description", + "description": "Search query for issue title or description", "type": { "kind": "SCALAR", "name": "String", @@ -25621,6 +27688,20 @@ "ofType": null }, "defaultValue": "created_desc" + }, + { + "name": "iterationId", + "description": "Iterations applied to the issue", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "defaultValue": null } ], "type": { @@ -25783,7 +27864,7 @@ }, { "name": "search", - "description": "Search query for finding issues by title or description", + "description": "Search query for issue title or description", "type": { "kind": "SCALAR", "name": "String", @@ -25802,6 +27883,20 @@ "defaultValue": "created_desc" }, { + "name": "iterationId", + "description": "Iterations applied to the issue", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "defaultValue": null + }, + { "name": "after", "description": "Returns the elements in the list that come after the specified cursor.", "type": { @@ -26619,7 +28714,7 @@ }, { "name": "release", - "description": "A single release of the project. Available only when feature flag `graphql_release_data` is enabled", + "description": "A single release of the project", "args": [ { "name": "tagName", @@ -26646,7 +28741,7 @@ }, { "name": "releases", - "description": "Releases of the project. Available only when feature flag `graphql_release_data` is enabled", + "description": "Releases of the project", "args": [ { "name": "after", @@ -26793,7 +28888,7 @@ }, { "name": "search", - "description": "Filter requirements by title search", + "description": "Search query for requirement title", "type": { "kind": "SCALAR", "name": "String", @@ -26896,7 +28991,7 @@ }, { "name": "search", - "description": "Filter requirements by title search", + "description": "Search query for requirement title", "type": { "kind": "SCALAR", "name": "String", @@ -26972,6 +29067,34 @@ "deprecationReason": null }, { + "name": "sastCiConfiguration", + "description": "SAST CI configuration for the project", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "SastCiConfiguration", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "securityScanners", + "description": "Information about security analyzers used in the project", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "SecurityScanners", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "sentryDetailedError", "description": "Detailed version of a Sentry error on the project", "args": [ @@ -27405,6 +29528,24 @@ "defaultValue": null }, { + "name": "scanner", + "description": "Filter vulnerabilities by scanner", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + }, + "defaultValue": null + }, + { "name": "after", "description": "Returns the elements in the list that come after the specified cursor.", "type": { @@ -27454,6 +29595,59 @@ "deprecationReason": null }, { + "name": "vulnerabilityScanners", + "description": "Vulnerability scanners reported on the project vulnerabilties", + "args": [ + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "VulnerabilityScannerConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "vulnerabilitySeveritiesCount", "description": "Counts for each severity of vulnerability of the project", "args": [ @@ -28652,6 +30846,47 @@ "possibleTypes": null }, { + "kind": "ENUM", + "name": "ProjectSettingEnum", + "description": "Names of compliance frameworks that can be assigned to a Project", + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": [ + { + "name": "gdpr", + "description": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "hipaa", + "description": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "pci_dss", + "description": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "soc_2", + "description": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "sox", + "description": null, + "isDeprecated": false, + "deprecationReason": null + } + ], + "possibleTypes": null + }, + { "kind": "OBJECT", "name": "ProjectStatistics", "description": null, @@ -28747,6 +30982,20 @@ "deprecationReason": null }, { + "name": "snippetsSize", + "description": "Snippets size of the project", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Float", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "storageSize", "description": "Storage size of the project", "args": [ @@ -29002,7 +31251,7 @@ }, { "name": "search", - "description": "Search criteria", + "description": "Search query for project name, path, or description", "type": { "kind": "SCALAR", "name": "String", @@ -29182,7 +31431,7 @@ }, { "name": "user", - "description": "Find a user on this instance", + "description": "Find a user", "args": [ { "name": "id", @@ -29389,6 +31638,24 @@ "defaultValue": null }, { + "name": "scanner", + "description": "Filter vulnerabilities by scanner", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + }, + "defaultValue": null + }, + { "name": "after", "description": "Returns the elements in the list that come after the specified cursor.", "type": { @@ -29564,7 +31831,7 @@ { "kind": "OBJECT", "name": "Release", - "description": null, + "description": "Represents a release", "fields": [ { "name": "assets", @@ -29704,6 +31971,20 @@ "deprecationReason": null }, { + "name": "links", + "description": "Links of the release", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "ReleaseLinks", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "milestones", "description": "Milestones associated to the release", "args": [ @@ -29791,11 +32072,66 @@ ], "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "tagPath", + "description": "Relative web path to the tag associated with the release", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "ReleaseAssetLink", + "description": "Represents an asset link associated with a release", + "fields": [ + { + "name": "external", + "description": "Indicates the link points to an external resource", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "id", + "description": "ID of the link", + "args": [ + + ], + "type": { "kind": "NON_NULL", "name": null, "ofType": { "kind": "SCALAR", - "name": "String", + "name": "ID", "ofType": null } }, @@ -29803,8 +32139,36 @@ "deprecationReason": null }, { - "name": "tagPath", - "description": "Relative web path to the tag associated with the release", + "name": "linkType", + "description": "Type of the link: `other`, `runbook`, `image`, `package`; defaults to `other`", + "args": [ + + ], + "type": { + "kind": "ENUM", + "name": "ReleaseAssetLinkType", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "name", + "description": "Name of the link", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "url", + "description": "URL of the link", "args": [ ], @@ -29826,11 +32190,158 @@ }, { "kind": "OBJECT", + "name": "ReleaseAssetLinkConnection", + "description": "The connection type for ReleaseAssetLink.", + "fields": [ + { + "name": "edges", + "description": "A list of edges.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "ReleaseAssetLinkEdge", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "nodes", + "description": "A list of nodes.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "ReleaseAssetLink", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "pageInfo", + "description": "Information to aid in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "PageInfo", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "ReleaseAssetLinkEdge", + "description": "An edge in a connection.", + "fields": [ + { + "name": "cursor", + "description": "A cursor for use in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "node", + "description": "The item at the end of the edge.", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "ReleaseAssetLink", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "ENUM", + "name": "ReleaseAssetLinkType", + "description": "Type of the link: `other`, `runbook`, `image`, `package`; defaults to `other`", + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": [ + { + "name": "OTHER", + "description": "Other link type", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "RUNBOOK", + "description": "Runbook link type", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "PACKAGE", + "description": "Package link type", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "IMAGE", + "description": "Image link type", + "isDeprecated": false, + "deprecationReason": null + } + ], + "possibleTypes": null + }, + { + "kind": "OBJECT", "name": "ReleaseAssets", - "description": null, + "description": "A container for all assets associated with a release", "fields": [ { - "name": "assetsCount", + "name": "count", "description": "Number of assets of the release", "args": [ @@ -29890,7 +32401,7 @@ ], "type": { "kind": "OBJECT", - "name": "ReleaseLinkConnection", + "name": "ReleaseAssetLinkConnection", "ofType": null }, "isDeprecated": false, @@ -30256,58 +32767,26 @@ }, { "kind": "OBJECT", - "name": "ReleaseLink", + "name": "ReleaseLinks", "description": null, "fields": [ { - "name": "external", - "description": "Indicates the link points to an external resource", + "name": "editUrl", + "description": "HTTP URL of the release's edit page", "args": [ ], "type": { "kind": "SCALAR", - "name": "Boolean", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "id", - "description": "ID of the link", - "args": [ - - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "ID", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "linkType", - "description": "Type of the link: `other`, `runbook`, `image`, `package`; defaults to `other`", - "args": [ - - ], - "type": { - "kind": "ENUM", - "name": "ReleaseLinkType", + "name": "String", "ofType": null }, "isDeprecated": false, "deprecationReason": null }, { - "name": "name", - "description": "Name of the link", + "name": "issuesUrl", + "description": "HTTP URL of the issues page filtered by this release", "args": [ ], @@ -30320,8 +32799,8 @@ "deprecationReason": null }, { - "name": "url", - "description": "URL of the link", + "name": "mergeRequestsUrl", + "description": "HTTP URL of the merge request page filtered by this release", "args": [ ], @@ -30332,114 +32811,16 @@ }, "isDeprecated": false, "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [ - - ], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "ReleaseLinkConnection", - "description": "The connection type for ReleaseLink.", - "fields": [ - { - "name": "edges", - "description": "A list of edges.", - "args": [ - - ], - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "ReleaseLinkEdge", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null }, { - "name": "nodes", - "description": "A list of nodes.", + "name": "selfUrl", + "description": "HTTP URL of the release", "args": [ ], "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "ReleaseLink", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "pageInfo", - "description": "Information to aid in pagination.", - "args": [ - - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "PageInfo", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [ - - ], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "ReleaseLinkEdge", - "description": "An edge in a connection.", - "fields": [ - { - "name": "cursor", - "description": "A cursor for use in pagination.", - "args": [ - - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "node", - "description": "The item at the end of the edge.", - "args": [ - - ], - "type": { - "kind": "OBJECT", - "name": "ReleaseLink", + "kind": "SCALAR", + "name": "String", "ofType": null }, "isDeprecated": false, @@ -30454,44 +32835,9 @@ "possibleTypes": null }, { - "kind": "ENUM", - "name": "ReleaseLinkType", - "description": "Type of the link: `other`, `runbook`, `image`, `package`; defaults to `other`", - "fields": null, - "inputFields": null, - "interfaces": null, - "enumValues": [ - { - "name": "OTHER", - "description": "Other link type", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "RUNBOOK", - "description": "Runbook link type", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "PACKAGE", - "description": "Package link type", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "IMAGE", - "description": "Image link type", - "isDeprecated": false, - "deprecationReason": null - } - ], - "possibleTypes": null - }, - { "kind": "OBJECT", "name": "ReleaseSource", - "description": null, + "description": "Represents the source code attached to a release in a particular format", "fields": [ { "name": "format", @@ -31628,6 +33974,24 @@ "deprecationReason": null }, { + "name": "snippetsSize", + "description": "The snippets size in bytes", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Float", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "storageSize", "description": "The total storage in bytes", "args": [ @@ -31817,6 +34181,927 @@ }, { "kind": "OBJECT", + "name": "SastCiConfiguration", + "description": "Represents a CI configuration of SAST", + "fields": [ + { + "name": "analyzers", + "description": "List of analyzers entities attached to SAST configuration.", + "args": [ + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "SastCiConfigurationAnalyzersEntityConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "global", + "description": "List of global entities related to SAST configuration.", + "args": [ + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "SastCiConfigurationEntityConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "pipeline", + "description": "List of pipeline entities related to SAST configuration.", + "args": [ + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "SastCiConfigurationEntityConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "SastCiConfigurationAnalyzersEntity", + "description": "Represents an analyzer entity in SAST CI configuration", + "fields": [ + { + "name": "description", + "description": "Analyzer description that is displayed on the form.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "enabled", + "description": "Indicates whether an analyzer is enabled.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "label", + "description": "Analyzer label used in the config UI.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "name", + "description": "Name of the analyzer.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "SastCiConfigurationAnalyzersEntityConnection", + "description": "The connection type for SastCiConfigurationAnalyzersEntity.", + "fields": [ + { + "name": "edges", + "description": "A list of edges.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "SastCiConfigurationAnalyzersEntityEdge", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "nodes", + "description": "A list of nodes.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "SastCiConfigurationAnalyzersEntity", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "pageInfo", + "description": "Information to aid in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "PageInfo", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "SastCiConfigurationAnalyzersEntityEdge", + "description": "An edge in a connection.", + "fields": [ + { + "name": "cursor", + "description": "A cursor for use in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "node", + "description": "The item at the end of the edge.", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "SastCiConfigurationAnalyzersEntity", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "SastCiConfigurationEntity", + "description": "Represents an entity in SAST CI configuration", + "fields": [ + { + "name": "defaultValue", + "description": "Default value that is used if value is empty.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "description", + "description": "Entity description that is displayed on the form.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "field", + "description": "CI keyword of entity.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "label", + "description": "Label for entity used in the form.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "options", + "description": "Different possible values of the field.", + "args": [ + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "SastCiConfigurationOptionsEntityConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "type", + "description": "Type of the field value.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "value", + "description": "Current value of the entity.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "SastCiConfigurationEntityConnection", + "description": "The connection type for SastCiConfigurationEntity.", + "fields": [ + { + "name": "edges", + "description": "A list of edges.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "SastCiConfigurationEntityEdge", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "nodes", + "description": "A list of nodes.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "SastCiConfigurationEntity", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "pageInfo", + "description": "Information to aid in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "PageInfo", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "SastCiConfigurationEntityEdge", + "description": "An edge in a connection.", + "fields": [ + { + "name": "cursor", + "description": "A cursor for use in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "node", + "description": "The item at the end of the edge.", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "SastCiConfigurationEntity", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "SastCiConfigurationOptionsEntity", + "description": "Represents an entity for options in SAST CI configuration", + "fields": [ + { + "name": "label", + "description": "Label of option entity.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "value", + "description": "Value of option entity.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "SastCiConfigurationOptionsEntityConnection", + "description": "The connection type for SastCiConfigurationOptionsEntity.", + "fields": [ + { + "name": "edges", + "description": "A list of edges.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "SastCiConfigurationOptionsEntityEdge", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "nodes", + "description": "A list of nodes.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "SastCiConfigurationOptionsEntity", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "pageInfo", + "description": "Information to aid in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "PageInfo", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "SastCiConfigurationOptionsEntityEdge", + "description": "An edge in a connection.", + "fields": [ + { + "name": "cursor", + "description": "A cursor for use in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "node", + "description": "The item at the end of the edge.", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "SastCiConfigurationOptionsEntity", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "ScannedResource", + "description": "Represents a resource scanned by a security scan", + "fields": [ + { + "name": "requestMethod", + "description": "The HTTP request method used to access the URL", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "url", + "description": "The URL scanned by the scanner", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "ScannedResourceConnection", + "description": "The connection type for ScannedResource.", + "fields": [ + { + "name": "edges", + "description": "A list of edges.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "ScannedResourceEdge", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "nodes", + "description": "A list of nodes.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "ScannedResource", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "pageInfo", + "description": "Information to aid in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "PageInfo", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "ScannedResourceEdge", + "description": "An edge in a connection.", + "fields": [ + { + "name": "cursor", + "description": "A cursor for use in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "node", + "description": "The item at the end of the edge.", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "ScannedResource", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", "name": "SecurityReportSummary", "description": "Represents summary of a security report", "fields": [ @@ -31835,6 +35120,20 @@ "deprecationReason": null }, { + "name": "coverageFuzzing", + "description": "Aggregated counts for the coverage_fuzzing scan", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "SecurityReportSummarySection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "dast", "description": "Aggregated counts for the dast scan", "args": [ @@ -31904,6 +35203,59 @@ "description": "Represents a section of a summary of a security report", "fields": [ { + "name": "scannedResources", + "description": "A list of the first 20 scanned resources", + "args": [ + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "ScannedResourceConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "scannedResourcesCount", "description": "Total number of scanned resources", "args": [ @@ -31918,6 +35270,20 @@ "deprecationReason": null }, { + "name": "scannedResourcesCsvPath", + "description": "Path to download all the scanned resources in CSV format", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "vulnerabilitiesCount", "description": "Total number of vulnerabilities", "args": [ @@ -31940,6 +35306,126 @@ "possibleTypes": null }, { + "kind": "ENUM", + "name": "SecurityScannerType", + "description": "The type of the security scanner.", + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": [ + { + "name": "SAST", + "description": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "DAST", + "description": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "DEPENDENCY_SCANNING", + "description": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "CONTAINER_SCANNING", + "description": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "SECRET_DETECTION", + "description": null, + "isDeprecated": false, + "deprecationReason": null + } + ], + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "SecurityScanners", + "description": "Represents a list of security scanners", + "fields": [ + { + "name": "available", + "description": "List of analyzers which are available for the project.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "ENUM", + "name": "SecurityScannerType", + "ofType": null + } + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "enabled", + "description": "List of analyzers which are enabled for the project.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "ENUM", + "name": "SecurityScannerType", + "ofType": null + } + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "pipelineRun", + "description": "List of analyzers which ran successfully in the latest pipeline.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "ENUM", + "name": "SecurityScannerType", + "ofType": null + } + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { "kind": "OBJECT", "name": "SentryDetailedError", "description": "A Sentry error.", @@ -32032,6 +35518,20 @@ }, { "name": "firstReleaseShortVersion", + "description": "Release short version the error was first seen", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "firstReleaseVersion", "description": "Release version the error was first seen", "args": [ @@ -32164,6 +35664,20 @@ }, { "name": "lastReleaseShortVersion", + "description": "Release short version the error was last seen", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "lastReleaseVersion", "description": "Release version the error was last seen", "args": [ @@ -32824,7 +36338,7 @@ }, { "name": "searchTerm", - "description": "Search term for the Sentry error.", + "description": "Search query for the Sentry error details", "type": { "kind": "SCALAR", "name": "String", @@ -33538,6 +37052,12 @@ "deprecationReason": null }, { + "name": "CONFLUENCE_SERVICE", + "description": null, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "CUSTOM_ISSUE_TRACKER_SERVICE", "description": null, "isDeprecated": false, @@ -34568,6 +38088,100 @@ "possibleTypes": null }, { + "kind": "ENUM", + "name": "SnippetFileInputActionEnum", + "description": "Type of a snippet file input action", + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": [ + { + "name": "create", + "description": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "update", + "description": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "delete", + "description": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "move", + "description": null, + "isDeprecated": false, + "deprecationReason": null + } + ], + "possibleTypes": null + }, + { + "kind": "INPUT_OBJECT", + "name": "SnippetFileInputType", + "description": "Represents an action to perform over a snippet file", + "fields": null, + "inputFields": [ + { + "name": "action", + "description": "Type of input action", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "ENUM", + "name": "SnippetFileInputActionEnum", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "previousPath", + "description": "Previous path of the snippet file", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "filePath", + "description": "Path of the snippet file", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "content", + "description": "Snippet file content", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { "kind": "OBJECT", "name": "SnippetPermissions", "description": null, @@ -35253,6 +38867,12 @@ "description": null, "isDeprecated": false, "deprecationReason": null + }, + { + "name": "FAILED", + "description": null, + "isDeprecated": false, + "deprecationReason": null } ], "possibleTypes": null @@ -36036,8 +39656,34 @@ "deprecationReason": null }, { + "name": "todos", + "description": "Updated todos", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "Todo", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "updatedIds", - "description": "The ids of the updated todo items", + "description": "The ids of the updated todo items. Deprecated in 13.2: Use todos", "args": [ ], @@ -36058,8 +39704,8 @@ } } }, - "isDeprecated": false, - "deprecationReason": null + "isDeprecated": true, + "deprecationReason": "Use todos. Deprecated in 13.2" } ], "inputFields": null, @@ -36196,6 +39842,12 @@ "deprecationReason": null }, { + "name": "ALERT", + "description": "An Alert", + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "EPIC", "description": "An Epic", "isDeprecated": false, @@ -36271,8 +39923,34 @@ "deprecationReason": null }, { + "name": "todos", + "description": "Updated todos", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "Todo", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "updatedIds", - "description": "Ids of the updated todos", + "description": "Ids of the updated todos. Deprecated in 13.2: Use todos", "args": [ ], @@ -36293,8 +39971,8 @@ } } }, - "isDeprecated": false, - "deprecationReason": null + "isDeprecated": true, + "deprecationReason": "Use todos. Deprecated in 13.2" } ], "inputFields": null, @@ -36911,6 +40589,16 @@ "possibleTypes": null }, { + "kind": "SCALAR", + "name": "UntrustedRegexp", + "description": "A regexp containing patterns sourced from user input", + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { "kind": "INPUT_OBJECT", "name": "UpdateAlertStatusInput", "description": "Autogenerated input type of UpdateAlertStatus", @@ -37045,6 +40733,20 @@ }, "isDeprecated": false, "deprecationReason": null + }, + { + "name": "todo", + "description": "The todo after mutation", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "Todo", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null } ], "inputFields": null, @@ -37115,6 +40817,26 @@ "defaultValue": null }, { + "name": "nameRegex", + "description": "Tags with names matching this regex pattern will expire", + "type": { + "kind": "SCALAR", + "name": "UntrustedRegexp", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "nameRegexKeep", + "description": "Tags with names matching this regex pattern will be preserved", + "type": { + "kind": "SCALAR", + "name": "UntrustedRegexp", + "ofType": null + }, + "defaultValue": null + }, + { "name": "clientMutationId", "description": "A unique identifier for the client performing the mutation.", "type": { @@ -37769,6 +41491,162 @@ }, { "kind": "INPUT_OBJECT", + "name": "UpdateIterationInput", + "description": "Autogenerated input type of UpdateIteration", + "fields": null, + "inputFields": [ + { + "name": "groupPath", + "description": "The group of the iteration", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "id", + "description": "The id of the iteration", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "title", + "description": "The title of the iteration", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "description", + "description": "The description of the iteration", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "startDate", + "description": "The start date of the iteration", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "dueDate", + "description": "The end date of the iteration", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "UpdateIterationPayload", + "description": "Autogenerated return type of UpdateIteration", + "fields": [ + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "errors", + "description": "Errors encountered during execution of the mutation.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "iteration", + "description": "The updated iteration", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "Iteration", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "INPUT_OBJECT", "name": "UpdateNoteInput", "description": "Autogenerated input type of UpdateNote", "fields": null, @@ -38090,6 +41968,24 @@ "defaultValue": null }, { + "name": "files", + "description": "The snippet files to update", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "SnippetFileInputType", + "ofType": null + } + } + }, + "defaultValue": null + }, + { "name": "clientMutationId", "description": "A unique identifier for the client performing the mutation.", "type": { @@ -39423,6 +43319,32 @@ "deprecationReason": null }, { + "name": "identifiers", + "description": "Identifiers of the vulnerability.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "VulnerabilityIdentifier", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "issueLinks", "description": "List of issue links related to the vulnerability", "args": [ @@ -39504,6 +43426,20 @@ "deprecationReason": null }, { + "name": "primaryIdentifier", + "description": "Primary identifier of the vulnerability.", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "VulnerabilityIdentifier", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "project", "description": "The project on which the vulnerability was found", "args": [ @@ -39519,7 +43455,7 @@ }, { "name": "reportType", - "description": "Type of the security report that found the vulnerability (SAST, DEPENDENCY_SCANNING, CONTAINER_SCANNING, DAST, SECRET_DETECTION)", + "description": "Type of the security report that found the vulnerability (SAST, DEPENDENCY_SCANNING, CONTAINER_SCANNING, DAST, SECRET_DETECTION, COVERAGE_FUZZING)", "args": [ ], @@ -39532,6 +43468,20 @@ "deprecationReason": null }, { + "name": "scanner", + "description": "Scanner metadata for the vulnerability.", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "VulnerabilityScanner", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "severity", "description": "Severity of the vulnerability (INFO, UNKNOWN, LOW, MEDIUM, HIGH, CRITICAL)", "args": [ @@ -39745,6 +43695,75 @@ }, { "kind": "OBJECT", + "name": "VulnerabilityIdentifier", + "description": "Represents a vulnerability identifier.", + "fields": [ + { + "name": "externalId", + "description": "External ID of the vulnerability identifier", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "externalType", + "description": "External type of the vulnerability identifier", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "name", + "description": "Name of the vulnerability identifier", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "url", + "description": "URL of the vulnerability identifier", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", "name": "VulnerabilityIssueLink", "description": "Represents an issue link of a vulnerability.", "fields": [ @@ -40506,8 +44525,195 @@ "description": null, "isDeprecated": false, "deprecationReason": null + }, + { + "name": "COVERAGE_FUZZING", + "description": null, + "isDeprecated": false, + "deprecationReason": null + } + ], + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "VulnerabilityScanner", + "description": "Represents a vulnerability scanner.", + "fields": [ + { + "name": "externalId", + "description": "External ID of the vulnerability scanner", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "name", + "description": "Name of the vulnerability scanner", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "reportType", + "description": "Type of the vulnerability report", + "args": [ + + ], + "type": { + "kind": "ENUM", + "name": "VulnerabilityReportType", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "vendor", + "description": "Vendor of the vulnerability scanner", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "VulnerabilityScannerConnection", + "description": "The connection type for VulnerabilityScanner.", + "fields": [ + { + "name": "edges", + "description": "A list of edges.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "VulnerabilityScannerEdge", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "nodes", + "description": "A list of nodes.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "VulnerabilityScanner", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "pageInfo", + "description": "Information to aid in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "PageInfo", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "VulnerabilityScannerEdge", + "description": "An edge in a connection.", + "fields": [ + { + "name": "cursor", + "description": "A cursor for use in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "node", + "description": "The item at the end of the edge.", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "VulnerabilityScanner", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null } ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, "possibleTypes": null }, { diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md index befb57c1cba..6df6632f3bd 100644 --- a/doc/api/graphql/reference/index.md +++ b/doc/api/graphql/reference/index.md @@ -69,6 +69,7 @@ Describes an alert from the project's Alert Management | `hosts` | String! => Array | List of hosts the alert came from | | `iid` | ID! | Internal ID of the alert | | `issueIid` | ID | Internal ID of the GitLab issue attached to the alert | +| `metricsDashboardUrl` | String | URL for metrics embed for the alert | | `monitoringTool` | String | Monitoring tool the alert came from | | `service` | String | Service the alert came from | | `severity` | AlertManagementSeverity | Severity of the alert | @@ -100,6 +101,19 @@ Autogenerated return type of AlertSetAssignees | `clientMutationId` | String | A unique identifier for the client performing the mutation. | | `errors` | String! => Array | Errors encountered during execution of the mutation. | | `issue` | Issue | The issue created after mutation | +| `todo` | Todo | The todo after mutation | + +## AlertTodoCreatePayload + +Autogenerated return type of AlertTodoCreate + +| Name | Type | Description | +| --- | ---- | ---------- | +| `alert` | AlertManagementAlert | The alert after mutation | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | +| `issue` | Issue | The issue created after mutation | +| `todo` | Todo | The todo after mutation | ## AwardEmoji @@ -114,6 +128,37 @@ An emoji awarded by a user. | `unicodeVersion` | String! | The unicode version for this emoji | | `user` | User! | The user who awarded the emoji | +## AwardEmojiAddPayload + +Autogenerated return type of AwardEmojiAdd + +| Name | Type | Description | +| --- | ---- | ---------- | +| `awardEmoji` | AwardEmoji | The award emoji after mutation | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | + +## AwardEmojiRemovePayload + +Autogenerated return type of AwardEmojiRemove + +| Name | Type | Description | +| --- | ---- | ---------- | +| `awardEmoji` | AwardEmoji | The award emoji after mutation | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | + +## AwardEmojiTogglePayload + +Autogenerated return type of AwardEmojiToggle + +| Name | Type | Description | +| --- | ---- | ---------- | +| `awardEmoji` | AwardEmoji | The award emoji after mutation | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | +| `toggledOn` | Boolean! | Indicates the status of the emoji. True if the toggle awarded the emoji, and false if the toggle removed the emoji. | + ## BaseService | Name | Type | Description | @@ -128,6 +173,7 @@ An emoji awarded by a user. | `flatPath` | String! | Flat path of the entry | | `id` | ID! | ID of the entry | | `lfsOid` | String | LFS ID of the blob | +| `mode` | String | Blob mode in numeric format | | `name` | String! | Name of the entry | | `path` | String! | Path of the entry | | `sha` | String! | Last commit sha for the entry | @@ -207,6 +253,14 @@ Autogenerated return type of CommitCreate | `commit` | Commit | The commit after mutation | | `errors` | String! => Array | Errors encountered during execution of the mutation. | +## ComplianceFramework + +Represents a ComplianceFramework associated with a Project + +| Name | Type | Description | +| --- | ---- | ---------- | +| `name` | ProjectSettingEnum! | Name of the compliance framework | + ## ContainerExpirationPolicy A tag expiration policy designed to keep only the images that matter most @@ -217,8 +271,8 @@ A tag expiration policy designed to keep only the images that matter most | `createdAt` | Time! | Timestamp of when the container expiration policy was created | | `enabled` | Boolean! | Indicates whether this container expiration policy is enabled | | `keepN` | ContainerExpirationPolicyKeepEnum | Number of tags to retain | -| `nameRegex` | String | Tags with names matching this regex pattern will expire | -| `nameRegexKeep` | String | Tags with names matching this regex pattern will be preserved | +| `nameRegex` | UntrustedRegexp | Tags with names matching this regex pattern will expire | +| `nameRegexKeep` | UntrustedRegexp | Tags with names matching this regex pattern will be preserved | | `nextRunAt` | Time | Next time that this container expiration policy will get executed | | `olderThan` | ContainerExpirationPolicyOlderThanEnum | Tags older that this will expire | | `updatedAt` | Time! | Timestamp of when the container expiration policy was updated | @@ -233,6 +287,7 @@ Autogenerated return type of CreateAlertIssue | `clientMutationId` | String | A unique identifier for the client performing the mutation. | | `errors` | String! => Array | Errors encountered during execution of the mutation. | | `issue` | Issue | The issue created after mutation | +| `todo` | Todo | The todo after mutation | ## CreateAnnotationPayload @@ -324,6 +379,16 @@ Autogenerated return type of CreateSnippet | `errors` | String! => Array | Errors encountered during execution of the mutation. | | `snippet` | Snippet | The snippet after mutation | +## DastSiteProfileCreatePayload + +Autogenerated return type of DastSiteProfileCreate + +| Name | Type | Description | +| --- | ---- | ---------- | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | +| `id` | ID | ID of the site profile. | + ## DeleteAnnotationPayload Autogenerated return type of DeleteAnnotation @@ -486,6 +551,27 @@ Autogenerated return type of DestroySnippet | `headSha` | String! | SHA of the HEAD at the time the comment was made | | `startSha` | String! | SHA of the branch being compared against | +## DiffStats + +Changes to a single file + +| Name | Type | Description | +| --- | ---- | ---------- | +| `additions` | Int! | Number of lines added to this file | +| `deletions` | Int! | Number of lines deleted from this file | +| `path` | String! | File path, relative to repository root | + +## DiffStatsSummary + +Aggregated summary of changes + +| Name | Type | Description | +| --- | ---- | ---------- | +| `additions` | Int! | Number of lines added | +| `changes` | Int! | Number of lines changed | +| `deletions` | Int! | Number of lines deleted | +| `fileCount` | Int! | Number of files changed | + ## Discussion | Name | Type | Description | @@ -745,7 +831,9 @@ Autogenerated return type of EpicTreeReorder | `requireTwoFactorAuthentication` | Boolean | Indicates if all users in this group are required to set up two-factor authentication | | `rootStorageStatistics` | RootStorageStatistics | Aggregated storage statistics of the namespace. Only available for root namespaces | | `shareWithGroupLock` | Boolean | Indicates if sharing a project with another group within this group is prevented | +| `storageSizeLimit` | Float | Total storage limit of the root namespace in bytes | | `subgroupCreationLevel` | String | The permission level required to create subgroups within the group | +| `temporaryStorageIncreaseEndsOn` | Time | Date until the temporary storage increase is active | | `twoFactorGracePeriod` | Int | Time before two-factor authentication is enforced | | `userPermissions` | GroupPermissions! | Permissions for the current user on the resource | | `visibility` | String | Visibility of the namespace | @@ -788,6 +876,7 @@ Represents a Group Member | `dueDate` | Time | Due date of the issue | | `epic` | Epic | Epic to which this issue belongs | | `healthStatus` | HealthStatus | Current health status. Returns null if `save_issuable_health_status` feature flag is disabled. | +| `id` | ID! | ID of the issue | | `iid` | ID! | Internal ID of the issue | | `iteration` | Iteration | Iteration of the issue | | `milestone` | Milestone | Milestone of the issue | @@ -853,6 +942,16 @@ Autogenerated return type of IssueSetIteration | `errors` | String! => Array | Errors encountered during execution of the mutation. | | `issue` | Issue | The issue after mutation | +## IssueSetLockedPayload + +Autogenerated return type of IssueSetLocked + +| Name | Type | Description | +| --- | ---- | ---------- | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | +| `issue` | Issue | The issue after mutation | + ## IssueSetWeightPayload Autogenerated return type of IssueSetWeight @@ -873,6 +972,7 @@ Represents an iteration object. | `description` | String | Description of the iteration | | `dueDate` | Time | Timestamp of the iteration due date | | `id` | ID! | ID of the iteration | +| `iid` | ID! | Internal ID of the iteration | | `startDate` | Time | Timestamp of the iteration start date | | `state` | IterationState! | State of the iteration | | `title` | String! | Title of the iteration | @@ -925,14 +1025,16 @@ Autogenerated return type of JiraImportUsers | Name | Type | Description | | --- | ---- | ---------- | | `active` | Boolean | Indicates if the service is active | -| `projects` | JiraProjectConnection | List of Jira projects fetched through Jira REST API | +| `projects` | JiraProjectConnection | List of all Jira projects fetched through Jira REST API | | `type` | String | Class name of the service | ## JiraUser | Name | Type | Description | | --- | ---- | ---------- | -| `gitlabId` | Int | Id of the matched GitLab user | +| `gitlabId` | Int | ID of the matched GitLab user | +| `gitlabName` | String | Name of the matched GitLab user | +| `gitlabUsername` | String | Username of the matched GitLab user | | `jiraAccountId` | String! | Account id of the Jira user | | `jiraDisplayName` | String! | Display name of the Jira user | | `jiraEmail` | String | Email of the Jira user, returned only for users with public emails | @@ -970,6 +1072,8 @@ Autogenerated return type of MarkAsSpamSnippet | `descriptionHtml` | String | The GitLab Flavored Markdown rendering of `description` | | `diffHeadSha` | String | Diff head SHA of the merge request | | `diffRefs` | DiffRefs | References of the base SHA, the head SHA, and the start SHA for this merge request | +| `diffStats` | DiffStats! => Array | Details about which files were changed in this merge request | +| `diffStatsSummary` | DiffStatsSummary | Summary of which files were changed in this merge request | | `discussionLocked` | Boolean! | Indicates if comments on the merge request are locked to members only | | `downvotes` | Int! | Number of downvotes for the merge request | | `forceRemoveSourceBranch` | Boolean | Indicates if the project settings will lead to source branch deletion after merge | @@ -1100,6 +1204,16 @@ Autogenerated return type of MergeRequestSetWip | `errors` | String! => Array | Errors encountered during execution of the mutation. | | `mergeRequest` | MergeRequest | The merge request after mutation | +## MergeRequestUpdatePayload + +Autogenerated return type of MergeRequestUpdate + +| Name | Type | Description | +| --- | ---- | ---------- | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | +| `mergeRequest` | MergeRequest | The merge request after mutation | + ## Metadata | Name | Type | Description | @@ -1138,11 +1252,21 @@ Represents a milestone. | `projectMilestone` | Boolean! | Indicates if milestone is at project level | | `startDate` | Time | Timestamp of the milestone start date | | `state` | MilestoneStateEnum! | State of the milestone | +| `stats` | MilestoneStats | Milestone statistics | | `subgroupMilestone` | Boolean! | Indicates if milestone is at subgroup level | | `title` | String! | Title of the milestone | | `updatedAt` | Time! | Timestamp of last milestone update | | `webPath` | String! | Web path of the milestone | +## MilestoneStats + +Contains statistics about a milestone + +| Name | Type | Description | +| --- | ---- | ---------- | +| `closedIssuesCount` | Int | Number of closed issues associated with the milestone | +| `totalIssuesCount` | Int | Total number of issues associated with the milestone | + ## Namespace | Name | Type | Description | @@ -1157,6 +1281,8 @@ Represents a milestone. | `path` | String! | Path of the namespace | | `requestAccessEnabled` | Boolean | Indicates if users can request access to namespace | | `rootStorageStatistics` | RootStorageStatistics | Aggregated storage statistics of the namespace. Only available for root namespaces | +| `storageSizeLimit` | Float | Total storage limit of the root namespace in bytes | +| `temporaryStorageIncreaseEndsOn` | Time | Date until the temporary storage increase is active | | `visibility` | String | Visibility of the namespace | ## Note @@ -1177,6 +1303,7 @@ Represents a milestone. | `resolvedAt` | Time | Timestamp of when the object was resolved | | `resolvedBy` | User | User who resolved the object | | `system` | Boolean! | Indicates whether this note was created by the system or by a user | +| `systemNoteIconName` | String | Name of the icon corresponding to a system note | | `updatedAt` | Time! | Timestamp of the note's last activity | | `userPermissions` | NotePermissions! | Permissions for the current user on the resource | @@ -1300,12 +1427,14 @@ Information about pagination in a connection. | `pipeline` | Pipeline | Build pipeline of the project | | `printingMergeRequestLinkEnabled` | Boolean | Indicates if a link to create or view a merge request should display after a push to Git repositories of the project from the command line | | `publicJobs` | Boolean | Indicates if there is public access to pipelines and job details of the project, including output logs and artifacts | -| `release` | Release | A single release of the project. Available only when feature flag `graphql_release_data` is enabled | +| `release` | Release | A single release of the project | | `removeSourceBranchAfterMerge` | Boolean | Indicates if `Delete source branch` option should be enabled by default for all new merge requests of the project | | `repository` | Repository | Git repository of the project | | `requestAccessEnabled` | Boolean | Indicates if users can request member access to the project | | `requirement` | Requirement | Find a single requirement. Available only when feature flag `requirements_management` is enabled. | | `requirementStatesCount` | RequirementStatesCount | Number of requirements for the project by their state | +| `sastCiConfiguration` | SastCiConfiguration | SAST CI configuration for the project | +| `securityScanners` | SecurityScanners | Information about security analyzers used in the project | | `sentryDetailedError` | SentryDetailedError | Detailed version of a Sentry error on the project | | `sentryErrors` | SentryErrorCollection | Paginated collection of Sentry errors on the project | | `serviceDeskAddress` | String | E-mail address of the service desk. | @@ -1395,11 +1524,14 @@ Represents a Project Member | `lfsObjectsSize` | Float! | Large File Storage (LFS) object size of the project | | `packagesSize` | Float! | Packages size of the project | | `repositorySize` | Float! | Repository size of the project | +| `snippetsSize` | Float | Snippets size of the project | | `storageSize` | Float! | Storage size of the project | | `wikiSize` | Float | Wiki size of the project | ## Release +Represents a release + | Name | Type | Description | | --- | ---- | ---------- | | `assets` | ReleaseAssets | Assets of the release | @@ -1408,16 +1540,31 @@ Represents a Project Member | `createdAt` | Time | Timestamp of when the release was created | | `description` | String | Description (also known as "release notes") of the release | | `descriptionHtml` | String | The GitLab Flavored Markdown rendering of `description` | +| `links` | ReleaseLinks | Links of the release | | `name` | String | Name of the release | | `releasedAt` | Time | Timestamp of when the release was released | -| `tagName` | String! | Name of the tag associated with the release | +| `tagName` | String | Name of the tag associated with the release | | `tagPath` | String | Relative web path to the tag associated with the release | +## ReleaseAssetLink + +Represents an asset link associated with a release + +| Name | Type | Description | +| --- | ---- | ---------- | +| `external` | Boolean | Indicates the link points to an external resource | +| `id` | ID! | ID of the link | +| `linkType` | ReleaseAssetLinkType | Type of the link: `other`, `runbook`, `image`, `package`; defaults to `other` | +| `name` | String | Name of the link | +| `url` | String | URL of the link | + ## ReleaseAssets +A container for all assets associated with a release + | Name | Type | Description | | --- | ---- | ---------- | -| `assetsCount` | Int | Number of assets of the release | +| `count` | Int | Number of assets of the release | ## ReleaseEvidence @@ -1430,18 +1577,19 @@ Evidence for a release | `id` | ID! | ID of the evidence | | `sha` | String | SHA1 ID of the evidence hash | -## ReleaseLink +## ReleaseLinks | Name | Type | Description | | --- | ---- | ---------- | -| `external` | Boolean | Indicates the link points to an external resource | -| `id` | ID! | ID of the link | -| `linkType` | ReleaseLinkType | Type of the link: `other`, `runbook`, `image`, `package`; defaults to `other` | -| `name` | String | Name of the link | -| `url` | String | URL of the link | +| `editUrl` | String | HTTP URL of the release's edit page | +| `issuesUrl` | String | HTTP URL of the issues page filtered by this release | +| `mergeRequestsUrl` | String | HTTP URL of the merge request page filtered by this release | +| `selfUrl` | String | HTTP URL of the release | ## ReleaseSource +Represents the source code attached to a release in a particular format + | Name | Type | Description | | --- | ---- | ---------- | | `format` | String | Format of the source | @@ -1520,6 +1668,7 @@ Counts of requirements by their state. | `lfsObjectsSize` | Float! | The LFS objects size in bytes | | `packagesSize` | Float! | The packages size in bytes | | `repositorySize` | Float! | The Git repository size in bytes | +| `snippetsSize` | Float! | The snippets size in bytes | | `storageSize` | Float! | The total storage in bytes | | `wikiSize` | Float! | The wiki size in bytes | @@ -1533,6 +1682,48 @@ Autogenerated return type of RunDASTScan | `errors` | String! => Array | Errors encountered during execution of the mutation. | | `pipelineUrl` | String | URL of the pipeline that was created. | +## SastCiConfigurationAnalyzersEntity + +Represents an analyzer entity in SAST CI configuration + +| Name | Type | Description | +| --- | ---- | ---------- | +| `description` | String | Analyzer description that is displayed on the form. | +| `enabled` | Boolean | Indicates whether an analyzer is enabled. | +| `label` | String | Analyzer label used in the config UI. | +| `name` | String | Name of the analyzer. | + +## SastCiConfigurationEntity + +Represents an entity in SAST CI configuration + +| Name | Type | Description | +| --- | ---- | ---------- | +| `defaultValue` | String | Default value that is used if value is empty. | +| `description` | String | Entity description that is displayed on the form. | +| `field` | String | CI keyword of entity. | +| `label` | String | Label for entity used in the form. | +| `type` | String | Type of the field value. | +| `value` | String | Current value of the entity. | + +## SastCiConfigurationOptionsEntity + +Represents an entity for options in SAST CI configuration + +| Name | Type | Description | +| --- | ---- | ---------- | +| `label` | String | Label of option entity. | +| `value` | String | Value of option entity. | + +## ScannedResource + +Represents a resource scanned by a security scan + +| Name | Type | Description | +| --- | ---- | ---------- | +| `requestMethod` | String | The HTTP request method used to access the URL | +| `url` | String | The URL scanned by the scanner | + ## SecurityReportSummary Represents summary of a security report @@ -1540,6 +1731,7 @@ Represents summary of a security report | Name | Type | Description | | --- | ---- | ---------- | | `containerScanning` | SecurityReportSummarySection | Aggregated counts for the container_scanning scan | +| `coverageFuzzing` | SecurityReportSummarySection | Aggregated counts for the coverage_fuzzing scan | | `dast` | SecurityReportSummarySection | Aggregated counts for the dast scan | | `dependencyScanning` | SecurityReportSummarySection | Aggregated counts for the dependency_scanning scan | | `sast` | SecurityReportSummarySection | Aggregated counts for the sast scan | @@ -1552,8 +1744,19 @@ Represents a section of a summary of a security report | Name | Type | Description | | --- | ---- | ---------- | | `scannedResourcesCount` | Int | Total number of scanned resources | +| `scannedResourcesCsvPath` | String | Path to download all the scanned resources in CSV format | | `vulnerabilitiesCount` | Int | Total number of vulnerabilities | +## SecurityScanners + +Represents a list of security scanners + +| Name | Type | Description | +| --- | ---- | ---------- | +| `available` | SecurityScannerType! => Array | List of analyzers which are available for the project. | +| `enabled` | SecurityScannerType! => Array | List of analyzers which are enabled for the project. | +| `pipelineRun` | SecurityScannerType! => Array | List of analyzers which ran successfully in the latest pipeline. | + ## SentryDetailedError A Sentry error. @@ -1565,7 +1768,8 @@ A Sentry error. | `externalBaseUrl` | String! | External Base URL of the Sentry Instance | | `externalUrl` | String! | External URL of the error | | `firstReleaseLastCommit` | String | Commit the error was first seen | -| `firstReleaseShortVersion` | String | Release version the error was first seen | +| `firstReleaseShortVersion` | String | Release short version the error was first seen | +| `firstReleaseVersion` | String | Release version the error was first seen | | `firstSeen` | Time! | Timestamp when the error was first seen | | `frequency` | SentryErrorFrequency! => Array | Last 24hr stats of the error | | `gitlabCommit` | String | GitLab commit SHA attributed to the Error based on the release version | @@ -1573,7 +1777,8 @@ A Sentry error. | `gitlabIssuePath` | String | URL of GitLab Issue | | `id` | ID! | ID (global ID) of the error | | `lastReleaseLastCommit` | String | Commit the error was last seen | -| `lastReleaseShortVersion` | String | Release version the error was last seen | +| `lastReleaseShortVersion` | String | Release short version the error was last seen | +| `lastReleaseVersion` | String | Release version the error was last seen | | `lastSeen` | Time! | Timestamp when the error was last seen | | `message` | String | Sentry metadata message of the error | | `sentryId` | String! | ID (Sentry ID) of the error | @@ -1814,7 +2019,8 @@ Autogenerated return type of TodoRestoreMany | --- | ---- | ---------- | | `clientMutationId` | String | A unique identifier for the client performing the mutation. | | `errors` | String! => Array | Errors encountered during execution of the mutation. | -| `updatedIds` | ID! => Array | The ids of the updated todo items | +| `todos` | Todo! => Array | Updated todos | +| `updatedIds` **{warning-solid}** | ID! => Array | **Deprecated:** Use todos. Deprecated in 13.2 | ## TodoRestorePayload @@ -1834,7 +2040,8 @@ Autogenerated return type of TodosMarkAllDone | --- | ---- | ---------- | | `clientMutationId` | String | A unique identifier for the client performing the mutation. | | `errors` | String! => Array | Errors encountered during execution of the mutation. | -| `updatedIds` | ID! => Array | Ids of the updated todos | +| `todos` | Todo! => Array | Updated todos | +| `updatedIds` **{warning-solid}** | ID! => Array | **Deprecated:** Use todos. Deprecated in 13.2 | ## ToggleAwardEmojiPayload @@ -1877,6 +2084,7 @@ Autogenerated return type of UpdateAlertStatus | `clientMutationId` | String | A unique identifier for the client performing the mutation. | | `errors` | String! => Array | Errors encountered during execution of the mutation. | | `issue` | Issue | The issue created after mutation | +| `todo` | Todo | The todo after mutation | ## UpdateContainerExpirationPolicyPayload @@ -1918,6 +2126,16 @@ Autogenerated return type of UpdateIssue | `errors` | String! => Array | Errors encountered during execution of the mutation. | | `issue` | Issue | The issue after mutation | +## UpdateIterationPayload + +Autogenerated return type of UpdateIteration + +| Name | Type | Description | +| --- | ---- | ---------- | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | +| `iteration` | Iteration | The updated iteration | + ## UpdateNotePayload Autogenerated return type of UpdateNote @@ -1984,9 +2202,12 @@ Represents a vulnerability. | --- | ---- | ---------- | | `description` | String | Description of the vulnerability | | `id` | ID! | GraphQL ID of the vulnerability | +| `identifiers` | VulnerabilityIdentifier! => Array | Identifiers of the vulnerability. | | `location` | VulnerabilityLocation | Location metadata for the vulnerability. Its fields depend on the type of security scan that found the vulnerability | +| `primaryIdentifier` | VulnerabilityIdentifier | Primary identifier of the vulnerability. | | `project` | Project | The project on which the vulnerability was found | -| `reportType` | VulnerabilityReportType | Type of the security report that found the vulnerability (SAST, DEPENDENCY_SCANNING, CONTAINER_SCANNING, DAST, SECRET_DETECTION) | +| `reportType` | VulnerabilityReportType | Type of the security report that found the vulnerability (SAST, DEPENDENCY_SCANNING, CONTAINER_SCANNING, DAST, SECRET_DETECTION, COVERAGE_FUZZING) | +| `scanner` | VulnerabilityScanner | Scanner metadata for the vulnerability. | | `severity` | VulnerabilitySeverity | Severity of the vulnerability (INFO, UNKNOWN, LOW, MEDIUM, HIGH, CRITICAL) | | `state` | VulnerabilityState | State of the vulnerability (DETECTED, DISMISSED, RESOLVED, CONFIRMED) | | `title` | String | Title of the vulnerability | @@ -1994,6 +2215,17 @@ Represents a vulnerability. | `userPermissions` | VulnerabilityPermissions! | Permissions for the current user on the resource | | `vulnerabilityPath` | String | URL to the vulnerability's details page | +## VulnerabilityIdentifier + +Represents a vulnerability identifier. + +| Name | Type | Description | +| --- | ---- | ---------- | +| `externalId` | String | External ID of the vulnerability identifier | +| `externalType` | String | External type of the vulnerability identifier | +| `name` | String | Name of the vulnerability identifier | +| `url` | String | URL of the vulnerability identifier | + ## VulnerabilityIssueLink Represents an issue link of a vulnerability. @@ -2073,6 +2305,17 @@ Check permissions for the current user on a vulnerability | `readVulnerabilityFeedback` | Boolean! | Indicates the user can perform `read_vulnerability_feedback` on this resource | | `updateVulnerabilityFeedback` | Boolean! | Indicates the user can perform `update_vulnerability_feedback` on this resource | +## VulnerabilityScanner + +Represents a vulnerability scanner. + +| Name | Type | Description | +| --- | ---- | ---------- | +| `externalId` | String | External ID of the vulnerability scanner | +| `name` | String | Name of the vulnerability scanner | +| `reportType` | VulnerabilityReportType | Type of the vulnerability report | +| `vendor` | String | Vendor of the vulnerability scanner | + ## VulnerabilitySeveritiesCount Represents vulnerability counts by severity diff --git a/doc/api/group_clusters.md b/doc/api/group_clusters.md index b600a92bc38..17413ea2a3b 100644 --- a/doc/api/group_clusters.md +++ b/doc/api/group_clusters.md @@ -1,3 +1,9 @@ +--- +stage: Configure +group: Configure +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Group clusters API > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/30213) in GitLab 12.1. @@ -22,7 +28,7 @@ Parameters: Example request: ```shell -curl --header 'Private-Token: <your_access_token>' "https://gitlab.example.com/api/v4/groups/26/clusters" +curl --header "Private-Token: <your_access_token>" "https://gitlab.example.com/api/v4/groups/26/clusters" ``` Example response: @@ -238,7 +244,7 @@ through the ["Add existing cluster to group"](#add-existing-cluster-to-group) en Example request: ```shell -curl --header 'Private-Token: <your_access_token>' "https://gitlab.example.com/api/v4/groups/26/clusters/24" \ +curl --header "Private-Token: <your_access_token>" "https://gitlab.example.com/api/v4/groups/26/clusters/24" \ -H "Content-Type:application/json" \ --request PUT --data '{"name":"new-cluster-name","domain":"new-domain.com","api_url":"https://new-api-url.com"}' ``` diff --git a/doc/api/group_labels.md b/doc/api/group_labels.md index 5ae5ea4286a..260ee669e08 100644 --- a/doc/api/group_labels.md +++ b/doc/api/group_labels.md @@ -170,7 +170,8 @@ Example response: } ``` -NOTE: **Note:** An older endpoint `PUT /groups/:id/labels` with `name` in the parameters is still available, but deprecated. +NOTE: **Note:** +An older endpoint `PUT /groups/:id/labels` with `name` in the parameters is still available, but deprecated. ## Delete a group label @@ -189,7 +190,8 @@ DELETE /groups/:id/labels/:label_id curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/5/labels/bug" ``` -NOTE: **Note:** An older endpoint `DELETE /groups/:id/labels` with `name` in the parameters is still available, but deprecated. +NOTE: **Note:** +An older endpoint `DELETE /groups/:id/labels` with `name` in the parameters is still available, but deprecated. ## Subscribe to a group label diff --git a/doc/api/group_wikis.md b/doc/api/group_wikis.md new file mode 100644 index 00000000000..62094ffc940 --- /dev/null +++ b/doc/api/group_wikis.md @@ -0,0 +1,196 @@ +# Wikis API + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/212199) in GitLab 13.2. + +Available only in APIv4. + +## List wiki pages + +List all wiki pages for a given group. + +```plaintext +GET /groups/:id/wikis +``` + +| Attribute | Type | Required | Description | +| --------- | ------- | -------- | --------------------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) | +| `with_content` | boolean | no | Include pages' content | + +```shell +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/1/wikis?with_content=1" +``` + +Example response: + +```json +[ + { + "content" : "Here is an instruction how to deploy this project.", + "format" : "markdown", + "slug" : "deploy", + "title" : "deploy" + }, + { + "content" : "Our development process is described here.", + "format" : "markdown", + "slug" : "development", + "title" : "development" + },{ + "content" : "* [Deploy](deploy)\n* [Development](development)", + "format" : "markdown", + "slug" : "home", + "title" : "home" + } +] +``` + +## Get a wiki page + +Get a wiki page for a given group. + +```plaintext +GET /groups/:id/wikis/:slug +``` + +| Attribute | Type | Required | Description | +| --------- | ------- | -------- | --------------------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) | +| `slug` | string | yes | The slug (a unique string) of the wiki page | + +```shell +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/1/wikis/home" +``` + +Example response: + +```json +{ + "content" : "home page", + "format" : "markdown", + "slug" : "home", + "title" : "home" +} +``` + +## Create a new wiki page + +Create a new wiki page for the given repository with the given title, slug, and content. + +```plaintext +POST /projects/:id/wikis +``` + +| Attribute | Type | Required | Description | +| ------------- | ------- | -------- | ---------------------------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) | +| `content` | string | yes | The content of the wiki page | +| `title` | string | yes | The title of the wiki page | +| `format` | string | no | The format of the wiki page. Available formats are: `markdown` (default), `rdoc`, `asciidoc` and `org` | + +```shell +curl --data "format=rdoc&title=Hello&content=Hello world" \ + --header "PRIVATE-TOKEN: <your_access_token>" \ + "https://gitlab.example.com/api/v4/groups/1/wikis" +``` + +Example response: + +```json +{ + "content" : "Hello world", + "format" : "markdown", + "slug" : "Hello", + "title" : "Hello" +} +``` + +## Edit an existing wiki page + +Update an existing wiki page. At least one parameter is required to update the wiki page. + +```plaintext +PUT /groups/:id/wikis/:slug +``` + +| Attribute | Type | Required | Description | +| --------------- | ------- | --------------------------------- | ------------------------------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) | +| `content` | string | yes if `title` is not provided | The content of the wiki page | +| `title` | string | yes if `content` is not provided | The title of the wiki page | +| `format` | string | no | The format of the wiki page. Available formats are: `markdown` (default), `rdoc`, `asciidoc` and `org` | +| `slug` | string | yes | The slug (a unique identifier) of the wiki page | + +```shell +curl --request PUT --data "format=rdoc&content=documentation&title=Docs" \ + --header "PRIVATE-TOKEN: <your_access_token>" \ + "https://gitlab.example.com/api/v4/groups/1/wikis/foo" +``` + +Example response: + +```json +{ + "content" : "documentation", + "format" : "markdown", + "slug" : "Docs", + "title" : "Docs" +} +``` + +## Delete a wiki page + +Delete a wiki page with a given slug. + +```plaintext +DELETE /groups/:id/wikis/:slug +``` + +| Attribute | Type | Required | Description | +| --------- | ------- | -------- | --------------------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) | +| `slug` | string | yes | The slug (a unique identifier) of the wiki page | + +```shell +curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/1/wikis/foo" +``` + +On success the HTTP status code is `204` and no JSON response is expected. + +## Upload an attachment to the wiki repository + +Upload a file to the attachment folder inside the wiki's repository. The +attachment folder is the `uploads` folder. + +```plaintext +POST /groups/:id/wikis/attachments +``` + +| Attribute | Type | Required | Description | +| ------------- | ------- | -------- | ---------------------------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) | +| `file` | string | yes | The attachment to be uploaded | +| `branch` | string | no | The name of the branch. Defaults to the wiki repository default branch | + +To upload a file from your filesystem, use the `--form` argument. This causes +cURL to post data using the header `Content-Type: multipart/form-data`. +The `file=` parameter must point to a file on your filesystem and be preceded +by `@`. For example: + +```shell +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --form "file=@dk.png" "https://gitlab.example.com/api/v4/groups/1/wikis/attachments" +``` + +Example response: + +```json +{ + "file_name" : "dk.png", + "file_path" : "uploads/6a061c4cf9f1c28cb22c384b4b8d4e3c/dk.png", + "branch" : "master", + "link" : { + "url" : "uploads/6a061c4cf9f1c28cb22c384b4b8d4e3c/dk.png", + "markdown" : "" + } +} +``` diff --git a/doc/api/groups.md b/doc/api/groups.md index e58506380d1..a5a0c210540 100644 --- a/doc/api/groups.md +++ b/doc/api/groups.md @@ -19,7 +19,7 @@ Parameters: | `statistics` | boolean | no | Include group statistics (admins only) | | `with_custom_attributes` | boolean | no | Include [custom attributes](custom_attributes.md) in response (admins only) | | `owned` | boolean | no | Limit to groups explicitly owned by the current user | -| `min_access_level` | integer | no | Limit to groups where current user has at least this [access level](members.md) | +| `min_access_level` | integer | no | Limit to groups where current user has at least this [access level](members.md#valid-access-levels) | | `top_level_only` | boolean | no | Limit to top level groups, excluding all subgroups | ```plaintext @@ -94,7 +94,8 @@ GET /groups?statistics=true "wiki_size" : 100, "lfs_objects_size" : 123, "job_artifacts_size" : 57, - "packages_size": 0 + "packages_size": 0, + "snippets_size" : 50, } } ] @@ -121,7 +122,7 @@ Parameters: | Attribute | Type | Required | Description | | ------------------------ | ----------------- | -------- | ----------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) of the parent group | +| `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) of the immediate parent group | | `skip_groups` | array of integers | no | Skip the group IDs passed | | `all_available` | boolean | no | Show all the groups you have access to (defaults to `false` for authenticated users, `true` for admin); Attributes `owned` and `min_access_level` have precedence | | `search` | string | no | Return the list of authorized groups matching the search criteria | @@ -130,7 +131,7 @@ Parameters: | `statistics` | boolean | no | Include group statistics (admins only) | | `with_custom_attributes` | boolean | no | Include [custom attributes](custom_attributes.md) in response (admins only) | | `owned` | boolean | no | Limit to groups explicitly owned by the current user | -| `min_access_level` | integer | no | Limit to groups where current user has at least this [access level](members.md) | +| `min_access_level` | integer | no | Limit to groups where current user has at least this [access level](members.md#valid-access-levels) | ```plaintext GET /groups/:id/subgroups @@ -193,7 +194,7 @@ Parameters: | `with_merge_requests_enabled` | boolean | no | Limit by projects with merge requests feature enabled. Default is `false` | | `with_shared` | boolean | no | Include projects shared to this group. Default is `true` | | `include_subgroups` | boolean | no | Include projects in subgroups of this group. Default is `false` | -| `min_access_level` | integer | no | Limit to projects where current user has at least this [access level](members.md) | +| `min_access_level` | integer | no | Limit to projects where current user has at least this [access level](members.md#valid-access-levels) | | `with_custom_attributes` | boolean | no | Include [custom attributes](custom_attributes.md) in response (admins only) | | `with_security_reports` | boolean | no | **(ULTIMATE)** Return only projects that have security reports artifacts present in any of their builds. This means "projects with security reports enabled". Default is `false` | @@ -268,7 +269,7 @@ Parameters: | `starred` | boolean | no | Limit by projects starred by the current user | | `with_issues_enabled` | boolean | no | Limit by projects with issues feature enabled. Default is `false` | | `with_merge_requests_enabled` | boolean | no | Limit by projects with merge requests feature enabled. Default is `false` | -| `min_access_level` | integer | no | Limit to projects where current user has at least this [access level](members.md) | +| `min_access_level` | integer | no | Limit to projects where current user has at least this [access level](members.md#valid-access-levels) | | `with_custom_attributes` | boolean | no | Include [custom attributes](custom_attributes.md) in response (admins only) | Example response: @@ -666,8 +667,8 @@ Parameters: | `request_access_enabled` | boolean | no | Allow users to request member access. | | `parent_id` | integer | no | The parent group ID for creating nested group. | | `default_branch_protection` | integer | no | See [Options for `default_branch_protection`](#options-for-default_branch_protection). Default to the global level default branch protection setting. | -| `shared_runners_minutes_limit` | integer | no | **(STARTER ONLY)** Pipeline minutes quota for this group. | -| `extra_shared_runners_minutes_limit` | integer | no | **(STARTER ONLY)** Extra pipeline minutes quota for this group. | +| `shared_runners_minutes_limit` | integer | no | **(STARTER ONLY)** Pipeline minutes quota for this group (included in plan). Can be `nil` (default; inherit system default), `0` (unlimited) or `> 0` | +| `extra_shared_runners_minutes_limit` | integer | no | **(STARTER ONLY)** Extra pipeline minutes quota for this group (purchased in addition to the minutes included in the plan). | ### Options for `default_branch_protection` @@ -740,8 +741,8 @@ PUT /groups/:id | `request_access_enabled` | boolean | no | Allow users to request member access. | | `default_branch_protection` | integer | no | See [Options for `default_branch_protection`](#options-for-default_branch_protection). | | `file_template_project_id` | integer | no | **(PREMIUM)** The ID of a project to load custom file templates from. | -| `shared_runners_minutes_limit` | integer | no | **(STARTER ONLY)** Pipeline minutes quota for this group. | -| `extra_shared_runners_minutes_limit` | integer | no | **(STARTER ONLY)** Extra pipeline minutes quota for this group. | +| `shared_runners_minutes_limit` | integer | no | **(STARTER ONLY)** Pipeline minutes quota for this group (included in plan). Can be `nil` (default; inherit system default), `0` (unlimited) or `> 0` | +| `extra_shared_runners_minutes_limit` | integer | no | **(STARTER ONLY)** Extra pipeline minutes quota for this group (purchased in addition to the minutes included in the plan). | NOTE: **Note:** The `projects` and `shared_projects` attributes in the response are deprecated and will be [removed in API v5](https://gitlab.com/gitlab-org/gitlab/-/issues/213797). @@ -1051,7 +1052,7 @@ POST /groups/:id/ldap_group_links | `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) | | `cn` | string | no | The CN of an LDAP group | | `filter` | string | no | The LDAP filter for the group | -| `group_access` | integer | yes | Minimum access level for members of the LDAP group | +| `group_access` | integer | yes | Minimum [access level](members.md#valid-access-levels) for members of the LDAP group | | `provider` | string | yes | LDAP provider for the LDAP group link | NOTE: **Note:** @@ -1140,7 +1141,7 @@ POST /groups/:id/share | --------- | -------------- | -------- | ----------- | | `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) | | `group_id` | integer | yes | The ID of the group to share with | -| `group_access` | integer | yes | The [permissions level](members.md) to grant the group | +| `group_access` | integer | yes | The [access level](members.md#valid-access-levels) to grant the group | | `expires_at` | string | no | Share expiration date in ISO 8601 format: 2016-09-26 | ### Delete link sharing group with another group diff --git a/doc/api/import.md b/doc/api/import.md index 307796f8acb..54e7eb12ed1 100644 --- a/doc/api/import.md +++ b/doc/api/import.md @@ -29,3 +29,40 @@ Example response: "full_name": "Administrator / my-repo" } ``` + +## Import repository from Bitbucket Server + +Import your projects from Bitbucket Server to GitLab via the API. + +NOTE: **Note:** +The Bitbucket Project Key is only used for finding the repository in Bitbucket. +You must specify a `target_namespace` if you want to import the repository to a GitLab group. +If you do not specify `target_namespace`, the project will import to your personal user namespace. + +```plaintext +POST /import/bitbucket_server +``` + +| Attribute | Type | Required | Description | +|------------|---------|----------|---------------------| +| `bitbucket_server_url` | string | yes | Bitbucket Server URL | +| `bitbucket_server_username` | string | yes | Bitbucket Server Username | +| `personal_access_token` | string | yes | Bitbucket Server personal access token/password | +| `bitbucket_server_project` | string | yes | Bitbucket Project Key | +| `bitbucket_server_repo` | string | yes | Bitbucket Repository Name | +| `new_name` | string | no | New repo name | +| `target_namespace` | string | no | Namespace to import repo into | + +```shell +curl --request POST \ + --url https://gitlab.example.com/api/v4/import/bitbucket_server \ + --header "content-type: application/json" \ + --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" \ + --data '{ + "bitbucket_server_url": "http://bitbucket.example.com", + "bitbucket_server_username": "root", + "personal_access_token": "Nzk4MDcxODY4MDAyOiP8y410zF3tGAyLnHRv/E0+3xYs", + "bitbucket_server_project": "NEW", + "bitbucket_server_repo": "my-repo" +}' +``` diff --git a/doc/api/instance_clusters.md b/doc/api/instance_clusters.md new file mode 100644 index 00000000000..1108550eee7 --- /dev/null +++ b/doc/api/instance_clusters.md @@ -0,0 +1,293 @@ +# Instance clusters API + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/36001) in GitLab 13.2. + +NOTE: **Note:** +User will need admin access to use these endpoints. + +Use these API endpoints with your instance clusters, which enable you to use the same cluster across multiple projects. [More information](../user/instance/clusters/index.md) + +## List instance clusters + +Returns a list of instance clusters. + +```plaintext +GET /admin/clusters +``` + +Example request: + +```shell +curl --header "Private-Token: <your_access_token>" "https://gitlab.example.com/api/v4/admin/clusters" +``` + +Example response: + +```json +[ + { + "id": 9, + "name": "cluster-1", + "created_at": "2020-07-14T18:36:10.440Z", + "domain": null, + "provider_type": "user", + "platform_type": "kubernetes", + "environment_scope": "*", + "cluster_type": "instance_type", + "user": { + "id": 1, + "name": "Administrator", + "username": "root", + "state": "active", + "avatar_url": "https://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon", + "web_url": "https://gitlab.example.com/root" + }, + "platform_kubernetes": { + "api_url": "https://example.com", + "namespace": null, + "authorization_type": "rbac", + "ca_cert":"-----BEGIN CERTIFICATE-----IxMDM1MV0ZDJkZjM...-----END CERTIFICATE-----" + }, + "provider_gcp": null, + "management_project": null + }, + { + "id": 10, + "name": "cluster-2", + "created_at": "2020-07-14T18:39:05.383Z", + "domain": null, + "provider_type": "user", + "platform_type": "kubernetes", + "environment_scope": "staging", + "cluster_type": "instance_type", + "user": { + "id": 1, + "name": "Administrator", + "username": "root", + "state": "active", + "avatar_url": "https://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon", + "web_url": "https://gitlab.example.com/root" + }, + "platform_kubernetes": { + "api_url": "https://example.com", + "namespace": null, + "authorization_type": "rbac", + "ca_cert":"-----BEGIN CERTIFICATE-----LzEtMCadtaLGxcsGAZjM...-----END CERTIFICATE-----" + }, + "provider_gcp": null, + "management_project": null + } + { + "id": 11, + "name": "cluster-3", + ... + } +] + +``` + +## Get a single instance cluster + +Returns a single instance cluster. + +Parameters: + +| Attribute | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `cluster_id` | integer | yes | The ID of the cluster | + +```plaintext +GET /admin/clusters/:cluster_id +``` + +Example request: + +```shell +curl --header "Private-Token: <your_access_token>" "https://gitlab.example.com/api/v4/admin/clusters/9" +``` + +Example response: + +```json +{ + "id": 9, + "name": "cluster-1", + "created_at": "2020-07-14T18:36:10.440Z", + "domain": null, + "provider_type": "user", + "platform_type": "kubernetes", + "environment_scope": "*", + "cluster_type": "instance_type", + "user": { + "id": 1, + "name": "Administrator", + "username": "root", + "state": "active", + "avatar_url": "https://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon", + "web_url": "https://gitlab.example.com/root" + }, + "platform_kubernetes": { + "api_url": "https://example.com", + "namespace": null, + "authorization_type": "rbac", + "ca_cert":"-----BEGIN CERTIFICATE-----IxMDM1MV0ZDJkZjM...-----END CERTIFICATE-----" + }, + "provider_gcp": null, + "management_project": null +} +``` + +## Add existing instance cluster + +Adds an existing Kubernetes instance cluster. + +```plaintext +POST /admin/clusters/add +``` + +Parameters: + +| Attribute | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `name` | string | yes | The name of the cluster | +| `domain` | string | no | The [base domain](../user/project/clusters/index.md#base-domain) of the cluster | +| `environment_scope` | string | no | The associated environment to the cluster. Defaults to `*` | +| `management_project_id` | integer | no | The ID of the [management project](../user/clusters/management_project.md) for the cluster | +| `enabled` | boolean | no | Determines if cluster is active or not, defaults to true | +| `managed` | boolean | no | Determines if GitLab will manage namespaces and service accounts for this cluster, defaults to true | +| `platform_kubernetes_attributes[api_url]` | string | yes | The URL to access the Kubernetes API | +| `platform_kubernetes_attributes[token]` | string | yes | The token to authenticate against Kubernetes | +| `platform_kubernetes_attributes[ca_cert]` | string | no | TLS certificate. Required if API is using a self-signed TLS certificate. | +| `platform_kubernetes_attributes[namespace]` | string | no | The unique namespace related to the project | +| `platform_kubernetes_attributes[authorization_type]` | string | no | The cluster authorization type: `rbac`, `abac` or `unknown_authorization`. Defaults to `rbac`. | + +Example request: + +```shell +curl --header "Private-Token:<your_access_token>" "http://gitlab.example.com/api/v4/admin/clusters/add" \ +-H "Accept:application/json" \ +-H "Content-Type:application/json" \ +-X POST --data '{"name":"cluster-3", "environment_scope":"production", "platform_kubernetes_attributes":{"api_url":"https://example.com", "token":"12345", "ca_cert":"-----BEGIN CERTIFICATE-----qpoeiXXZafCM0ZDJkZjM...-----END CERTIFICATE-----"}}' + +``` + +Example response: + +```json +{ + "id": 11, + "name": "cluster-3", + "created_at": "2020-07-14T18:42:50.805Z", + "domain": null, + "provider_type": "user", + "platform_type": "kubernetes", + "environment_scope": "production", + "cluster_type": "instance_type", + "user": { + "id": 1, + "name": "Administrator", + "username": "root", + "state": "active", + "avatar_url": "https://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon", + "web_url": "http://gitlab.example.com:3000/root" + }, + "platform_kubernetes": { + "api_url": "https://example.com", + "namespace": null, + "authorization_type": "rbac", + "ca_cert":"-----BEGIN CERTIFICATE-----qpoeiXXZafCM0ZDJkZjM...-----END CERTIFICATE-----" + }, + "provider_gcp": null, + "management_project": null +} +``` + +## Edit instance cluster + +Updates an existing instance cluster. + +```shell +PUT /admin/clusters/:cluster_id +``` + +Parameters: + +| Attribute | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `cluster_id` | integer | yes | The ID of the cluster | +| `name` | string | no | The name of the cluster | +| `domain` | string | no | The [base domain](../user/project/clusters/index.md#base-domain) of the cluster | +| `environment_scope` | string | no | The associated environment to the cluster | +| `management_project_id` | integer | no | The ID of the [management project](../user/clusters/management_project.md) for the cluster | +| `enabled` | boolean | no | Determines if cluster is active or not, defaults to true | +| `platform_kubernetes_attributes[api_url]` | string | no | The URL to access the Kubernetes API | +| `platform_kubernetes_attributes[token]` | string | no | The token to authenticate against Kubernetes | +| `platform_kubernetes_attributes[ca_cert]` | string | no | TLS certificate. Required if API is using a self-signed TLS certificate. | +| `platform_kubernetes_attributes[namespace]` | string | no | The unique namespace related to the project | + +NOTE: **Note:** +`name`, `api_url`, `ca_cert` and `token` can only be updated if the cluster was added +through the [Add existing Kubernetes cluster](../user/project/clusters/add_remove_clusters.md#add-existing-cluster) option or +through the [Add existing instance cluster](#add-existing-instance-cluster) endpoint. + +Example request: + +```shell +curl --header "Private-Token: <your_access_token>" "http://gitlab.example.com/api/v4/admin/clusters/9" \ +-H "Content-Type:application/json" \ +-X PUT --data '{"name":"update-cluster-name", "platform_kubernetes_attributes":{"api_url":"https://new-example.com","token":"new-token"}}' + +``` + +Example response: + +```json +{ + "id": 9, + "name": "update-cluster-name", + "created_at": "2020-07-14T18:36:10.440Z", + "domain": null, + "provider_type": "user", + "platform_type": "kubernetes", + "environment_scope": "*", + "cluster_type": "instance_type", + "user": { + "id": 1, + "name": "Administrator", + "username": "root", + "state": "active", + "avatar_url": "https://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon", + "web_url": "https://gitlab.example.com/root" + }, + "platform_kubernetes": { + "api_url": "https://new-example.com", + "namespace": null, + "authorization_type": "rbac", + "ca_cert":"-----BEGIN CERTIFICATE-----IxMDM1MV0ZDJkZjM...-----END CERTIFICATE-----" + }, + "provider_gcp": null, + "management_project": null, + "project": null +} + +``` + +## Delete instance cluster + +Deletes an existing instance cluster. + +```plaintext +DELETE /admin/clusters/:cluster_id +``` + +Parameters: + +| Attribute | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `cluster_id` | integer | yes | The ID of the cluster | + +Example request: + +```shell +curl --request DELETE --header "Private-Token: <your_access_token>" "https://gitlab.example.com/api/v4/admin/clusters/11" +``` diff --git a/doc/api/instance_level_ci_variables.md b/doc/api/instance_level_ci_variables.md index 72d20109fbd..ceaf7e30c48 100644 --- a/doc/api/instance_level_ci_variables.md +++ b/doc/api/instance_level_ci_variables.md @@ -1,10 +1,7 @@ # Instance-level CI/CD variables API > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/14108) in GitLab 13.0 -> - It's deployed behind a feature flag, enabled by default. -> - It's enabled on GitLab.com. -> - It's recommended for production use. -> - For GitLab self-managed instances, GitLab administrators can opt to [disable it](#enable-or-disable-instance-level-cicd-variables-core-only). **(CORE ONLY)** +> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/218249) in GitLab 13.2. ## List all instance variables @@ -140,22 +137,3 @@ DELETE /admin/ci/variables/:key ```shell curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/admin/ci/variables/VARIABLE_1" ``` - -### Enable or disable instance-level CI/CD variables **(CORE ONLY)** - -Instance-level CI/CD variables is under development but ready for production use. -It is deployed behind a feature flag that is **enabled by default**. -[GitLab administrators with access to the GitLab Rails console](../administration/feature_flags.md) -can opt to disable it for your instance. - -To disable it: - -```ruby -Feature.disable(:ci_instance_level_variables) -``` - -To enable it: - -```ruby -Feature.enable(:ci_instance_level_variables) -``` diff --git a/doc/api/issues.md b/doc/api/issues.md index f640300e3ae..22f5d994f85 100644 --- a/doc/api/issues.md +++ b/doc/api/issues.md @@ -6,8 +6,6 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Issues API -Every API call to issues must be authenticated. - If a user is not a member of a project and the project is private, a `GET` request on that project will result in a `404` status code. @@ -18,11 +16,11 @@ are paginated. Read more on [pagination](README.md#pagination). -CAUTION: **Deprecation** +CAUTION: **Deprecation:** > `reference` attribute in response is deprecated in favour of `references`. > Introduced [GitLab 12.6](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/20354) -NOTE: **Note** +NOTE: **Note:** > `references.relative` is relative to the group / project that the issue is being requested. When issue is fetched from its project > `relative` format would be the same as `short` format and when requested across groups / projects it is expected to be the same as `full` format. @@ -72,7 +70,7 @@ GET /issues?confidential=true | `updated_after` | datetime | no | Return issues updated on or after the given time | | `updated_before` | datetime | no | Return issues updated on or before the given time | | `confidential` | boolean | no | Filter confidential or public issues. | -| `not` | Hash | no | Return issues that do not match the parameters supplied. Accepts: `labels`, `milestone`, `author_id`, `author_username`, `assignee_id`, `assignee_username`, `my_reaction_emoji`, `search`, `in` | +| `not` | Hash | no | Return issues that do not match the parameters supplied. Accepts: `labels`, `milestone`, `author_id`, `author_username`, `assignee_id`, `assignee_username`, `my_reaction_emoji` | | `non_archived` | boolean | no | Return issues only from non-archived projects. If `false`, response will return issues from both archived and non-archived projects. Default is `true`. _(Introduced in [GitLab 13.0](https://gitlab.com/gitlab-org/gitlab/-/issues/197170))_ | ```shell @@ -188,6 +186,9 @@ the `weight` parameter: Get a list of a group's issues. +If the group is private, credentials will need to be provided for authorization. +The preferred way to do this, is by using [personal access tokens](../user/profile/personal_access_tokens.md). + ```plaintext GET /groups/:id/issues GET /groups/:id/issues?state=opened @@ -343,6 +344,9 @@ the `weight` parameter: Get a list of a project's issues. +If the project is private, credentials will need to be provided for authorization. +The preferred way to do this, is by using [personal access tokens](../user/profile/personal_access_tokens.md). + ```plaintext GET /projects/:id/issues GET /projects/:id/issues?state=opened @@ -504,6 +508,9 @@ the `weight` parameter: Get a single project issue. +If the project is private or the issue is confidential, credentials will need to be provided for authorization. +The preferred way to do this, is by using [personal access tokens](../user/profile/personal_access_tokens.md). + ```plaintext GET /projects/:id/issues/:issue_iid ``` @@ -613,14 +620,13 @@ the `weight` parameter: } ``` -Users on GitLab [Ultimate](https://about.gitlab.com/pricing/) will additionally see +Users on GitLab [Premium](https://about.gitlab.com/pricing/) will additionally see the `epic` property: ```javascript { "project_id" : 4, "description" : "Omnis vero earum sunt corporis dolor et placeat.", - "epic": { "epic_iid" : 5, //deprecated, use `iid` of the `epic` attribute "epic": { "id" : 42, @@ -663,8 +669,8 @@ POST /projects/:id/issues | `merge_request_to_resolve_discussions_of` | integer | no | The IID of a merge request in which to resolve all issues. This will fill the issue with a default description and mark all discussions as resolved. When passing a description or title, these values will take precedence over the default values.| | `discussion_to_resolve` | string | no | The ID of a discussion to resolve. This will fill in the issue with a default description and mark the discussion as resolved. Use in combination with `merge_request_to_resolve_discussions_of`. | | `weight` **(STARTER)** | integer | no | The weight of the issue. Valid values are greater than or equal to 0. | -| `epic_id` **(ULTIMATE)** | integer | no | ID of the epic to add the issue to. Valid values are greater than or equal to 0. | -| `epic_iid` **(ULTIMATE)** | integer | no | IID of the epic to add the issue to. Valid values are greater than or equal to 0. (deprecated, [will be removed in version 5](https://gitlab.com/gitlab-org/gitlab/-/issues/35157)) | +| `epic_id` **(PREMIUM)** | integer | no | ID of the epic to add the issue to. Valid values are greater than or equal to 0. | +| `epic_iid` **(PREMIUM)** | integer | no | IID of the epic to add the issue to. Valid values are greater than or equal to 0. (deprecated, [will be removed in version 5](https://gitlab.com/gitlab-org/gitlab/-/issues/35157)) | ```shell curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/4/issues?title=Issues%20with%20auth&labels=bug" @@ -781,8 +787,8 @@ PUT /projects/:id/issues/:issue_iid | `due_date` | string | no | Date time string in the format YEAR-MONTH-DAY, for example `2016-03-11` | | `weight` **(STARTER)** | integer | no | The weight of the issue. Valid values are greater than or equal to 0. 0 | | `discussion_locked` | boolean | no | Flag indicating if the issue's discussion is locked. If the discussion is locked only project members can add or edit comments. | -| `epic_id` **(ULTIMATE)** | integer | no | ID of the epic to add the issue to. Valid values are greater than or equal to 0. | -| `epic_iid` **(ULTIMATE)** | integer | no | IID of the epic to add the issue to. Valid values are greater than or equal to 0. (deprecated, [will be removed in version 5](https://gitlab.com/gitlab-org/gitlab/-/issues/35157)) | +| `epic_id` **(PREMIUM)** | integer | no | ID of the epic to add the issue to. Valid values are greater than or equal to 0. | +| `epic_iid` **(PREMIUM)** | integer | no | IID of the epic to add the issue to. Valid values are greater than or equal to 0. (deprecated, [will be removed in version 5](https://gitlab.com/gitlab-org/gitlab/-/issues/35157)) | ```shell curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/4/issues/85?state_event=close" @@ -871,10 +877,10 @@ the `weight` parameter: NOTE: **Note:** At least one of following parameters is required to be passed for the request to be successful: `:assignee_id`, `:assignee_ids`, `:confidential`, `:created_at`, `:description`, `:discussion_locked`, `:due_date`, `:labels`, `:milestone_id`, `:state_event`, or `:title`. -NOTE: **Note**: +NOTE: **Note:** `assignee` column is deprecated. We now show it as a single-sized array `assignees` to conform to the GitLab EE API. -NOTE: **Note**: +NOTE: **Note:** The `closed_by` attribute was [introduced in GitLab 10.6](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/17042). This value will only be present for issues which were closed after GitLab 10.6 and when the user account that closed the issue still exists. ## Delete an issue @@ -894,6 +900,27 @@ DELETE /projects/:id/issues/:issue_iid curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/4/issues/85" ``` +## Reorder an issue + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/211864) as a [community contribution](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/35349) in GitLab 13.2. + +Reorders an issue, you can see the results when sorting issues manually + +```plaintext +PUT /projects/:id/issues/:issue_iid/reorder +``` + +| Attribute | Type | Required | Description | +|-------------|---------|----------|--------------------------------------| +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | +| `issue_iid` | integer | yes | The internal ID of a project's issue | +| `move_after_id` | integer | no | The ID of a projet's issue to move this issue after | +| `move_before_id` | integer | no | The ID of a projet's issue to move this issue before | + +```shell +curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/4/issues/85/reorder?move_after_id=51&move_before_id=92" +``` + ## Move an issue Moves an issue to a different project. If the target project @@ -1413,6 +1440,9 @@ Example response: ## Get time tracking stats +If the project is private or the issue is confidential, credentials will need to be provided for authorization. +The preferred way to do this, is by using [personal access tokens](../user/profile/personal_access_tokens.md). + ```plaintext GET /projects/:id/issues/:issue_iid/time_stats ``` @@ -1441,6 +1471,9 @@ Example response: Get all the merge requests that are related to the issue. +If the project is private or the issue is confidential, credentials will need to be provided for authorization. +The preferred way to do this, is by using [personal access tokens](../user/profile/personal_access_tokens.md). + ```plaintext GET /projects/:id/issues/:issue_id/related_merge_requests ``` @@ -1597,6 +1630,9 @@ Example response: Get all the merge requests that will close issue when merged. +If the project is private or the issue is confidential, credentials will need to be provided for authorization. +The preferred way to do this, is by using [personal access tokens](../user/profile/personal_access_tokens.md). + ```plaintext GET /projects/:id/issues/:issue_iid/closed_by ``` @@ -1670,6 +1706,9 @@ Example response: ## Participants on issues +If the project is private or the issue is confidential, credentials will need to be provided for authorization. +The preferred way to do this, is by using [personal access tokens](../user/profile/personal_access_tokens.md). + ```plaintext GET /projects/:id/issues/:issue_iid/participants ``` diff --git a/doc/api/labels.md b/doc/api/labels.md index 3ab059fca7c..30290f18653 100644 --- a/doc/api/labels.md +++ b/doc/api/labels.md @@ -199,7 +199,8 @@ DELETE /projects/:id/labels/:label_id curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/labels/bug" ``` -NOTE: **Note:** An older endpoint `DELETE /projects/:id/labels` with `name` in the parameters is still available, but deprecated. +NOTE: **Note:** +An older endpoint `DELETE /projects/:id/labels` with `name` in the parameters is still available, but deprecated. ## Edit an existing label @@ -242,7 +243,8 @@ Example response: } ``` -NOTE: **Note:** An older endpoint `PUT /projects/:id/labels` with `name` or `label_id` in the parameters is still available, but deprecated. +NOTE: **Note:** +An older endpoint `PUT /projects/:id/labels` with `name` or `label_id` in the parameters is still available, but deprecated. ## Promote a project label to a group label @@ -279,7 +281,8 @@ Example response: } ``` -NOTE: **Note:** An older endpoint `PUT /projects/:id/labels/promote` with `name` in the parameters is still available, but deprecated. +NOTE: **Note:** +An older endpoint `PUT /projects/:id/labels/promote` with `name` in the parameters is still available, but deprecated. ## Subscribe to a label diff --git a/doc/api/members.md b/doc/api/members.md index dadd609b7ed..8cd7bafdd77 100644 --- a/doc/api/members.md +++ b/doc/api/members.md @@ -1,6 +1,6 @@ # Group and project members API -**Valid access levels** +## Valid access levels The access levels are defined in the `Gitlab::Access` module. Currently, these levels are recognized: @@ -290,7 +290,7 @@ Example response: ### Set override flag for a member of a group -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/4875) in GitLab 12.10. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/4875) in GitLab 13.0. By default, the access level of LDAP group members is set to the value specified by LDAP through Group Sync. You can allow access level overrides by calling this endpoint. @@ -326,7 +326,7 @@ Example response: ### Remove override for a member of a group -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/4875) in GitLab 12.10. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/4875) in GitLab 13.0. Sets the override flag to false and allows LDAP Group Sync to reset the access level to the LDAP-prescribed value. @@ -373,6 +373,7 @@ DELETE /projects/:id/members/:user_id | --------- | ---- | -------- | ----------- | | `id` | integer/string | yes | The ID or [URL-encoded path of the project or group](README.md#namespaced-path-encoding) owned by the authenticated user | | `user_id` | integer | yes | The user ID of the member | +| `unassign_issuables` | boolean | false | Flag indicating if the removed member should be unassigned from any issues or merge requests within given group or project | ```shell curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/:id/members/:user_id" diff --git a/doc/api/merge_requests.md b/doc/api/merge_requests.md index 41c0428485f..959cf87ba62 100644 --- a/doc/api/merge_requests.md +++ b/doc/api/merge_requests.md @@ -2,11 +2,11 @@ Every API call to merge requests must be authenticated. -CAUTION: **Deprecation** +CAUTION: **Deprecation:** > `reference` attribute in response is deprecated in favour of `references`. > Introduced [GitLab 12.6](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/20354) -NOTE: **Note** +NOTE: **Note:** > `references.relative` is relative to the group / project that the merge request is being requested. When merge request is fetched from its project > `relative` format would be the same as `short` format and when requested across groups / projects it is expected to be the same as `full` format. @@ -64,6 +64,7 @@ Parameters: | `search` | string | no | Search merge requests against their `title` and `description` | | `in` | string | no | Modify the scope of the `search` attribute. `title`, `description`, or a string joining them with comma. Default is `title,description` | | `wip` | string | no | Filter merge requests against their `wip` status. `yes` to return *only* WIP merge requests, `no` to return *non* WIP merge requests | +| `not` | Hash | no | Return merge requests that do not match the parameters supplied. Accepts: `labels`, `milestone`, `author_id`, `author_username`, `assignee_id`, `assignee_username`, `my_reaction_emoji` | NOTE: **Note:** [Starting in GitLab 13.0](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/31890), @@ -1302,7 +1303,7 @@ curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://git Merge changes submitted with MR using this API. -If merge request is unable to be accepted (ie: Work in Progress, Closed, Pipeline Pending Completion, or Failed while requiring Success) - you'll get a `405` and the error message 'Method Not Allowed' +If merge request is unable to be accepted (ie: Draft, Closed, Pipeline Pending Completion, or Failed while requiring Success) - you'll get a `405` and the error message 'Method Not Allowed' If it has some conflicts and can not be merged - you'll get a `406` and the error message 'Branch cannot be merged' diff --git a/doc/api/metrics_dashboard_annotations.md b/doc/api/metrics_dashboard_annotations.md index 05bf7156a7e..10dfd3d1c3b 100644 --- a/doc/api/metrics_dashboard_annotations.md +++ b/doc/api/metrics_dashboard_annotations.md @@ -1,3 +1,10 @@ +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: concepts, howto +--- + # Dashboard annotations API > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/29089) in GitLab 12.10 behind a disabled feature flag. diff --git a/doc/api/metrics_user_starred_dashboards.md b/doc/api/metrics_user_starred_dashboards.md index dd9144d1319..df9cdd3b0e4 100644 --- a/doc/api/metrics_user_starred_dashboards.md +++ b/doc/api/metrics_user_starred_dashboards.md @@ -1,3 +1,10 @@ +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: concepts, howto +--- + # User-starred metrics dashboards API The starred dashboard feature makes navigating to frequently-used dashboards easier @@ -15,6 +22,7 @@ Parameters: | Attribute | Type | Required | Description | |:---------------|:---------------|:---------|:-----------------------------------------------------------------------------| +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) | | `dashboard_path` | string | yes | URL-encoded path to file defining the dashboard which should be marked as favorite. | ```shell @@ -45,6 +53,7 @@ Parameters: | Attribute | Type | Required | Description | |:---------------|:---------------|:---------|:-----------------------------------------------------------------------------| +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) | | `dashboard_path` | string | no | URL-encoded path to file defining the dashboard which should no longer be marked as favorite. When not supplied all dashboards within given projects will be removed from favorites. | ```shell diff --git a/doc/api/namespaces.md b/doc/api/namespaces.md index d1a2812bfb4..e38e725fb97 100644 --- a/doc/api/namespaces.md +++ b/doc/api/namespaces.md @@ -68,7 +68,8 @@ the `plan` parameter associated with a namespace: ] ``` -NOTE: **Note:** Only group maintainers/owners are presented with `members_count_with_descendants`, as well as `plan` **(BRONZE ONLY)**. +NOTE: **Note:** +Only group maintainers/owners are presented with `members_count_with_descendants`, as well as `plan` **(BRONZE ONLY)**. ## Search for namespace diff --git a/doc/api/notes.md b/doc/api/notes.md index 74d941edec1..9a75b950f28 100644 --- a/doc/api/notes.md +++ b/doc/api/notes.md @@ -116,6 +116,7 @@ Parameters: - `id` (required) - The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) - `issue_iid` (required) - The IID of an issue - `body` (required) - The content of a note. Limited to 1,000,000 characters. +- `confidential` (optional) - The confidential flag of a note. Default is false. - `created_at` (optional) - Date time string, ISO 8601 formatted, e.g. 2016-03-11T03:45:40Z (requires admin or project/group owner rights) ```shell diff --git a/doc/api/packages.md b/doc/api/packages.md index ca7113bc743..19828208a26 100644 --- a/doc/api/packages.md +++ b/doc/api/packages.md @@ -80,7 +80,7 @@ GET /groups/:id/packages curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/:id/packages?exclude_subgroups=true" ``` -CAUTION: **Deprecation** +CAUTION: **Deprecation:** > The `build_info` attribute in the response is deprecated in favour of `pipeline`. > Introduced [GitLab 12.10](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/28040). @@ -165,7 +165,7 @@ GET /projects/:id/packages/:package_id curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/:id/packages/:package_id" ``` -CAUTION: **Deprecation** +CAUTION: **Deprecation:** > The `build_info` attribute in the response is deprecated in favour of `pipeline`. > Introduced [GitLab 12.10](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/28040). diff --git a/doc/api/pipelines.md b/doc/api/pipelines.md index e84d7663bdb..563829b8192 100644 --- a/doc/api/pipelines.md +++ b/doc/api/pipelines.md @@ -142,7 +142,7 @@ Example of response > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/202525) in GitLab 13.0. CAUTION: **Caution:** -This API route is part of the [JUnit test report](../ci/junit_test_reports.md) feature. It is protected by a [feature flag](../development/feature_flags/index.md) that is **disabled** due to performance issues with very large data sets. See [the documentation for the feature](../ci/junit_test_reports.md#enabling-the-feature) for further details. +This API route is part of the [JUnit test report](../ci/junit_test_reports.md) feature. It is protected by a [feature flag](../development/feature_flags/index.md) that is **disabled** due to performance issues with very large data sets. ```plaintext GET /projects/:id/pipelines/:pipeline_id/test_report diff --git a/doc/api/project_clusters.md b/doc/api/project_clusters.md index f7c8ffc8df6..04694157561 100644 --- a/doc/api/project_clusters.md +++ b/doc/api/project_clusters.md @@ -1,3 +1,9 @@ +--- +stage: Configure +group: Configure +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Project clusters API > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/23922) in GitLab 11.7. @@ -22,7 +28,7 @@ Parameters: Example request: ```shell -curl --header 'Private-Token: <your_access_token>' "https://gitlab.example.com/api/v4/projects/26/clusters" +curl --header "Private-Token: <your_access_token>" "https://gitlab.example.com/api/v4/projects/26/clusters" ``` Example response: @@ -192,7 +198,7 @@ Parameters: Example request: ```shell -curl --header 'Private-Token: <your_access_token>' "https://gitlab.example.com/api/v4/projects/26/clusters/user" \ +curl --header "Private-Token: <your_access_token>" "https://gitlab.example.com/api/v4/projects/26/clusters/user" \ -H "Accept: application/json" \ -H "Content-Type:application/json" \ -X POST --data '{"name":"cluster-5", "platform_kubernetes_attributes":{"api_url":"https://35.111.51.20","token":"12345","namespace":"cluster-5-namespace","ca_cert":"-----BEGIN CERTIFICATE-----\r\nhFiK1L61owwDQYJKoZIhvcNAQELBQAw\r\nLzEtMCsGA1UEAxMkZDA1YzQ1YjctNzdiMS00NDY0LThjNmEtMTQ0ZDJkZjM4ZDBj\r\nMB4XDTE4MTIyNzIwMDM1MVoXDTIzMTIyNjIxMDM1MVowLzEtMCsGA1UEAxMkZDA1\r\nYzQ1YjctNzdiMS00NDY0LThjNmEtMTQ0ZDJkZjM.......-----END CERTIFICATE-----"}}' @@ -289,7 +295,7 @@ through the ["Add existing cluster to project"](#add-existing-cluster-to-project Example request: ```shell -curl --header 'Private-Token: <your_access_token>' "https://gitlab.example.com/api/v4/projects/26/clusters/24" \ +curl --header "Private-Token: <your_access_token>" "https://gitlab.example.com/api/v4/projects/26/clusters/24" \ -H "Content-Type:application/json" \ -X PUT --data '{"name":"new-cluster-name","domain":"new-domain.com","api_url":"https://new-api-url.com"}' ``` @@ -383,5 +389,5 @@ Parameters: Example request: ```shell -curl --request DELETE --header 'Private-Token: <your_access_token>' "https://gitlab.example.com/api/v4/projects/26/clusters/23" +curl --request DELETE --header "Private-Token: <your_access_token>" "https://gitlab.example.com/api/v4/projects/26/clusters/23" ``` diff --git a/doc/api/project_level_variables.md b/doc/api/project_level_variables.md index fbeba9d6c7d..407e506e082 100644 --- a/doc/api/project_level_variables.md +++ b/doc/api/project_level_variables.md @@ -43,6 +43,7 @@ GET /projects/:id/variables/:key |-----------|---------|----------|-----------------------| | `id` | integer/string | yes | The ID of a project or [urlencoded NAMESPACE/PROJECT_NAME of the project](README.md#namespaced-path-encoding) owned by the authenticated user | | `key` | string | yes | The `key` of a variable | +| `filter` | hash | no | Available filters: `[environment_scope]`. See the [`filter` parameter details](#the-filter-parameter). | ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/variables/TEST_VARIABLE_1" @@ -108,6 +109,7 @@ PUT /projects/:id/variables/:key | `protected` | boolean | no | Whether the variable is protected | | `masked` | boolean | no | Whether the variable is masked | | `environment_scope` | string | no | The `environment_scope` of the variable | +| `filter` | hash | no | Available filters: `[environment_scope]`. See the [`filter` parameter details](#the-filter-parameter). | ```shell curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/variables/NEW_VARIABLE" --form "value=updated value" @@ -136,7 +138,40 @@ DELETE /projects/:id/variables/:key |-----------|---------|----------|-------------------------| | `id` | integer/string | yes | The ID of a project or [urlencoded NAMESPACE/PROJECT_NAME of the project](README.md#namespaced-path-encoding) owned by the authenticated user | | `key` | string | yes | The `key` of a variable | +| `filter` | hash | no | Available filters: `[environment_scope]`. See the [`filter` parameter details](#the-filter-parameter). | ```shell curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/variables/VARIABLE_1" ``` + +## The `filter` parameter + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/34490) in GitLab 13.2. +> - It's deployed behind a feature flag, disabled by default. +> - It's disabled on GitLab.com. +> - To use it in GitLab self-managed instances, ask a GitLab administrator to enable it. + +This parameter is used for filtering by attributes, such as `environment_scope`. + +Example usage: + +```shell +curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/variables/VARIABLE_1?filter[environment_scope]=production" +``` + +### Enable or disable + +[GitLab administrators with access to the GitLab Rails console](../administration/feature_flags.md) +can enable it for your instance. + +To enable it: + +```ruby +Feature.enable(:ci_variables_api_filter_environment_scope) +``` + +To disable it: + +```ruby +Feature.disable(:ci_variables_api_filter_environment_scope) +``` diff --git a/doc/api/project_repository_storage_moves.md b/doc/api/project_repository_storage_moves.md index c55d4a19feb..f7fb361bf53 100644 --- a/doc/api/project_repository_storage_moves.md +++ b/doc/api/project_repository_storage_moves.md @@ -5,11 +5,12 @@ info: To determine the technical writer assigned to the Stage/Group associated w type: reference --- -# Project repository storage move API +# Project repository storage moves API **(CORE ONLY)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/31285) in GitLab 13.0. -Project repository storage can be moved. To retrieve project repository storage moves using the API, you must [authenticate yourself](README.md#authentication) as an administrator. +Project repository storage can be moved. To retrieve project repository storage moves using the API, +you must [authenticate yourself](README.md#authentication) as an administrator. ## Retrieve all project repository storage moves @@ -23,7 +24,7 @@ are [paginated](README.md#pagination). Example request: ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" 'https://gitlab.example.com/api/v4/project_repository_storage_moves' +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/project_repository_storage_moves" ``` Example response: @@ -66,7 +67,7 @@ Parameters: Example request: ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" 'https://gitlab.example.com/api/v4/projects/1/repository_storage_moves' +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/repository_storage_moves" ``` Example response: @@ -106,7 +107,7 @@ Parameters: Example request: ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" 'https://gitlab.example.com/api/v4/project_repository_storage_moves/1' +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/project_repository_storage_moves/1" ``` Example response: @@ -145,7 +146,7 @@ Parameters: Example request: ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" 'https://gitlab.example.com/api/v4/projects/1/repository_storage_moves/1' +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/repository_storage_moves/1" ``` Example response: @@ -185,7 +186,7 @@ Example request: ```shell curl --request POST --header "PRIVATE_TOKEN: <your_access_token>" --header "Content-Type: application/json" \ ---data '{"destination_storage_name":"storage2"}' 'https://gitlab.example.com/api/v4/projects/1/repository_storage_moves' +--data '{"destination_storage_name":"storage2"}' "https://gitlab.example.com/api/v4/projects/1/repository_storage_moves" ``` Example response: diff --git a/doc/api/project_snippets.md b/doc/api/project_snippets.md index e5dd85fb3bb..fd8cbd6e256 100644 --- a/doc/api/project_snippets.md +++ b/doc/api/project_snippets.md @@ -183,6 +183,28 @@ curl "https://gitlab.com/api/v4/projects/:id/snippets/:snippet_id/raw" \ --header "PRIVATE-TOKEN: <your_access_token>" ``` +## Snippet repository file content + +Returns the raw file content as plain text. + +```plaintext +GET /projects/:id/snippets/:snippet_id/files/:ref/:file_path/raw +``` + +Parameters: + +- `id` (required) - The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user +- `snippet_id` (required) - The ID of a project's snippet +- `ref` (required) - The name of a branch, tag or commit e.g. master +- `file_path` (required) - The URL-encoded path to the file, e.g. snippet%2Erb + +Example request: + +```shell +curl "https://gitlab.com/api/v4/projects/1/snippets/2/files/master/snippet%2Erb/raw" \ + --header "PRIVATE-TOKEN: <your_access_token>" +``` + ## Get user agent details > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/29508) in GitLab 9.4. diff --git a/doc/api/projects.md b/doc/api/projects.md index b9ba632cd9e..6257f37f0e6 100644 --- a/doc/api/projects.md +++ b/doc/api/projects.md @@ -45,7 +45,7 @@ GET /projects | --------- | ---- | -------- | ----------- | | `archived` | boolean | no | Limit by archived status | | `visibility` | string | no | Limit by visibility `public`, `internal`, or `private` | -| `order_by` | string | no | Return projects ordered by `id`, `name`, `path`, `created_at`, `updated_at`, or `last_activity_at` fields. Default is `created_at` | +| `order_by` | string | no | Return projects ordered by `id`, `name`, `path`, `created_at`, `updated_at`, or `last_activity_at` fields. `repository_size`, `storage_size`, or `wiki_size` fields are only allowed for admins. Default is `created_at` | | `sort` | string | no | Return projects sorted in `asc` or `desc` order. Default is `desc` | | `search` | string | no | Return list of projects matching the search criteria | | `search_namespaces` | boolean | no | Include ancestor namespaces when matching search criteria. Default is `false` | @@ -60,11 +60,12 @@ GET /projects | `with_programming_language` | string | no | Limit by projects which use the given programming language | | `wiki_checksum_failed` | boolean | no | **(PREMIUM)** Limit projects where the wiki checksum calculation has failed ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/6137) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.2) | | `repository_checksum_failed` | boolean | no | **(PREMIUM)** Limit projects where the repository checksum calculation has failed ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/6137) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.2) | -| `min_access_level` | integer | no | Limit by current user minimal [access level](members.md) | +| `min_access_level` | integer | no | Limit by current user minimal [access level](members.md#valid-access-levels) | | `id_after` | integer | no | Limit results to projects with IDs greater than the specified ID | | `id_before` | integer | no | Limit results to projects with IDs less than the specified ID | | `last_activity_after` | datetime | no | Limit results to projects with last_activity after specified time. Format: ISO 8601 YYYY-MM-DDTHH:MM:SSZ | | `last_activity_before` | datetime | no | Limit results to projects with last_activity before specified time. Format: ISO 8601 YYYY-MM-DDTHH:MM:SSZ | +| `repository_storage` | string | no | Limit results to projects stored on repository_storage. Available for admins only. | NOTE: **Note:** This endpoint supports [keyset pagination](README.md#keyset-based-pagination) for selected `order_by` options. @@ -174,7 +175,8 @@ When the user is authenticated and `simple` is not set this returns something li "wiki_size" : 0, "lfs_objects_size": 0, "job_artifacts_size": 0, - "packages_size": 0 + "packages_size": 0, + "snippets_size": 0 }, "_links": { "self": "http://example.com/api/v4/projects", @@ -276,7 +278,8 @@ When the user is authenticated and `simple` is not set this returns something li "wiki_size" : 0, "lfs_objects_size": 0, "job_artifacts_size": 0, - "packages_size": 0 + "packages_size": 0, + "snippets_size": 0 }, "_links": { "self": "http://example.com/api/v4/projects", @@ -348,7 +351,7 @@ GET /users/:user_id/projects | `with_issues_enabled` | boolean | no | Limit by enabled issues feature | | `with_merge_requests_enabled` | boolean | no | Limit by enabled merge requests feature | | `with_programming_language` | string | no | Limit by projects which use the given programming language | -| `min_access_level` | integer | no | Limit by current user minimal [access level](members.md) | +| `min_access_level` | integer | no | Limit by current user minimal [access level](members.md#valid-access-levels) | | `id_after` | integer | no | Limit results to projects with IDs greater than the specified ID | | `id_before` | integer | no | Limit results to projects with IDs less than the specified ID | @@ -425,7 +428,8 @@ This endpoint supports [keyset pagination](README.md#keyset-based-pagination) fo "wiki_size" : 0, "lfs_objects_size": 0, "job_artifacts_size": 0, - "packages_size": 0 + "packages_size": 0, + "snippets_size": 0 }, "_links": { "self": "http://example.com/api/v4/projects", @@ -527,7 +531,8 @@ This endpoint supports [keyset pagination](README.md#keyset-based-pagination) fo "wiki_size" : 0, "lfs_objects_size": 0, "job_artifacts_size": 0, - "packages_size": 0 + "packages_size": 0, + "snippets_size": 0 }, "_links": { "self": "http://example.com/api/v4/projects", @@ -566,7 +571,7 @@ GET /users/:user_id/starred_projects | `with_custom_attributes` | boolean | no | Include [custom attributes](custom_attributes.md) in response (admins only). | | `with_issues_enabled` | boolean | no | Limit by enabled issues feature. | | `with_merge_requests_enabled` | boolean | no | Limit by enabled merge requests feature. | -| `min_access_level` | integer | no | Limit by current user minimal [access level](members.md). | +| `min_access_level` | integer | no | Limit by current user minimal [access level](members.md#valid-access-levels). | ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/users/5/starred_projects" @@ -890,6 +895,7 @@ GET /projects/:id "suggestion_commit_message": null, "marked_for_deletion_at": "2020-04-03", // Deprecated and will be removed in API v5 in favor of marked_for_deletion_on "marked_for_deletion_on": "2020-04-03", + "compliance_frameworks": [ "sox" ], "statistics": { "commit_count": 37, "storage_size": 1038090, @@ -897,7 +903,8 @@ GET /projects/:id "wiki_size" : 0, "lfs_objects_size": 0, "job_artifacts_size": 0, - "packages_size": 0 + "packages_size": 0, + "snippets_size": 0 }, "_links": { "self": "http://example.com/api/v4/projects", @@ -1045,7 +1052,7 @@ POST /projects | `show_default_award_emojis` | boolean | no | Show default award emojis | | `resolve_outdated_diff_discussions` | boolean | no | Automatically resolve merge request diffs discussions on lines changed with a push | | `container_registry_enabled` | boolean | no | Enable container registry for this project | -| `container_expiration_policy_attributes` | hash | no | Update the image expiration policy for this project. Accepts: `cadence` (string), `keep_n` (string), `older_than` (string), `name_regex` (string), `name_regex_delete` (string), `name_regex_keep` (string), `enabled` (boolean) | +| `container_expiration_policy_attributes` | hash | no | Update the image cleanup policy for this project. Accepts: `cadence` (string), `keep_n` (string), `older_than` (string), `name_regex` (string), `name_regex_delete` (string), `name_regex_keep` (string), `enabled` (boolean) | | `shared_runners_enabled` | boolean | no | Enable shared runners for this project | | `visibility` | string | no | See [project visibility level](#project-visibility-level) | | `import_url` | string | no | URL to import repository from | @@ -1080,7 +1087,8 @@ POST /projects | `group_with_project_templates_id` | integer | no | **(PREMIUM)** For group-level custom templates, specifies ID of group from which all the custom project templates are sourced. Leave empty for instance-level templates. Requires `use_custom_template` to be true | | `packages_enabled` | boolean | no | **(PREMIUM ONLY)** Enable or disable packages repository feature | -NOTE: **Note:** If your HTTP repository is not publicly accessible, +NOTE: **Note:** +If your HTTP repository is not publicly accessible, add authentication information to the URL: `https://username:password@gitlab.company.com/group/project.git` where `password` is a public access key with the `api` scope enabled. @@ -1150,7 +1158,8 @@ POST /projects/user/:user_id | `group_with_project_templates_id` | integer | no | **(PREMIUM)** For group-level custom templates, specifies ID of group from which all the custom project templates are sourced. Leave empty for instance-level templates. Requires `use_custom_template` to be true | | `packages_enabled` | boolean | no | **(PREMIUM ONLY)** Enable or disable packages repository feature | -NOTE: **Note:** If your HTTP repository is not publicly accessible, +NOTE: **Note:** +If your HTTP repository is not publicly accessible, add authentication information to the URL: `https://username:password@gitlab.company.com/group/project.git` where `password` is a public access key with the `api` scope enabled. @@ -1186,7 +1195,7 @@ PUT /projects/:id | `show_default_award_emojis` | boolean | no | Show default award emojis | | `resolve_outdated_diff_discussions` | boolean | no | Automatically resolve merge request diffs discussions on lines changed with a push | | `container_registry_enabled` | boolean | no | Enable container registry for this project | -| `container_expiration_policy_attributes` | hash | no | Update the image expiration policy for this project. Accepts: `cadence` (string), `keep_n` (string), `older_than` (string), `name_regex` (string), `name_regex_delete` (string), `name_regex_keep` (string), `enabled` (boolean) | +| `container_expiration_policy_attributes` | hash | no | Update the image cleanup policy for this project. Accepts: `cadence` (string), `keep_n` (string), `older_than` (string), `name_regex` (string), `name_regex_delete` (string), `name_regex_keep` (string), `enabled` (boolean) | | `shared_runners_enabled` | boolean | no | Enable shared runners for this project | | `visibility` | string | no | See [project visibility level](#project-visibility-level) | | `import_url` | string | no | URL to import repository from | @@ -1219,9 +1228,10 @@ PUT /projects/:id | `only_mirror_protected_branches` | boolean | no | **(STARTER)** Only mirror protected branches | | `mirror_overwrites_diverged_branches` | boolean | no | **(STARTER)** Pull mirror overwrites diverged branches | | `packages_enabled` | boolean | no | **(PREMIUM ONLY)** Enable or disable packages repository feature | -| `service_desk_enabled` | boolean | no | **(PREMIUM ONLY)** Enable or disable service desk feature | +| `service_desk_enabled` | boolean | no | Enable or disable service desk feature | -NOTE: **Note:** If your HTTP repository is not publicly accessible, +NOTE: **Note:** +If your HTTP repository is not publicly accessible, add authentication information to the URL: `https://username:password@gitlab.company.com/group/project.git` where `password` is a public access key with the `api` scope enabled. @@ -1272,7 +1282,7 @@ GET /projects/:id/forks | `with_custom_attributes` | boolean | no | Include [custom attributes](custom_attributes.md) in response (admins only) | | `with_issues_enabled` | boolean | no | Limit by enabled issues feature | | `with_merge_requests_enabled` | boolean | no | Limit by enabled merge requests feature | -| `min_access_level` | integer | no | Limit by current user minimal [access level](members.md) | +| `min_access_level` | integer | no | Limit by current user minimal [access level](members.md#valid-access-levels) | ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/forks" @@ -1824,12 +1834,19 @@ Example response: ## Remove project -This endpoint either: +This endpoint: - Removes a project including all associated resources (issues, merge requests etc). -- From [GitLab 12.6](https://gitlab.com/gitlab-org/gitlab/-/issues/32935) on [Premium or Silver](https://about.gitlab.com/pricing/) or higher tiers, marks a project for deletion. Actual - deletion happens after number of days specified in - [instance settings](../user/admin_area/settings/visibility_and_access_controls.md#default-deletion-adjourned-period-premium-only). +- From [GitLab 13.2](https://gitlab.com/gitlab-org/gitlab/-/issues/220382) on [Premium or Silver](https://about.gitlab.com/pricing/) or higher tiers, +group admins can [configure](../user/group/index.md#enabling-delayed-project-removal-premium) projects within a group +to be deleted after a delayed period. +When enabled, actual deletion happens after the number of days +specified in the [default deletion period](../user/admin_area/settings/visibility_and_access_controls.md#default-deletion-adjourned-period-premium-only). + +CAUTION: **Warning:** +The default behavior of [Delayed Project deletion](https://gitlab.com/gitlab-org/gitlab/-/issues/32935) in GitLab 12.6 +was changed to [Immediate deletion](https://gitlab.com/gitlab-org/gitlab/-/issues/220382) +in GitLab 13.2, as discussed in [Enabling delayed project removal](../user/group/index.md#enabling-delayed-project-removal-premium). ```plaintext DELETE /projects/:id @@ -1902,7 +1919,7 @@ POST /projects/:id/share | --------- | ---- | -------- | ----------- | | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) | | `group_id` | integer | yes | The ID of the group to share with | -| `group_access` | integer | yes | The [permissions level](members.md) to grant the group | +| `group_access` | integer | yes | The [access level](members.md#valid-access-levels) to grant the group | | `expires_at` | string | no | Share expiration date in ISO 8601 format: 2016-09-26 | ## Delete a shared project link within a group diff --git a/doc/api/protected_environments.md b/doc/api/protected_environments.md index 765b8d2364d..56b399cec9b 100644 --- a/doc/api/protected_environments.md +++ b/doc/api/protected_environments.md @@ -1,3 +1,10 @@ +--- +stage: Release +group: Release Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: concepts, howto +--- + # Protected environments API **(PREMIUM)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/30595) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.8. diff --git a/doc/api/releases/links.md b/doc/api/releases/links.md index 35cb66e59a1..242b5eb41f5 100644 --- a/doc/api/releases/links.md +++ b/doc/api/releases/links.md @@ -138,7 +138,7 @@ PUT /projects/:id/releases/:tag_name/assets/links/:link_id | `url` | string | no | The URL of the link. | | `link_type` | string | no | The type of the link: `other`, `runbook`, `image`, `package`. Defaults to `other`. | -NOTE: **NOTE** +NOTE: **Note:** You have to specify at least one of `name` or `url` Example request: diff --git a/doc/api/resource_milestone_events.md b/doc/api/resource_milestone_events.md index 695687ada6d..8a81615857c 100644 --- a/doc/api/resource_milestone_events.md +++ b/doc/api/resource_milestone_events.md @@ -6,8 +6,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Resource milestone events API -Resource milestone events keep track of what happens to GitLab [issues](../user/project/issues/), -[merge requests](../user/project/merge_requests/), and [epics](../user/group/epics/). +Resource milestone events keep track of what happens to GitLab [issues](../user/project/issues/) and +[merge requests](../user/project/merge_requests/). Use them to track which milestone was added or removed, who did it, and when it happened. diff --git a/doc/api/resource_state_events.md b/doc/api/resource_state_events.md new file mode 100644 index 00000000000..6b257f10c6e --- /dev/null +++ b/doc/api/resource_state_events.md @@ -0,0 +1,212 @@ +--- +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Resource state events API + +Resource state events keep track of what happens to GitLab [issues](../user/project/issues/) and +[merge requests](../user/project/merge_requests/). + +Use them to track which state was set, who did it, and when it happened. + +## Issues + +### List project issue state events + +Gets a list of all state events for a single issue. + +```plaintext +GET /projects/:id/issues/:issue_iid/resource_state_events +``` + +| Attribute | Type | Required | Description | +| ----------- | -------------- | -------- | ------------------------------------------------------------------------------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) | +| `issue_iid` | integer | yes | The IID of an issue | + +Example request: + +```shell +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/issues/11/resource_state_events" +``` + +Example response: + +```json +[ + { + "id": 142, + "user": { + "id": 1, + "name": "Administrator", + "username": "root", + "state": "active", + "avatar_url": "https://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon", + "web_url": "http://gitlab.example.com/root" + }, + "created_at": "2018-08-20T13:38:20.077Z", + "resource_type": "Issue", + "resource_id": 11, + "state": "opened" + }, + { + "id": 143, + "user": { + "id": 1, + "name": "Administrator", + "username": "root", + "state": "active", + "avatar_url": "https://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon", + "web_url": "http://gitlab.example.com/root" + }, + "created_at": "2018-08-21T14:38:20.077Z", + "resource_type": "Issue", + "resource_id": 11, + "state": "closed" + } +] +``` + +### Get single issue state event + +Returns a single state event for a specific project issue + +```plaintext +GET /projects/:id/issues/:issue_iid/resource_state_events/:resource_state_event_id +``` + +Parameters: + +| Attribute | Type | Required | Description | +| ----------------------------- | -------------- | -------- | ------------------------------------------------------------------------------- | +| `id` | integer/string | yes | The ID or [URL-encoded path](README.md#namespaced-path-encoding) of the project | +| `issue_iid` | integer | yes | The IID of an issue | +| `resource_state_event_id` | integer | yes | The ID of a state event | + +Example request: + +```shell +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/issues/11/resource_state_events/143" +``` + +Example response: + +```json +{ + "id": 143, + "user": { + "id": 1, + "name": "Administrator", + "username": "root", + "state": "active", + "avatar_url": "https://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon", + "web_url": "http://gitlab.example.com/root" + }, + "created_at": "2018-08-21T14:38:20.077Z", + "resource_type": "Issue", + "resource_id": 11, + "state": "closed" +} +``` + +## Merge requests + +### List project merge request state events + +Gets a list of all state events for a single merge request. + +```plaintext +GET /projects/:id/merge_requests/:merge_request_iid/resource_state_events +``` + +| Attribute | Type | Required | Description | +| ------------------- | -------------- | -------- | ------------------------------------------------------------------------------- | +| `id` | integer/string | yes | The ID or [URL-encoded path](README.md#namespaced-path-encoding) of the project | +| `merge_request_iid` | integer | yes | The IID of a merge request | + +Example request: + +```shell +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/merge_requests/11/resource_state_events" +``` + +Example response: + +```json +[ + { + "id": 142, + "user": { + "id": 1, + "name": "Administrator", + "username": "root", + "state": "active", + "avatar_url": "https://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon", + "web_url": "http://gitlab.example.com/root" + }, + "created_at": "2018-08-20T13:38:20.077Z", + "resource_type": "MergeRequest", + "resource_id": 11, + "state": "opened" + }, + { + "id": 143, + "user": { + "id": 1, + "name": "Administrator", + "username": "root", + "state": "active", + "avatar_url": "https://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon", + "web_url": "http://gitlab.example.com/root" + }, + "created_at": "2018-08-21T14:38:20.077Z", + "resource_type": "MergeRequest", + "resource_id": 11, + "state": "closed" + } +] +``` + +### Get single merge request state event + +Returns a single state event for a specific project merge request + +```plaintext +GET /projects/:id/merge_requests/:merge_request_iid/resource_state_events/:resource_state_event_id +``` + +Parameters: + +| Attribute | Type | Required | Description | +| ----------------------------- | -------------- | -------- | ------------------------------------------------------------------------------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) | +| `merge_request_iid` | integer | yes | The IID of a merge request | +| `resource_state_event_id` | integer | yes | The ID of a state event | + +Example request: + +```shell +curl --request GET --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/merge_requests/11/resource_state_events/120" +``` + +Example response: + +```json +{ + "id": 120, + "user": { + "id": 1, + "name": "Administrator", + "username": "root", + "state": "active", + "avatar_url": "https://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon", + "web_url": "http://gitlab.example.com/root" + }, + "created_at": "2018-08-21T14:38:20.077Z", + "resource_type": "MergeRequest", + "resource_id": 11, + "state": "closed" +} +``` diff --git a/doc/api/resource_weight_events.md b/doc/api/resource_weight_events.md new file mode 100644 index 00000000000..700ef288440 --- /dev/null +++ b/doc/api/resource_weight_events.md @@ -0,0 +1,108 @@ +--- +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Resource weight events API + +Resource weight events keep track of what happens to GitLab [issues](../user/project/issues/). + +Use them to track which weight was set, who did it, and when it happened. + +## Issues + +### List project issue weight events + +Gets a list of all weight events for a single issue. + +```plaintext +GET /projects/:id/issues/:issue_iid/resource_weight_events +``` + +| Attribute | Type | Required | Description | +| ----------- | -------------- | -------- | ------------------------------------------------------------------------------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) | +| `issue_iid` | integer | yes | The IID of an issue | + +Example request: + +```shell +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/issues/11/resource_weight_events" +``` + +Example response: + +```json +[ + { + "id": 142, + "user": { + "id": 1, + "name": "Administrator", + "username": "root", + "state": "active", + "avatar_url": "https://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon", + "web_url": "http://gitlab.example.com/root" + }, + "created_at": "2018-08-20T13:38:20.077Z", + "issue_id": 253, + "weight": 3 + }, + { + "id": 143, + "user": { + "id": 1, + "name": "Administrator", + "username": "root", + "state": "active", + "avatar_url": "https://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon", + "web_url": "http://gitlab.example.com/root" + }, + "created_at": "2018-08-21T14:38:20.077Z", + "issue_id": 253, + "weight": 2 + } +] +``` + +### Get single issue weight event + +Returns a single weight event for a specific project issue + +```plaintext +GET /projects/:id/issues/:issue_iid/resource_weight_events/:resource_weight_event_id +``` + +Parameters: + +| Attribute | Type | Required | Description | +| ----------------------------- | -------------- | -------- | ------------------------------------------------------------------------------- | +| `id` | integer/string | yes | The ID or [URL-encoded path](README.md#namespaced-path-encoding) of the project | +| `issue_iid` | integer | yes | The IID of an issue | +| `resource_weight_event_id` | integer | yes | The ID of a weight event | + +Example request: + +```shell +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/issues/11/resource_weight_events/143" +``` + +Example response: + +```json +{ +"id": 143, +"user": { + "id": 1, + "name": "Administrator", + "username": "root", + "state": "active", + "avatar_url": "https://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon", + "web_url": "http://gitlab.example.com/root" +}, +"created_at": "2018-08-21T14:38:20.077Z", +"issue_id": 253, +"weight": 2 +} +``` diff --git a/doc/api/services.md b/doc/api/services.md index 02048a27c1b..4052fd22641 100644 --- a/doc/api/services.md +++ b/doc/api/services.md @@ -493,6 +493,42 @@ Get Emails on push service settings for a project. GET /projects/:id/services/emails-on-push ``` +## Confluence service + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/220934) in GitLab 13.2. + +Replaces the link to the internal wiki with a link to a Confluence Cloud Workspace. + +### Create/Edit Confluence service + +Set Confluence service for a project. + +```plaintext +PUT /projects/:id/services/confluence +``` + +Parameters: + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `confluence_url` | string | true | The URL of the Confluence Cloud Workspace hosted on atlassian.net. | + +### Delete Confluence service + +Delete Confluence service for a project. + +```plaintext +DELETE /projects/:id/services/confluence +``` + +### Get Confluence service settings + +Get Confluence service settings for a project. + +```plaintext +GET /projects/:id/services/confluence +``` + ## External Wiki Replaces the link to the internal wiki with a link to an external wiki. @@ -1008,7 +1044,7 @@ Parameters: | --------- | ---- | -------- | ----------- | | `api_url` | string | true | Prometheus API Base URL. For example, `http://prometheus.example.com/`. | | `google_iap_audience_client_id` | string | false | Client ID of the IAP secured resource (looks like IAP_CLIENT_ID.apps.googleusercontent.com) | -| `google_iap_service_account_json` | string | false | credentials.json file for your service account, like { "type": "service_account", "project_id": ... } | +| `google_iap_service_account_json` | string | false | `credentials.json` file for your service account, like { "type": "service_account", "project_id": ... } | ### Delete Prometheus service diff --git a/doc/api/settings.md b/doc/api/settings.md index 78d992cff58..d87a3c72a7e 100644 --- a/doc/api/settings.md +++ b/doc/api/settings.md @@ -48,7 +48,7 @@ Example response: "sign_in_text" : null, "container_expiration_policies_enable_historic_entries": true, "container_registry_token_expire_delay": 5, - "repository_storages": ["default"], + "repository_storages_weighted": {"default": 100}, "plantuml_enabled": false, "plantuml_url": null, "terminal_max_session_time": 0, @@ -291,6 +291,8 @@ are listed in the descriptions of the relevant settings. | `mirror_max_capacity` | integer | no | **(PREMIUM)** Maximum number of mirrors that can be synchronizing at the same time. | | `mirror_max_delay` | integer | no | **(PREMIUM)** Maximum time (in minutes) between updates that a mirror can have when scheduled to synchronize. | | `npm_package_requests_forwarding` | boolean | no | **(PREMIUM)** Use npmjs.org as a default remote repository when the package is not found in the GitLab NPM Registry | +| `maintenance_mode` | boolean | no | **(PREMIUM)** When instance is in maintenance mode, non-admin users can sign in with read-only access and make read-only API requests | +| `maintenance_mode_message` | string | no | **(PREMIUM)** Message displayed when instance is in maintenance mode | | `outbound_local_requests_whitelist` | array of strings | no | Define a list of trusted domains or ip addresses to which local requests are allowed when local requests for hooks and services are disabled. | `pages_domain_verification_enabled` | boolean | no | Require users to prove ownership of custom domains. Domain verification is an essential security measure for public GitLab sites. Users are required to demonstrate they control a domain before it is enabled. | | `password_authentication_enabled_for_git` | boolean | no | Enable authentication for Git over HTTP(S) via a GitLab account password. Default is `true`. | @@ -314,7 +316,8 @@ are listed in the descriptions of the relevant settings. | `receive_max_input_size` | integer | no | Maximum push size (MB). | | `repository_checks_enabled` | boolean | no | GitLab will periodically run `git fsck` in all project and wiki repositories to look for silent disk corruption issues. | | `repository_size_limit` | integer | no | **(PREMIUM)** Size limit per repository (MB) | -| `repository_storages` | array of strings | no | A list of names of enabled storage paths, taken from `gitlab.yml`. New projects will be created in one of these stores, chosen at random. | +| `repository_storages` | array of strings | no | (GitLab 13.0 and earlier) List of names of enabled storage paths, taken from `gitlab.yml`. New projects are created in one of these stores, chosen at random. | +| `repository_storages_weighted` | hash of strings to integers | no | (GitLab 13.1 and later) Hash of names of taken from `gitlab.yml` to weights. New projects are created in one of these stores, chosen by a weighted random selection. | | `require_two_factor_authentication` | boolean | no | (**If enabled, requires:** `two_factor_grace_period`) Require all users to set up Two-factor authentication. | | `restricted_visibility_levels` | array of strings | no | Selected levels cannot be used by non-admin users for groups, projects or snippets. Can take `private`, `internal` and `public` as a parameter. Default is `null` which means there is no restriction. | | `rsa_key_restriction` | integer | no | The minimum allowed bit length of an uploaded RSA key. Default is `0` (no restriction). `-1` disables RSA keys. | diff --git a/doc/api/snippets.md b/doc/api/snippets.md index 1aa3eecfd29..db94716c2d4 100644 --- a/doc/api/snippets.md +++ b/doc/api/snippets.md @@ -150,6 +150,34 @@ Example response: Hello World snippet ``` +## Snippet repository file content + +Returns the raw file content as plain text. + +```plaintext +GET /snippets/:id/files/:ref/:file_path/raw +``` + +Parameters: + +| Attribute | Type | Required | Description | +|:------------|:--------|:---------|:-------------------------------------------------------------------| +| `id` | integer | yes | ID of snippet to retrieve | +| `ref` | string | yes | Reference to a tag, branch or commit | +| `file_path` | string | yes | URL-encoded path to the file | + +Example request: + +```shell +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/snippets/1/files/master/snippet%2Erb/raw" +``` + +Example response: + +```plaintext +Hello World snippet +``` + ## Create new snippet Create a new snippet. diff --git a/doc/api/users.md b/doc/api/users.md index 6ac1cd089e7..505468945cb 100644 --- a/doc/api/users.md +++ b/doc/api/users.md @@ -89,7 +89,8 @@ GET /users "web_url": "http://localhost:3000/john_smith", "created_at": "2012-05-23T08:00:58Z", "is_admin": false, - "bio": null, + "bio": "", + "bio_html": "", "location": null, "skype": "", "linkedin": "", @@ -128,7 +129,8 @@ GET /users "web_url": "http://localhost:3000/jack_smith", "created_at": "2012-05-23T08:01:01Z", "is_admin": false, - "bio": null, + "bio": "", + "bio_html": "", "location": null, "skype": "", "linkedin": "", @@ -245,7 +247,8 @@ Parameters: "avatar_url": "http://localhost:3000/uploads/user/avatar/1/cd8.jpeg", "web_url": "http://localhost:3000/john_smith", "created_at": "2012-05-23T08:00:58Z", - "bio": null, + "bio": "", + "bio_html": "", "location": null, "public_email": "john@example.com", "skype": "", @@ -280,7 +283,8 @@ Example Responses: "web_url": "http://localhost:3000/john_smith", "created_at": "2012-05-23T08:00:58Z", "is_admin": false, - "bio": null, + "bio": "", + "bio_html": "", "location": null, "public_email": "john@example.com", "skype": "", @@ -314,7 +318,8 @@ Example Responses: } ``` -NOTE: **Note:** The `plan` and `trial` parameters are only available on GitLab Enterprise Edition. +NOTE: **Note:** +The `plan` and `trial` parameters are only available on GitLab Enterprise Edition. Users on GitLab [Starter, Bronze, or higher](https://about.gitlab.com/pricing/) will also see the `shared_runners_minutes_limit`, and `extra_shared_runners_minutes_limit` parameters. @@ -368,6 +373,9 @@ over `password`. In addition, `reset_password` and NOTE: **Note:** From [GitLab 12.1](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/29888/), `private_profile` will default to `false`. +NOTE: **Note:** +From [GitLab 13.2](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/35604), `bio` will default to `""` instead of `null`. + ```plaintext POST /users ``` @@ -384,7 +392,7 @@ Parameters: | `email` | Yes | Email | | `extern_uid` | No | External UID | | `external` | No | Flags the user as external - true or false (default) | -| `extra_shared_runners_minutes_limit` | No | Extra pipeline minutes quota for this user **(STARTER)** | +| `extra_shared_runners_minutes_limit` | No | Extra pipeline minutes quota for this user (purchased in addition to the minutes included in the plan) **(STARTER)** | | `force_random_password` | No | Set user password to a random value - true or false (default) | | `group_id_for_saml` | No | ID of group where SAML has been configured | | `linkedin` | No | LinkedIn | @@ -398,7 +406,7 @@ Parameters: | `provider` | No | External provider name | | `public_email` | No | The public email of the user | | `reset_password` | No | Send user password reset link - true or false(default) | -| `shared_runners_minutes_limit` | No | Pipeline minutes quota for this user **(STARTER)** | +| `shared_runners_minutes_limit` | No | Pipeline minutes quota for this user (included in plan). Can be `nil` (default; inherit system default), `0` (unlimited) or `> 0` **(STARTER)** | | `skip_confirmation` | No | Skip confirmation - true or false (default) | | `skype` | No | Skype ID | | `theme_id` | No | The GitLab theme for the user (see [the user preference docs](../user/profile/preferences.md#navigation-theme) for more information) | @@ -426,7 +434,7 @@ Parameters: | `email` | No | Email | | `extern_uid` | No | External UID | | `external` | No | Flags the user as external - true or false (default) | -| `extra_shared_runners_minutes_limit` | No | Extra pipeline minutes quota for this user **(STARTER)** | +| `extra_shared_runners_minutes_limit` | No | Extra pipeline minutes quota for this user (purchased in addition to the minutes included in the plan) **(STARTER)** | | `group_id_for_saml` | No | ID of group where SAML has been configured | | `id` | Yes | The ID of the user | | `linkedin` | No | LinkedIn | @@ -439,7 +447,7 @@ Parameters: | `projects_limit` | No | Limit projects each user can create | | `provider` | No | External provider name | | `public_email` | No | The public email of the user | -| `shared_runners_minutes_limit` | No | Pipeline minutes quota for this user **(STARTER)** | +| `shared_runners_minutes_limit` | No | Pipeline minutes quota for this user (included in plan). Can be `nil` (default; inherit system default), `0` (unlimited) or `> 0` **(STARTER)** | | `skip_reconfirmation` | No | Skip reconfirmation - true or false (default) | | `skype` | No | Skype ID | | `theme_id` | No | The GitLab theme for the user (see [the user preference docs](../user/profile/preferences.md#navigation-theme) for more information) | @@ -499,7 +507,8 @@ GET /user "avatar_url": "http://localhost:3000/uploads/user/avatar/1/index.jpg", "web_url": "http://localhost:3000/john_smith", "created_at": "2012-05-23T08:00:58Z", - "bio": null, + "bio": "", + "bio_html": "", "location": null, "public_email": "john@example.com", "skype": "", @@ -548,7 +557,8 @@ GET /user "web_url": "http://localhost:3000/john_smith", "created_at": "2012-05-23T08:00:58Z", "is_admin": false, - "bio": null, + "bio": "", + "bio_html": "", "location": null, "public_email": "john@example.com", "skype": "", @@ -799,6 +809,9 @@ Parameters: - `key` (required) - new SSH key - `expires_at` (optional) - The expiration date of the SSH key in ISO 8601 format (`YYYY-MM-DDTHH:MM:SSZ`) +NOTE: **Note:** +This also adds an audit event, as described in [audit instance events](../administration/audit_events.md#instance-events-premium-only). **(PREMIUM)** + ## Delete SSH key for current user Deletes key owned by currently authenticated user. @@ -1400,7 +1413,8 @@ Parameters: ### Get user activities (admin only) -NOTE: **Note:** This API endpoint is only available on 8.15 (EE) and 9.1 (CE) and above. +NOTE: **Note:** +This API endpoint is only available on 8.15 (EE) and 9.1 (CE) and above. Get the last activity date for all users, sorted from oldest to newest. diff --git a/doc/api/vulnerability_findings.md b/doc/api/vulnerability_findings.md index 7fbd58ea62c..e21d903e474 100644 --- a/doc/api/vulnerability_findings.md +++ b/doc/api/vulnerability_findings.md @@ -43,6 +43,7 @@ GET /projects/:id/vulnerability_findings?scope=all GET /projects/:id/vulnerability_findings?scope=dismissed GET /projects/:id/vulnerability_findings?severity=high GET /projects/:id/vulnerability_findings?confidence=unknown,experimental +GET /projects/:id/vulnerability_findings?scanner=bandit,find_sec_bugs GET /projects/:id/vulnerability_findings?pipeline_id=42 ``` @@ -56,6 +57,7 @@ Beginning with GitLab 12.9, the `undefined` severity and confidence level is no | `scope` | string | no | Returns vulnerability findings for the given scope: `all` or `dismissed`. Defaults to `dismissed`. | | `severity` | string array | no | Returns vulnerability findings belonging to specified severity level: `info`, `unknown`, `low`, `medium`, `high`, or `critical`. Defaults to all. | | `confidence` | string array | no | Returns vulnerability findings belonging to specified confidence level: `ignore`, `unknown`, `experimental`, `low`, `medium`, `high`, or `confirmed`. Defaults to all. | +| `scanner` | string array | no | Returns vulnerability findings detected by specified scanner. | `pipeline_id` | integer/string | no | Returns vulnerability findings belonging to specified pipeline. | ```shell diff --git a/doc/ci/README.md b/doc/ci/README.md index 150f160b762..43e66adbd35 100644 --- a/doc/ci/README.md +++ b/doc/ci/README.md @@ -16,9 +16,9 @@ through the [continuous methodologies](introduction/index.md#introduction-to-cic - Continuous Delivery (CD) - Continuous Deployment (CD) -NOTE: **Out-of-the-box management systems can decrease hours spent on maintaining toolchains by 10% or more.** -Watch our -["Mastering continuous software development"](https://about.gitlab.com/webcast/mastering-ci-cd/) +NOTE: **Note:** +Out-of-the-box management systems can decrease hours spent on maintaining toolchains by 10% or more. +Watch our ["Mastering continuous software development"](https://about.gitlab.com/webcast/mastering-ci-cd/) webcast to learn about continuous methods and how GitLab’s built-in CI can help you simplify and scale software development. ## Overview @@ -58,7 +58,7 @@ the following documents: - [How GitLab CI/CD works](introduction/index.md#how-gitlab-cicd-works). - [Fundamental pipeline architectures](pipelines/pipeline_architectures.md). - [GitLab CI/CD basic workflow](introduction/index.md#basic-cicd-workflow). -- [Step-by-step guide for writing `.gitlab-ci.yml` for the first time](../user/project/pages/getting_started_part_four.md). +- [Step-by-step guide for writing `.gitlab-ci.yml` for the first time](../user/project/pages/getting_started/pages_from_scratch.md). If you're migrating from another CI/CD tool, check out our handy references: @@ -126,9 +126,10 @@ Its feature set is listed on the table below according to DevOps stages. | [ChatOps](chatops/README.md) | Trigger CI jobs from chat, with results sent back to the channel. | |---+---| | **Verify** || -| [Browser Performance Testing](../user/project/merge_requests/browser_performance_testing.md) | Quickly determine the performance impact of pending code changes. | +| [Browser Performance Testing](../user/project/merge_requests/browser_performance_testing.md) | Quickly determine the browser performance impact of pending code changes. | +| [Load Performance Testing](../user/project/merge_requests/load_performance_testing.md) | Quickly determine the server performance impact of pending code changes. | | [CI services](services/README.md) | Link Docker containers with your base image.| -| [Code Quality](../user/project/merge_requests/code_quality.md) **(STARTER)** | Analyze your source code quality. | +| [Code Quality](../user/project/merge_requests/code_quality.md) | Analyze your source code quality. | | [GitLab CI/CD for external repositories](ci_cd_for_external_repos/index.md) **(PREMIUM)** | Get the benefits of GitLab CI/CD combined with repositories in GitHub and Bitbucket Cloud. | | [Interactive Web Terminals](interactive_web_terminal/index.md) **(CORE ONLY)** | Open an interactive web terminal to debug the running jobs. | | [JUnit tests](junit_test_reports.md) | Identify script failures directly on merge requests. | @@ -139,7 +140,7 @@ Its feature set is listed on the table below according to DevOps stages. | [Building Docker images](docker/using_docker_build.md) | Maintain Docker-based projects using GitLab CI/CD. | | [Canary Deployments](../user/project/canary_deployments.md) **(PREMIUM)** | Ship features to only a portion of your pods and let a percentage of your user base to visit the temporarily deployed feature. | | [Deploy Boards](../user/project/deploy_boards.md) **(PREMIUM)** | Check the current health and status of each CI/CD environment running on Kubernetes. | -| [Feature Flags](../user/project/operations/feature_flags.md) **(PREMIUM)** | Deploy your features behind Feature Flags. | +| [Feature Flags](../operations/feature_flags.md) **(PREMIUM)** | Deploy your features behind Feature Flags. | | [GitLab Pages](../user/project/pages/index.md) | Deploy static websites. | | [GitLab Releases](../user/project/releases/index.md) | Add release notes to Git tags. | | [Review Apps](review_apps/index.md) | Configure GitLab CI/CD to preview code changes. | diff --git a/doc/ci/caching/index.md b/doc/ci/caching/index.md index 392dd4fdeba..b6bd01ecf58 100644 --- a/doc/ci/caching/index.md +++ b/doc/ci/caching/index.md @@ -226,14 +226,14 @@ image: node:latest cache: key: ${CI_COMMIT_REF_SLUG} paths: - - .npm/ + - .npm/ before_script: - npm ci --cache .npm --prefer-offline test_async: script: - - node ./specs/start.js ./specs/async.spec.js + - node ./specs/start.js ./specs/async.spec.js ``` ### Caching PHP dependencies @@ -253,16 +253,16 @@ image: php:7.2 cache: key: ${CI_COMMIT_REF_SLUG} paths: - - vendor/ + - vendor/ before_script: -# Install and run Composer -- curl --show-error --silent https://getcomposer.org/installer | php -- php composer.phar install + # Install and run Composer + - curl --show-error --silent https://getcomposer.org/installer | php + - php composer.phar install test: script: - - vendor/bin/phpunit --configuration phpunit.xml --coverage-text --colors=never + - vendor/bin/phpunit --configuration phpunit.xml --coverage-text --colors=never ``` ### Caching Python dependencies @@ -301,9 +301,9 @@ before_script: test: script: - - python setup.py test - - pip install flake8 - - flake8 . + - python setup.py test + - pip install flake8 + - flake8 . ``` ### Caching Ruby dependencies @@ -330,7 +330,7 @@ before_script: rspec: script: - - rspec spec + - rspec spec ``` ### Caching Go dependencies @@ -354,7 +354,7 @@ test: image: golang:1.13 extends: .go-cache script: - - go test ./... -v -short + - go test ./... -v -short ``` ## Availability of the cache @@ -391,28 +391,28 @@ stages: ```yaml stages: -- build -- test + - build + - test before_script: -- echo "Hello" + - echo "Hello" job A: stage: build script: - - mkdir vendor/ - - echo "build" > vendor/hello.txt + - mkdir vendor/ + - echo "build" > vendor/hello.txt cache: key: build-cache paths: - - vendor/ + - vendor/ after_script: - - echo "World" + - echo "World" job B: stage: test script: - - cat vendor/hello.txt + - cat vendor/hello.txt cache: key: build-cache ``` @@ -483,8 +483,8 @@ cache when the pipeline is run for a second time. ```yaml stages: -- build -- test + - build + - test job A: stage: build @@ -492,7 +492,7 @@ job A: cache: key: same-key paths: - - public/ + - public/ job B: stage: test @@ -500,7 +500,7 @@ job B: cache: key: same-key paths: - - vendor/ + - vendor/ ``` 1. `job A` runs. @@ -520,8 +520,8 @@ will be different): ```yaml stages: -- build -- test + - build + - test job A: stage: build @@ -529,7 +529,7 @@ job A: cache: key: keyA paths: - - vendor/ + - vendor/ job B: stage: test @@ -537,7 +537,7 @@ job B: cache: key: keyB paths: - - vendor/ + - vendor/ ``` In that case, even if the `key` is different (no fear of overwriting), you diff --git a/doc/ci/chatops/README.md b/doc/ci/chatops/README.md index a8fea0836c1..ec0c41032ca 100644 --- a/doc/ci/chatops/README.md +++ b/doc/ci/chatops/README.md @@ -10,47 +10,75 @@ type: index, concepts, howto > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/4466) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 10.6. > - [Moved](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/24780) to [GitLab Core](https://about.gitlab.com/pricing/) in 11.9. -GitLab ChatOps provides a method to interact with CI/CD jobs through chat services like Slack. Many organizations' discussion, collaboration, and troubleshooting is taking place in chat services these days, and having a method to run CI/CD jobs with output posted back to the channel can significantly augment a team's workflow. +GitLab ChatOps provides a method to interact with CI/CD jobs through chat services +like Slack. Many organizations' discussion, collaboration, and troubleshooting takes +place in chat services. Having a method to run CI/CD jobs with output +posted back to the channel can significantly augment your team's workflow. -NOTE: **Note:** -ChatOps is currently in alpha with some important features missing, like access control. +## How GitLab ChatOps works -## How it works +GitLab ChatOps is built upon [GitLab CI/CD](../README.md) and +[Slack Slash Commands](../../user/project/integrations/slack_slash_commands.md). +ChatOps provides a `run` action for [slash commands](../../integration/slash_commands.md) +with the following arguments: -GitLab ChatOps is built upon two existing features: +- A `<job name>` to execute. +- The `<job arguments>`. -- [GitLab CI/CD](../README.md). -- [Slack Slash Commands](../../user/project/integrations/slack_slash_commands.md). +ChatOps passes the following [CI/CD variables](../variables/README.md#predefined-environment-variables) +to the job: -A new `run` action has been added to the [slash commands](../../integration/slash_commands.md), which takes two arguments: a `<job name>` to execute and the `<job arguments>`. When executed, ChatOps will look up the specified job name and attempt to match it to a corresponding job in [`.gitlab-ci.yml`](../yaml/README.md). If a matching job is found on `master`, a pipeline containing just that job is scheduled. Two additional [CI/CD variables](../variables/README.md#predefined-environment-variables) are passed to the job: `CHAT_INPUT` contains any additional arguments, and `CHAT_CHANNEL` is set to the name of channel the action was triggered in. +- `CHAT_INPUT` contains any additional arguments. +- `CHAT_CHANNEL` is set to the name of channel the action was triggered in. -After the job has finished, its output is sent back to Slack provided it has completed within 30 minutes. If a job takes more than 30 minutes to run it must use the Slack API to manually send data back to a channel. +When executed, ChatOps looks up the specified job name and attempts to match it +to a corresponding job in [`.gitlab-ci.yml`](../yaml/README.md). If a matching job +is found on `master`, a pipeline containing only that job is scheduled. After the +job completes: -[Developer access and above](../../user/permissions.md#project-members-permissions) is required to use the `run` command. If a job should not be able to be triggered from chat, it can be set to `except: [chat]`. +- If the job completes in *less than 30 minutes*, the ChatOps sends the job's output to Slack. +- If the job completes in *more than 30 minutes*, the job must use the + [Slack API](https://api.slack.com/) to send data to the channel. -## Creating a ChatOps CI job +To use the `run` command, you must have +[Developer access or above](../../user/permissions.md#project-members-permissions). +If a job shouldn't be able to be triggered from chat, you can set the job to `except: [chat]`. -Since ChatOps is built upon GitLab CI/CD, the job has all the same features and functions available. There a few best practices to consider however when creating ChatOps jobs: +## Best practices for ChatOps CI jobs -- It is strongly recommended to set `only: [chat]` so the job does not run as part of the standard CI pipeline. -- If the job is set to `when: manual`, the pipeline will be created however the job will wait to be started. -- It is important to keep in mind that there is limited support for access control. If the user who triggered the slash command is a developer in the project, the job will run. The job itself can utilize existing [CI/CD variables](../variables/README.md#predefined-environment-variables) like `GITLAB_USER_ID` to perform additional rights validation, however these variables can be [overridden](../variables/README.md#priority-of-environment-variables). +Since ChatOps is built upon GitLab CI/CD, the job has all the same features and +functions available. Consider these best practices when creating ChatOps jobs: + +- GitLab strongly recommends you set `only: [chat]` so the job does not run as part + of the standard CI pipeline. +- If the job is set to `when: manual`, ChatOps creates the pipeline, but the job waits to be started. +- ChatOps provides limited support for access control. If the user triggering the + slash command has [Developer access or above](../../user/permissions.md#project-members-permissions) + in the project, the job runs. The job itself can use existing + [CI/CD variables](../variables/README.md#predefined-environment-variables) like + `GITLAB_USER_ID` to perform additional rights validation, but + these variables can be [overridden](../variables/README.md#priority-of-environment-variables). ### Controlling the ChatOps reply -For jobs with a single command, its output is automatically sent back to the channel as a reply. For example the chat reply of the following job is simply `Hello World.` +The output for jobs with a single command is sent to the channel as a reply. For +example, the chat reply of the following job is `Hello World` in the channel: ```yaml hello-world: stage: chatops only: [chat] script: - - echo "Hello World." + - echo "Hello World" ``` -Jobs that contain multiple commands, or have a `before_script`, include additional content in the chat reply. In these cases both the commands and their output are included, with the commands wrapped in ANSI colors codes. +Jobs that contain multiple commands (or `before_script`) return additional +content in the chat reply. In these cases, both the commands and their output are +included, with the commands wrapped in ANSI color codes. -To selectively reply with the output of one command, its output must be bounded by the `chat_reply` section. For example, the following job will list the files in the current directory. +To selectively reply with the output of one command, its output must be bounded by +the `chat_reply` section. For example, the following job lists the files in the +current directory: ```yaml ls: @@ -61,20 +89,20 @@ ls: - echo -e "section_start:$( date +%s ):chat_reply\r\033[0K\n$( ls -la )\nsection_end:$( date +%s ):chat_reply\r\033[0K" ``` -## GitLab ChatOps Examples +## GitLab ChatOps examples -The GitLab.com team created a repository of [common ChatOps scripts they use to interact with our Production instance of GitLab](https://gitlab.com/gitlab-com/chatops). They are likely useful -to other administrators of GitLab instances and can serve as inspiration for ChatOps scripts you can write to interact with your own applications. +The GitLab.com team created a repository of [common ChatOps scripts](https://gitlab.com/gitlab-com/chatops) +they use to interact with our Production instance of GitLab. Administrators of +other GitLab instances may find them useful. They can serve as inspiration for ChatOps +scripts you can write to interact with your own applications. ## GitLab ChatOps icon -Say Hi to our ChatOps bot. +The [official GitLab ChatOps icon](img/gitlab-chatops-icon.png) is available for download. You can find and download the official GitLab ChatOps icon here.  -[Download bigger image](img/gitlab-chatops-icon.png) - <!-- ## Troubleshooting Include any troubleshooting steps that you can foresee. If you know beforehand what issues diff --git a/doc/ci/cloud_deployment/index.md b/doc/ci/cloud_deployment/index.md index 26eb99bd55a..29ce8bdf625 100644 --- a/doc/ci/cloud_deployment/index.md +++ b/doc/ci/cloud_deployment/index.md @@ -15,7 +15,7 @@ cloud provider more easily. ## AWS -GitLab provides Docker images to simplify working with AWS, and a template to make +GitLab provides Docker images that you can use to [run AWS commands from GitLab CI/CD](#run-aws-commands-from-gitlab-cicd), and a template to make it easier to [deploy to AWS](#deploy-your-application-to-the-aws-elastic-container-service-ecs). ### Run AWS commands from GitLab CI/CD @@ -87,6 +87,11 @@ GitLab provides a series of [CI templates that you can include in your project]( To automate deployments of your application to your [Amazon Elastic Container Service](https://aws.amazon.com/ecs/) (AWS ECS) cluster, you can `include` the `Deploy-ECS.gitlab-ci.yml` template in your `.gitlab-ci.yml` file. +GitLab also provides [Docker images](https://gitlab.com/gitlab-org/cloud-deploy/-/tree/master/aws) that can be used in your `gitlab-ci.yml` file to simplify working with AWS: + +- Use `registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest` to use AWS CLI commands. +- Use `registry.gitlab.com/gitlab-org/cloud-deploy/aws-ecs:latest` to deploy your application to AWS ECS. + Before getting started with this process, you need a cluster on AWS ECS, as well as related components, like an ECS service, ECS task definition, a database on AWS RDS, etc. [Read more about AWS ECS](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/Welcome.html). diff --git a/doc/ci/directed_acyclic_graph/index.md b/doc/ci/directed_acyclic_graph/index.md index fff0fda0ab4..b7dd74a0230 100644 --- a/doc/ci/directed_acyclic_graph/index.md +++ b/doc/ci/directed_acyclic_graph/index.md @@ -82,10 +82,10 @@ are certain use cases that you may need to work around. For more information: ## DAG Visualization > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/215517) in GitLab 13.1 as a [Beta feature](https://about.gitlab.com/handbook/product/#beta). -> - It's deployed behind a feature flag, disabled by default. +> - It was deployed behind a feature flag, disabled by default. +> - It became [enabled by default](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/36802) in 13.2. > - It's enabled on GitLab.com. -> - It's not recommended for production use. -> - For GitLab self-managed instances, GitLab administrators can opt to [enable it](#enable-or-disable-dag-visualization-core-only) +> - For GitLab self-managed instances, GitLab administrators can opt to [disable it](#enable-or-disable-dag-visualization-core-only). The DAG visualization makes it easier to visualize the relationships between dependent jobs in a DAG. This graph will display all the jobs in a pipeline that need or are needed by other jobs. Jobs with no relationships are not displayed in this view. @@ -97,15 +97,15 @@ Clicking a node will highlight all the job paths it depends on. ### Enable or disable DAG Visualization **(CORE ONLY)** -DAG Visualization is under development and requires more testing, but is being made available as a beta features so users can check its limitations and uses. +DAG Visualization is under development, but is being made available as a beta feature so users can check its limitations and uses. -It is deployed behind a feature flag that is **disabled by default**. +It is deployed behind a feature flag that is **enabled by default**. [GitLab administrators with access to the GitLab Rails console](../../administration/feature_flags.md) -can opt to enable it for your instance: +can opt to disable it for your instance: ```ruby # Instance-wide -Feature.enable(:dag_pipeline_tab) +Feature.disable(:dag_pipeline_tab) # or by project -Feature.enable(:dag_pipeline_tab, Project.find(<project id>)) +Feature.disable(:dag_pipeline_tab, Project.find(<project id>)) ``` diff --git a/doc/ci/docker/using_docker_build.md b/doc/ci/docker/using_docker_build.md index 65b9c03186b..4bed6d9e323 100644 --- a/doc/ci/docker/using_docker_build.md +++ b/doc/ci/docker/using_docker_build.md @@ -123,7 +123,7 @@ not without its own challenges: - By default, Docker 17.09 and higher uses `--storage-driver overlay2` which is the recommended storage driver. See [Using the overlayfs driver](#use-the-overlayfs-driver) for details. -- Since the `docker:19.03.11-dind` container and the Runner container don't share their +- Since the `docker:19.03.12-dind` container and the Runner container don't share their root filesystem, the job's working directory can be used as a mount point for child containers. For example, if you have files you want to share with a child container, you may create a subdirectory under `/builds/$CI_PROJECT_PATH` @@ -142,14 +142,14 @@ not without its own challenges: An example project using this approach can be found here: <https://gitlab.com/gitlab-examples/docker>. In the examples below, we are using Docker images tags to specify a -specific version, such as `docker:19.03.11`. If tags like `docker:stable` +specific version, such as `docker:19.03.12`. If tags like `docker:stable` are used, you have no control over what version is going to be used and this can lead to unpredictable behavior, especially when new versions are released. #### TLS enabled -NOTE: **Note** +NOTE: **Note:** Requires GitLab Runner 11.11 or later, but is not supported if GitLab Runner is installed using the [Helm chart](https://docs.gitlab.com/runner/install/kubernetes.html). See the @@ -158,7 +158,7 @@ issue](https://gitlab.com/gitlab-org/charts/gitlab-runner/-/issues/83) for details. The Docker daemon supports connection over TLS and it's done by default -for Docker 19.03.11 or higher. This is the **suggested** way to use the +for Docker 19.03.12 or higher. This is the **suggested** way to use the Docker-in-Docker service and [GitLab.com Shared Runners](../../user/gitlab_com/index.md#shared-runners) support this. @@ -174,13 +174,13 @@ support this. --registration-token REGISTRATION_TOKEN \ --executor docker \ --description "My Docker Runner" \ - --docker-image "docker:19.03.11" \ + --docker-image "docker:19.03.12" \ --docker-privileged \ --docker-volumes "/certs/client" ``` The above command will register a new Runner to use the special - `docker:19.03.11` image, which is provided by Docker. **Notice that it's + `docker:19.03.12` image, which is provided by Docker. **Notice that it's using the `privileged` mode to start the build and service containers.** If you want to use [Docker-in-Docker](https://www.docker.com/blog/docker-can-now-run-within-docker/) mode, you always have to use `privileged = true` in your Docker containers. @@ -199,7 +199,7 @@ support this. executor = "docker" [runners.docker] tls_verify = false - image = "docker:19.03.11" + image = "docker:19.03.12" privileged = true disable_cache = false volumes = ["/certs/client", "/cache"] @@ -209,10 +209,10 @@ support this. ``` 1. You can now use `docker` in the build script (note the inclusion of the - `docker:19.03.11-dind` service): + `docker:19.03.12-dind` service): ```yaml - image: docker:19.03.11 + image: docker:19.03.12 variables: # When using dind service, we need to instruct docker, to talk with @@ -237,7 +237,7 @@ support this. DOCKER_TLS_CERTDIR: "/certs" services: - - docker:19.03.11-dind + - docker:19.03.12-dind before_script: - docker info @@ -264,7 +264,7 @@ Assuming that the Runner `config.toml` is similar to: executor = "docker" [runners.docker] tls_verify = false - image = "docker:19.03.11" + image = "docker:19.03.12" privileged = true disable_cache = false volumes = ["/cache"] @@ -274,10 +274,10 @@ Assuming that the Runner `config.toml` is similar to: ``` You can now use `docker` in the build script (note the inclusion of the -`docker:19.03.11-dind` service): +`docker:19.03.12-dind` service): ```yaml -image: docker:19.03.11 +image: docker:19.03.12 variables: # When using dind service we need to instruct docker, to talk with the @@ -298,7 +298,7 @@ variables: DOCKER_TLS_CERTDIR: "" services: - - docker:19.03.11-dind + - docker:19.03.12-dind before_script: - docker info @@ -318,7 +318,7 @@ container so that Docker is available in the context of that image. NOTE: **Note:** If you bind the Docker socket [when using GitLab Runner 11.11 or newer](https://gitlab.com/gitlab-org/gitlab-runner/-/merge_requests/1261), -you can no longer use `docker:19.03.11-dind` as a service because volume bindings +you can no longer use `docker:19.03.12-dind` as a service because volume bindings are done to the services as well, making these incompatible. In order to do that, follow the steps: @@ -333,12 +333,12 @@ In order to do that, follow the steps: --registration-token REGISTRATION_TOKEN \ --executor docker \ --description "My Docker Runner" \ - --docker-image "docker:19.03.11" \ + --docker-image "docker:19.03.12" \ --docker-volumes /var/run/docker.sock:/var/run/docker.sock ``` The above command will register a new Runner to use the special - `docker:19.03.11` image which is provided by Docker. **Notice that it's using + `docker:19.03.12` image which is provided by Docker. **Notice that it's using the Docker daemon of the Runner itself, and any containers spawned by Docker commands will be siblings of the Runner rather than children of the Runner.** This may have complications and limitations that are unsuitable for your workflow. @@ -352,7 +352,7 @@ In order to do that, follow the steps: executor = "docker" [runners.docker] tls_verify = false - image = "docker:19.03.11" + image = "docker:19.03.12" privileged = false disable_cache = false volumes = ["/var/run/docker.sock:/var/run/docker.sock", "/cache"] @@ -361,11 +361,11 @@ In order to do that, follow the steps: ``` 1. You can now use `docker` in the build script (note that you don't need to - include the `docker:19.03.11-dind` service as when using the Docker in Docker + include the `docker:19.03.12-dind` service as when using the Docker in Docker executor): ```yaml - image: docker:19.03.11 + image: docker:19.03.12 before_script: - docker info @@ -419,10 +419,10 @@ any image that's used with the `--cache-from` argument must first be pulled Here's a `.gitlab-ci.yml` file showing how Docker caching can be used: ```yaml -image: docker:19.03.11 +image: docker:19.03.12 services: - - docker:19.03.11-dind + - docker:19.03.12-dind variables: # Use TLS https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#tls-enabled diff --git a/doc/ci/docker/using_docker_images.md b/doc/ci/docker/using_docker_images.md index 2448bb536ab..735cf35584f 100644 --- a/doc/ci/docker/using_docker_images.md +++ b/doc/ci/docker/using_docker_images.md @@ -149,14 +149,14 @@ the job will fail: ```yaml job: services: - - php:7 - - node:latest - - golang:1.10 + - php:7 + - node:latest + - golang:1.10 image: alpine:3.7 script: - - php -v - - node -v - - go version + - php -v + - node -v + - go version ``` If you need to have `php`, `node` and `go` available for your script, you should @@ -176,7 +176,7 @@ You can then use for example the [tutum/wordpress](https://hub.docker.com/r/tutu ```yaml services: -- tutum/wordpress:latest + - tutum/wordpress:latest ``` If you don't [specify a service alias](#available-settings-for-services), @@ -219,7 +219,7 @@ default: test: script: - - bundle exec rake spec + - bundle exec rake spec ``` The image name must be in one of the following formats: @@ -238,16 +238,16 @@ default: test:2.6: image: ruby:2.6 services: - - postgres:11.7 + - postgres:11.7 script: - - bundle exec rake spec + - bundle exec rake spec test:2.7: image: ruby:2.7 services: - - postgres:12.2 + - postgres:12.2 script: - - bundle exec rake spec + - bundle exec rake spec ``` Or you can pass some [extended configuration options](#extended-docker-configuration-options) @@ -260,17 +260,17 @@ default: entrypoint: ["/bin/bash"] services: - - name: my-postgres:11.7 - alias: db-postgres - entrypoint: ["/usr/local/bin/db-postgres"] - command: ["start"] + - name: my-postgres:11.7 + alias: db-postgres + entrypoint: ["/usr/local/bin/db-postgres"] + command: ["start"] before_script: - - bundle install + - bundle install test: script: - - bundle exec rake spec + - bundle exec rake spec ``` ## Passing environment variables to services @@ -292,21 +292,21 @@ variables: POSTGRES_INITDB_ARGS: "--encoding=UTF8 --data-checksums" services: -- name: postgres:11.7 - alias: db - entrypoint: ["docker-entrypoint.sh"] - command: ["postgres"] + - name: postgres:11.7 + alias: db + entrypoint: ["docker-entrypoint.sh"] + command: ["postgres"] image: name: ruby:2.6 entrypoint: ["/bin/bash"] before_script: -- bundle install + - bundle install test: script: - - bundle exec rake spec + - bundle exec rake spec ``` ## Extended Docker configuration options @@ -330,8 +330,8 @@ For example, the following two definitions are equal: image: "registry.example.com/my/image:latest" services: - - postgresql:9.4 - - redis:latest + - postgresql:9.4 + - redis:latest ``` 1. Using a map as an option to `image` and `services`. The use of `image:name` is @@ -342,8 +342,8 @@ For example, the following two definitions are equal: name: "registry.example.com/my/image:latest" services: - - name: postgresql:9.4 - - name: redis:latest + - name: postgresql:9.4 + - name: redis:latest ``` ### Available settings for `image` @@ -378,8 +378,8 @@ would not work properly: ```yaml services: -- mysql:latest -- mysql:latest + - mysql:latest + - mysql:latest ``` The Runner would start two containers using the `mysql:latest` image, but both @@ -392,10 +392,10 @@ look like: ```yaml services: -- name: mysql:latest - alias: mysql-1 -- name: mysql:latest - alias: mysql-2 + - name: mysql:latest + alias: mysql-1 + - name: mysql:latest + alias: mysql-2 ``` The Runner will still start two containers using the `mysql:latest` image, @@ -427,7 +427,7 @@ CMD ["/usr/bin/super-sql", "run"] # .gitlab-ci.yml services: -- my-super-sql:latest + - my-super-sql:latest ``` After the new extended Docker configuration options, you can now simply @@ -437,8 +437,8 @@ set a `command` in `.gitlab-ci.yml`, like: # .gitlab-ci.yml services: -- name: super/sql:latest - command: ["/usr/bin/super-sql", "run"] + - name: super/sql:latest + command: ["/usr/bin/super-sql", "run"] ``` As you can see, the syntax of `command` is similar to [Dockerfile's `CMD`](https://docs.docker.com/engine/reference/builder/#cmd). @@ -545,9 +545,8 @@ runtime. support for using private registries, which required manual configuration of credentials on runner's host. We recommend to upgrade your Runner to at least version **1.8** if you want to use private registries. -- Not available for [Kubernetes executor](https://docs.gitlab.com/runner/executors/kubernetes.html), - follow <https://gitlab.com/gitlab-org/gitlab-runner/-/issues/2673> for - details. +- Available for [Kubernetes executor](https://docs.gitlab.com/runner/executors/kubernetes.html) + in GitLab Runner 13.1 and later. ### Using statically-defined credentials @@ -601,6 +600,7 @@ There are two ways to determine the value of `DOCKER_AUTH_CONFIG`: Open a terminal and execute the following command: ```shell + # Note the use of "-n" - it prevents encoding a newline in the password. echo -n "my_username:my_password" | base64 # Example output to copy @@ -681,11 +681,13 @@ To add `DOCKER_AUTH_CONFIG` to a Runner: 1. Restart the Runner service. -NOTE: **Note:** The double quotes included in the `DOCKER_AUTH_CONFIG` +NOTE: **Note:** +The double quotes included in the `DOCKER_AUTH_CONFIG` data must be escaped with backslashes. This prevents them from being interpreted as TOML. -NOTE: **Note:** The `environment` option is a list. So your Runner may +NOTE: **Note:** +The `environment` option is a list. So your Runner may have existing entries and you should add this to the list, not replace it. @@ -715,7 +717,8 @@ To configure credentials store, follow these steps: `${GITLAB_RUNNER_HOME}/.docker/config.json`. GitLab Runner will read this configuration file and will use the needed helper for this specific repository. -NOTE: **Note:** `credsStore` is used to access ALL the registries. +NOTE: **Note:** +`credsStore` is used to access ALL the registries. If you will want to use both images from private registry and public images from DockerHub, pulling from DockerHub will fail, because Docker daemon will try to use the same credentials for **ALL** the registries. diff --git a/doc/ci/docker/using_kaniko.md b/doc/ci/docker/using_kaniko.md index d53430400ec..1580080ac6e 100644 --- a/doc/ci/docker/using_kaniko.md +++ b/doc/ci/docker/using_kaniko.md @@ -90,7 +90,7 @@ store: - | echo "-----BEGIN CERTIFICATE----- ... - -----END CERTIFICATE-----" >> /kaniko/ssl/certs/ca-certificates.crt + -----END CERTIFICATE-----" >> /kaniko/ssl/certs/additional-ca-cert-bundle.crt ``` ## Video walkthrough of a working example diff --git a/doc/ci/environments/deployment_safety.md b/doc/ci/environments/deployment_safety.md index 055baa78743..a4ff164a5ba 100644 --- a/doc/ci/environments/deployment_safety.md +++ b/doc/ci/environments/deployment_safety.md @@ -7,7 +7,7 @@ that help maintain deployment security and stability. You can: - [Restrict write-access to a critical environment](#restrict-write-access-to-a-critical-environment) -- [Restrict deployments for a particular period](#restrict-deployments-for-a-particular-period) +- [Prevent deployments during deploy freeze windows](#prevent-deployments-during-deploy-freeze-windows) If you are using a continuous deployment workflow and want to ensure that concurrent deployments to the same environment do not happen, you should enable the following options: @@ -77,10 +77,10 @@ The improved pipeline flow **after** enabling Skip outdated deployment jobs: 1. The `deploy` job in Pipeline-B finishes first, and deploys the newer code. 1. The `deploy` job in Pipeline-A is automatically cancelled, so that it doesn't overwrite the deployment from the newer pipeline. -## Restrict deployments for a particular period +## Prevent deployments during deploy freeze windows If you want to prevent deployments for a particular period, for example during a planned -vacation period when most employees are out, you can set up a [Deploy Freeze](../../user/project/releases/index.md#set-a-deploy-freeze). +vacation period when most employees are out, you can set up a [Deploy Freeze](../../user/project/releases/index.md#prevent-unintentional-releases-by-setting-a-deploy-freeze). During a deploy freeze period, no deployment can be executed. This is helpful to ensure that deployments do not happen unexpectedly. diff --git a/doc/ci/environments/index.md b/doc/ci/environments/index.md index 84480b836f8..0273ab290a6 100644 --- a/doc/ci/environments/index.md +++ b/doc/ci/environments/index.md @@ -94,7 +94,7 @@ deploy_staging: name: staging url: https://staging.example.com only: - - master + - master ``` We have defined three [stages](../yaml/README.md#stages): @@ -124,7 +124,7 @@ The environment `name` and `url` is exposed in various places within GitLab. Each time a job that has an environment specified succeeds, a deployment is recorded, along with the Git SHA, and environment name. -CAUTION: **Caution**: +CAUTION: **Caution:** Some characters are not allowed in environment names. Use only letters, numbers, spaces, and `-`, `_`, `/`, `{`, `}`, or `.`. Also, it must not start nor end with `/`. @@ -173,8 +173,8 @@ If you want to use the environment URL in GitLab, you would have to update it ma To address this problem, you can configure a deployment job to report back a set of variables, including the URL that was dynamically-generated by the external service. -GitLab supports [dotenv](https://github.com/bkeepers/dotenv) file as the format, -and expands the `environment:url` value with variables defined in the dotenv file. +GitLab supports the [dotenv (`.env`)](https://github.com/bkeepers/dotenv) file format, +and expands the `environment:url` value with variables defined in the `.env` file. To use this feature, specify the [`artifacts:reports:dotenv`](../pipelines/job_artifacts.md#artifactsreportsdotenv) keyword in `.gitlab-ci.yml`. @@ -213,14 +213,13 @@ stop_review: As soon as the `review` job finishes, GitLab updates the `review/your-branch-name` environment's URL. -It parses the report artifact `deploy.env`, registers a list of variables as runtime-created, +It parses the `deploy.env` report artifact, registers a list of variables as runtime-created, uses it for expanding `environment:url: $DYNAMIC_ENVIRONMENT_URL` and sets it to the environment URL. You can also specify a static part of the URL at `environment:url:`, such as `https://$DYNAMIC_ENVIRONMENT_URL`. If the value of `DYNAMIC_ENVIRONMENT_URL` is -`123.awesome.com`, the final result will be `https://123.awesome.com`. +`example.com`, the final result will be `https://example.com`. -The assigned URL for the `review/your-branch-name` environment is visible in the UI. -[See where the environment URL is displayed](#using-the-environment-url). +The assigned URL for the `review/your-branch-name` environment is [visible in the UI](#using-the-environment-url). > **Notes:** > @@ -260,7 +259,7 @@ deploy_staging: name: staging url: https://staging.example.com only: - - master + - master deploy_prod: stage: deploy @@ -271,7 +270,7 @@ deploy_prod: url: https://example.com when: manual only: - - master + - master ``` The `when: manual` action: @@ -306,11 +305,6 @@ declaring their names dynamically in `.gitlab-ci.yml`. Dynamic environments are a fundamental part of [Review apps](../review_apps/index.md). -### Configuring incremental rollouts - -Learn how to release production changes to only a portion of your Kubernetes pods with -[incremental rollouts](../environments/incremental_rollouts.md). - #### Allowed variables The `name` and `url` parameters for dynamic environments can use most available CI/CD variables, @@ -408,7 +402,7 @@ deploy: kubernetes: namespace: production only: - - master + - master ``` When deploying to a Kubernetes cluster using GitLab's Kubernetes integration, @@ -423,6 +417,11 @@ that are [managed by GitLab](../../user/project/clusters/index.md#gitlab-managed To follow progress on support for GitLab-managed clusters, see the [relevant issue](https://gitlab.com/gitlab-org/gitlab/-/issues/38054). +#### Configuring incremental rollouts + +Learn how to release production changes to only a portion of your Kubernetes pods with +[incremental rollouts](../environments/incremental_rollouts.md). + ### Deployment safety Deployment jobs can be more sensitive than other jobs in a pipeline, @@ -432,7 +431,7 @@ in GitLab that helps maintain deployment security and stability. - [Restrict write-access to a critical environment](deployment_safety.md#restrict-write-access-to-a-critical-environment) - [Limit the job-concurrency for deployment jobs](deployment_safety.md#ensure-only-one-deployment-job-runs-at-a-time) - [Skip outdated deployment jobs](deployment_safety.md#skip-outdated-deployment-jobs) -- [Restrict deployments for a particular period](deployment_safety.md#restrict-deployments-for-a-particular-period) +- [Prevent deployments during deploy freeze windows](deployment_safety.md#prevent-deployments-during-deploy-freeze-windows) ### Complete example @@ -484,7 +483,7 @@ deploy_staging: name: staging url: https://staging.example.com only: - - master + - master deploy_prod: stage: deploy @@ -495,7 +494,7 @@ deploy_prod: url: https://example.com when: manual only: - - master + - master ``` A more realistic example would also include copying files to a location where a @@ -740,6 +739,12 @@ To enable this feature, you need to specify the [`environment:auto_stop_in`](../ You can specify a human-friendly date as the value, such as `1 hour and 30 minutes` or `1 day`. `auto_stop_in` uses the same format of [`artifacts:expire_in` docs](../yaml/README.md#artifactsexpire_in). +NOTE: **Note:** +Due to the resource limitation, a background worker for stopping environments only +runs once every hour. This means environments will not be stopped at the exact +timestamp as the specified period, but will be stopped when the hourly cron worker +detects expired environments. + ##### Auto-stop example In the following example, there is a basic review app setup that creates a new environment @@ -779,15 +784,9 @@ and the environment will be active until it's stopped manually.  -NOTE: **NOTE** -Due to the resource limitation, a background worker for stopping environments only -runs once every hour. This means environments will not be stopped at the exact -timestamp as the specified period, but will be stopped when the hourly cron worker -detects expired environments. - #### Delete a stopped environment -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/22629) in GitLab 12.9. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/20620) in GitLab 12.10. You can delete [stopped environments](#stopping-an-environment) in one of two ways: through the GitLab UI or through the API. @@ -812,6 +811,31 @@ stopped environment: Environments can also be deleted by using the [Environments API](../../api/environments.md#delete-an-environment). +### Prepare an environment + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/208655) in GitLab 13.2. + +By default, GitLab creates a [deployment](#viewing-deployment-history) every time a +build with the specified environment runs. Newer deployments can also +[cancel older ones](deployment_safety.md#skip-outdated-deployment-jobs). + +You may want to specify an environment keyword to +[protect builds from unauthorized access](protected_environments.md), or to get +access to [scoped variables](#scoping-environments-with-specs). In these cases, +you can use the `action: prepare` keyword to ensure deployments won't be created, +and no builds would be canceled: + +```yaml +build: + stage: build + script: + - echo "Building the app" + environment: + name: staging + action: prepare + url: https://staging.example.com +``` + ### Grouping similar environments > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/7015) in GitLab 8.14. @@ -868,7 +892,7 @@ versions of the app, all without leaving GitLab. #### Embedding metrics in GitLab Flavored Markdown -Metric charts can be embedded within GitLab Flavored Markdown. See [Embedding Metrics within GitLab Flavored Markdown](../../user/project/integrations/prometheus.md#embedding-metric-charts-within-gitlab-flavored-markdown) for more details. +Metric charts can be embedded within GitLab Flavored Markdown. See [Embedding Metrics within GitLab Flavored Markdown](../../operations/metrics/embed.md) for more details. ### Web terminals diff --git a/doc/ci/environments/protected_environments.md b/doc/ci/environments/protected_environments.md index 3a44fcfb182..b6141ddc7ac 100644 --- a/doc/ci/environments/protected_environments.md +++ b/doc/ci/environments/protected_environments.md @@ -22,7 +22,7 @@ specific environments are "protected" to prevent unauthorized people from affect By default, a protected environment does one thing: it ensures that only people with the right privileges can deploy to it, thus keeping it safe. -NOTE: **Note**: +NOTE: **Note:** A GitLab admin is always allowed to use environments, even if they are protected. To protect, update, or unprotect an environment, you need to have at least diff --git a/doc/ci/examples/README.md b/doc/ci/examples/README.md index c54e4b96332..37ccef10567 100644 --- a/doc/ci/examples/README.md +++ b/doc/ci/examples/README.md @@ -25,6 +25,7 @@ The following table lists examples with step-by-step tutorials that are containe | Use case | Resource | |:------------------------------|:---------------------------------------------------------------------------------------------------------------------------| | Browser performance testing | [Browser Performance Testing with the Sitespeed.io container](../../user/project/merge_requests/browser_performance_testing.md). | +| Load performance testing | [Load Performance Testing with the k6 container](../../user/project/merge_requests/load_performance_testing.md). | | Clojure | [Test a Clojure application with GitLab CI/CD](test-clojure-application.md). | | Deployment with Dpl | [Using `dpl` as deployment tool](deployment/README.md). | | Elixir | [Testing a Phoenix application with GitLab CI/CD](test_phoenix_app_with_gitlab_ci_cd/index.md). | @@ -46,8 +47,6 @@ The following table lists examples with step-by-step tutorials that are containe Contributions are welcome! You can help your favorite programming language users and GitLab by sending a merge request with a guide for that language. -You may want to apply for the [GitLab Community Writers Program](https://about.gitlab.com/community/writers/) -to get paid for writing complete articles for GitLab. ## CI/CD templates @@ -63,6 +62,7 @@ choose one of these templates: - [C++ (`C++.gitlab-ci.yml`)](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/C++.gitlab-ci.yml) - [Chef (`Chef.gitlab-ci.yml`)](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Chef.gitlab-ci.yml) - [Clojure (`Clojure.gitlab-ci.yml`)](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml) +- [Composer `Composer.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Composer.gitlab-ci.yml) - [Crystal (`Crystal.gitlab-ci.yml`)](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml) - [Django (`Django.gitlab-ci.yml`)](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Django.gitlab-ci.yml) - [Docker (`Docker.gitlab-ci.yml`)](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Docker.gitlab-ci.yml) @@ -78,6 +78,7 @@ choose one of these templates: - [LaTeX (`LaTeX.gitlab-ci.yml`)](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/LaTeX.gitlab-ci.yml) - [Maven (`Maven.gitlab-ci.yml`)](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Maven.gitlab-ci.yml) - [Mono (`Mono.gitlab-ci.yml`)](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Mono.gitlab-ci.yml) +- [NPM (`npm.gitlab-ci.yml`)](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/npm.gitlab-ci.yml) - [Node.js (`Nodejs.gitlab-ci.yml`)](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml) - [OpenShift (`OpenShift.gitlab-ci.yml`)](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml) - [Packer (`Packer.gitlab-ci.yml`)](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Packer.gitlab-ci.yml) diff --git a/doc/ci/examples/authenticating-with-hashicorp-vault/index.md b/doc/ci/examples/authenticating-with-hashicorp-vault/index.md index 3b8be54ae59..9a87786ec70 100644 --- a/doc/ci/examples/authenticating-with-hashicorp-vault/index.md +++ b/doc/ci/examples/authenticating-with-hashicorp-vault/index.md @@ -50,7 +50,7 @@ The JWT's payload looks like this: } ``` -The JWT is encoded by using RS256 and signed with your GitLab instance's OpenID Connect private key. The expire time for the token will be set to job's timeout, if specifed, or 5 minutes if it is not. The key used to sign this token may change without any notice. In such case retrying the job will generate new JWT using the current signing key. +The JWT is encoded by using RS256 and signed with your GitLab instance's OpenID Connect private key. The expire time for the token will be set to job's timeout, if specified, or 5 minutes if it is not. The key used to sign this token may change without any notice. In such case retrying the job will generate new JWT using the current signing key. You can use this JWT and your instance's JWKS endpoint (`https://gitlab.example.com/-/jwks`) to authenticate with a Vault server that is configured to allow the JWT Authentication method for authentication. @@ -60,7 +60,7 @@ To communicate with Vault, you can use either its CLI client or perform API requ ## Example -CAUTION: **Caution**: +CAUTION: **Caution:** JWTs are credentials, which can grant access to resources. Be careful where you paste them! Let's say you have the passwords for your staging and production databases stored in a Vault server that is running on `http://vault.example.com:8200`. Your staging password is `pa$$w0rd` and your production password is `real-pa$$w0rd`. @@ -156,7 +156,7 @@ Combined with GitLab's [protected branches](../../../user/project/protected_bran For the full list of options, see Vault's [Create Role documentation](https://www.vaultproject.io/api/auth/jwt#create-role). -CAUTION: **Caution**: +CAUTION: **Caution:** Always restrict your roles to project or namespace by using one of the provided claims (e.g. `project_id` or `namespace_id`). Otherwise any JWT generated by this instance may be allowed to authenticate using this role. Now, configure the JWT Authentication method: diff --git a/doc/ci/examples/deploy_spring_boot_to_cloud_foundry/index.md b/doc/ci/examples/deploy_spring_boot_to_cloud_foundry/index.md index 87cee1820bc..e0d4f3f2402 100644 --- a/doc/ci/examples/deploy_spring_boot_to_cloud_foundry/index.md +++ b/doc/ci/examples/deploy_spring_boot_to_cloud_foundry/index.md @@ -61,10 +61,10 @@ content: ```yaml --- applications: -- name: gitlab-hello-world - random-route: true - memory: 1G - path: target/demo-0.0.1-SNAPSHOT.jar + - name: gitlab-hello-world + random-route: true + memory: 1G + path: target/demo-0.0.1-SNAPSHOT.jar ``` ## Configure GitLab CI/CD to deploy your application @@ -96,11 +96,11 @@ build: production: stage: deploy script: - - curl --location "https://cli.run.pivotal.io/stable?release=linux64-binary&source=github" | tar zx - - ./cf login -u $CF_USERNAME -p $CF_PASSWORD -a api.run.pivotal.io - - ./cf push + - curl --location "https://cli.run.pivotal.io/stable?release=linux64-binary&source=github" | tar zx + - ./cf login -u $CF_USERNAME -p $CF_PASSWORD -a api.run.pivotal.io + - ./cf push only: - - master + - master ``` We've used the `java:8` [Docker diff --git a/doc/ci/examples/deployment/README.md b/doc/ci/examples/deployment/README.md index ec02fb6dd43..3192b365c83 100644 --- a/doc/ci/examples/deployment/README.md +++ b/doc/ci/examples/deployment/README.md @@ -45,8 +45,8 @@ All possible parameters can be found here: <https://github.com/travis-ci/dpl#her staging: stage: deploy script: - - gem install dpl - - dpl --provider=heroku --app=my-app-staging --api-key=$HEROKU_STAGING_API_KEY + - gem install dpl + - dpl --provider=heroku --app=my-app-staging --api-key=$HEROKU_STAGING_API_KEY ``` In the above example we use Dpl to deploy `my-app-staging` to Heroku server with API key stored in `HEROKU_STAGING_API_KEY` secure variable. @@ -64,12 +64,12 @@ You will have to install it: staging: stage: deploy script: - - apt-get update -yq - - apt-get install -y ruby-dev - - gem install dpl - - dpl --provider=heroku --app=my-app-staging --api-key=$HEROKU_STAGING_API_KEY + - apt-get update -yq + - apt-get install -y ruby-dev + - gem install dpl + - dpl --provider=heroku --app=my-app-staging --api-key=$HEROKU_STAGING_API_KEY only: - - master + - master ``` The first line `apt-get update -yq` updates the list of available packages, @@ -89,18 +89,18 @@ The final `.gitlab-ci.yml` for that setup would look like this: staging: stage: deploy script: - - gem install dpl - - dpl --provider=heroku --app=my-app-staging --api-key=$HEROKU_STAGING_API_KEY + - gem install dpl + - dpl --provider=heroku --app=my-app-staging --api-key=$HEROKU_STAGING_API_KEY only: - - master + - master production: stage: deploy script: - - gem install dpl - - dpl --provider=heroku --app=my-app-production --api-key=$HEROKU_PRODUCTION_API_KEY + - gem install dpl + - dpl --provider=heroku --app=my-app-production --api-key=$HEROKU_PRODUCTION_API_KEY only: - - tags + - tags ``` We created two deploy jobs that are executed on different events: diff --git a/doc/ci/examples/deployment/composer-npm-deploy.md b/doc/ci/examples/deployment/composer-npm-deploy.md index cea6f26181f..067d92c2275 100644 --- a/doc/ci/examples/deployment/composer-npm-deploy.md +++ b/doc/ci/examples/deployment/composer-npm-deploy.md @@ -150,7 +150,7 @@ before_script: stage_deploy: artifacts: paths: - - build/ + - build/ only: - dev script: diff --git a/doc/ci/examples/devops_and_game_dev_with_gitlab_ci_cd/index.md b/doc/ci/examples/devops_and_game_dev_with_gitlab_ci_cd/index.md index 51d9f169939..35a35d97a4b 100644 --- a/doc/ci/examples/devops_and_game_dev_with_gitlab_ci_cd/index.md +++ b/doc/ci/examples/devops_and_game_dev_with_gitlab_ci_cd/index.md @@ -274,19 +274,19 @@ just pack them up in the cache. Here is the full `build` job: ```yaml build: - stage: build - script: - - npm i gulp -g - - npm i - - gulp - - gulp build-test - cache: - policy: push - paths: - - node_modules - artifacts: - paths: - - built + stage: build + script: + - npm i gulp -g + - npm i + - gulp + - gulp build-test + cache: + policy: push + paths: + - node_modules + artifacts: + paths: + - built ``` ### Test your game with GitLab CI/CD @@ -301,18 +301,18 @@ Following the YAML structure, the `test` job should look like this: ```yaml test: - stage: test - script: - - npm i gulp -g - - npm i - - gulp run-test - cache: - policy: push - paths: - - node_modules/ - artifacts: - paths: - - built/ + stage: test + script: + - npm i gulp -g + - npm i + - gulp run-test + cache: + policy: push + paths: + - node_modules/ + artifacts: + paths: + - built/ ``` We have added unit tests for a `Weapon` class that shoots on a specified interval. @@ -325,33 +325,33 @@ Our entire `.gitlab-ci.yml` file should now look like this: image: node:10 build: - stage: build - script: - - npm i gulp -g - - npm i - - gulp - - gulp build-test - cache: - policy: push - paths: - - node_modules/ - artifacts: - paths: - - built/ + stage: build + script: + - npm i gulp -g + - npm i + - gulp + - gulp build-test + cache: + policy: push + paths: + - node_modules/ + artifacts: + paths: + - built/ test: - stage: test - script: - - npm i gulp -g - - npm i - - gulp run-test - cache: - policy: pull - paths: - - node_modules/ - artifacts: - paths: - - built/ + stage: test + script: + - npm i gulp -g + - npm i + - gulp run-test + cache: + policy: pull + paths: + - node_modules/ + artifacts: + paths: + - built/ ``` ### Run your CI/CD pipeline @@ -445,18 +445,18 @@ trigger the `deploy` job of our pipeline. Put these together to get the followin ```yaml deploy: - stage: deploy - variables: - AWS_ACCESS_KEY_ID: "$AWS_KEY_ID" - AWS_SECRET_ACCESS_KEY: "$AWS_KEY_SECRET" - script: - - apt-get update - - apt-get install -y python3-dev python3-pip - - easy_install3 -U pip - - pip3 install --upgrade awscli - - aws s3 sync ./built s3://gitlab-game-demo --region "us-east-1" --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers --cache-control "no-cache, no-store, must-revalidate" --delete - only: - - master + stage: deploy + variables: + AWS_ACCESS_KEY_ID: "$AWS_KEY_ID" + AWS_SECRET_ACCESS_KEY: "$AWS_KEY_SECRET" + script: + - apt-get update + - apt-get install -y python3-dev python3-pip + - easy_install3 -U pip + - pip3 install --upgrade awscli + - aws s3 sync ./built s3://gitlab-game-demo --region "us-east-1" --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers --cache-control "no-cache, no-store, must-revalidate" --delete + only: + - master ``` Be sure to update the region and S3 URL in that last script command to fit your setup. @@ -466,46 +466,46 @@ Our final configuration file `.gitlab-ci.yml` looks like: image: node:10 build: - stage: build - script: - - npm i gulp -g - - npm i - - gulp - - gulp build-test - cache: - policy: push - paths: - - node_modules/ - artifacts: - paths: - - built/ + stage: build + script: + - npm i gulp -g + - npm i + - gulp + - gulp build-test + cache: + policy: push + paths: + - node_modules/ + artifacts: + paths: + - built/ test: - stage: test - script: - - npm i gulp -g - - gulp run-test - cache: - policy: pull - paths: - - node_modules/ - artifacts: - paths: - - built/ + stage: test + script: + - npm i gulp -g + - gulp run-test + cache: + policy: pull + paths: + - node_modules/ + artifacts: + paths: + - built/ deploy: - stage: deploy - variables: - AWS_ACCESS_KEY_ID: "$AWS_KEY_ID" - AWS_SECRET_ACCESS_KEY: "$AWS_KEY_SECRET" - script: - - apt-get update - - apt-get install -y python3-dev python3-pip - - easy_install3 -U pip - - pip3 install --upgrade awscli - - aws s3 sync ./built s3://gitlab-game-demo --region "us-east-1" --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers --cache-control "no-cache, no-store, must-revalidate" --delete - only: - - master + stage: deploy + variables: + AWS_ACCESS_KEY_ID: "$AWS_KEY_ID" + AWS_SECRET_ACCESS_KEY: "$AWS_KEY_SECRET" + script: + - apt-get update + - apt-get install -y python3-dev python3-pip + - easy_install3 -U pip + - pip3 install --upgrade awscli + - aws s3 sync ./built s3://gitlab-game-demo --region "us-east-1" --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers --cache-control "no-cache, no-store, must-revalidate" --delete + only: + - master ``` ## Conclusion diff --git a/doc/ci/examples/laravel_with_gitlab_and_envoy/index.md b/doc/ci/examples/laravel_with_gitlab_and_envoy/index.md index 5b442436baf..ee0938b4d4c 100644 --- a/doc/ci/examples/laravel_with_gitlab_and_envoy/index.md +++ b/doc/ci/examples/laravel_with_gitlab_and_envoy/index.md @@ -448,7 +448,7 @@ On your GitLab project repository navigate to the **Registry** tab.  -You may need to [enable Container Registry](../../../user/packages/container_registry/index.md#enable-the-container-registry-for-your-project) to your project to see this tab. You'll find it under your project's **Settings > General > Permissions**. +You may need to [enable Container Registry](../../../user/packages/container_registry/index.md#enable-the-container-registry-for-your-project) to your project to see this tab. You'll find it under your project's **Settings > General > Visibility, project features, permissions**. To start using Container Registry on our machine, we first need to login to the GitLab registry using our GitLab username and password: diff --git a/doc/ci/examples/php.md b/doc/ci/examples/php.md index e7768868c15..cc62e9316f2 100644 --- a/doc/ci/examples/php.md +++ b/doc/ci/examples/php.md @@ -76,7 +76,7 @@ environment, let's add it in `.gitlab-ci.yml`: ... before_script: -- bash ci/docker_install.sh > /dev/null + - bash ci/docker_install.sh > /dev/null ... ``` @@ -88,7 +88,7 @@ Last step, run the actual tests using `phpunit`: test:app: script: - - phpunit --configuration phpunit_myapp.xml + - phpunit --configuration phpunit_myapp.xml ... ``` @@ -104,11 +104,11 @@ image: php:5.6 before_script: # Install dependencies -- bash ci/docker_install.sh > /dev/null + - bash ci/docker_install.sh > /dev/null test:app: script: - - phpunit --configuration phpunit_myapp.xml + - phpunit --configuration phpunit_myapp.xml ``` ### Test against different PHP versions in Docker builds @@ -119,19 +119,19 @@ with a different Docker image version and the runner will do the rest: ```yaml before_script: # Install dependencies -- bash ci/docker_install.sh > /dev/null + - bash ci/docker_install.sh > /dev/null # We test PHP5.6 test:5.6: image: php:5.6 script: - - phpunit --configuration phpunit_myapp.xml + - phpunit --configuration phpunit_myapp.xml # We test PHP7.0 (good luck with that) test:7.0: image: php:7.0 script: - - phpunit --configuration phpunit_myapp.xml + - phpunit --configuration phpunit_myapp.xml ``` ### Custom PHP configuration in Docker builds @@ -142,7 +142,7 @@ add a `before_script` action: ```yaml before_script: -- cp my_php.ini /usr/local/etc/php/conf.d/test.ini + - cp my_php.ini /usr/local/etc/php/conf.d/test.ini ``` Of course, `my_php.ini` must be present in the root directory of your repository. @@ -166,7 +166,7 @@ Next, add the following snippet to your `.gitlab-ci.yml`: ```yaml test:app: script: - - phpunit --configuration phpunit_myapp.xml + - phpunit --configuration phpunit_myapp.xml ``` Finally, push to GitLab and let the tests begin! @@ -217,11 +217,11 @@ you can use [atoum](https://github.com/atoum/atoum): ```yaml before_script: -- wget http://downloads.atoum.org/nightly/mageekguy.atoum.phar + - wget http://downloads.atoum.org/nightly/mageekguy.atoum.phar test:atoum: script: - - php mageekguy.atoum.phar + - php mageekguy.atoum.phar ``` ### Using Composer @@ -238,16 +238,16 @@ following in your `.gitlab-ci.yml`: # your git repository. cache: paths: - - vendor/ + - vendor/ before_script: # Install composer dependencies -- wget https://composer.github.io/installer.sig -O - -q | tr -d '\n' > installer.sig -- php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');" -- php -r "if (hash_file('SHA384', 'composer-setup.php') === file_get_contents('installer.sig')) { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;" -- php composer-setup.php -- php -r "unlink('composer-setup.php'); unlink('installer.sig');" -- php composer.phar install + - wget https://composer.github.io/installer.sig -O - -q | tr -d '\n' > installer.sig + - php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');" + - php -r "if (hash_file('SHA384', 'composer-setup.php') === file_get_contents('installer.sig')) { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;" + - php composer-setup.php + - php -r "unlink('composer-setup.php'); unlink('installer.sig');" + - php composer.phar install ... ``` diff --git a/doc/ci/examples/test-and-deploy-python-application-to-heroku.md b/doc/ci/examples/test-and-deploy-python-application-to-heroku.md index 30a64e65607..d01e9663795 100644 --- a/doc/ci/examples/test-and-deploy-python-application-to-heroku.md +++ b/doc/ci/examples/test-and-deploy-python-application-to-heroku.md @@ -23,32 +23,32 @@ stages: test: stage: test script: - # this configures Django application to use attached postgres database that is run on `postgres` host - - export DATABASE_URL=postgres://postgres:@postgres:5432/python-test-app - - apt-get update -qy - - apt-get install -y python-dev python-pip - - pip install -r requirements.txt - - python manage.py test + # this configures Django application to use attached postgres database that is run on `postgres` host + - export DATABASE_URL=postgres://postgres:@postgres:5432/python-test-app + - apt-get update -qy + - apt-get install -y python-dev python-pip + - pip install -r requirements.txt + - python manage.py test staging: stage: deploy script: - - apt-get update -qy - - apt-get install -y ruby-dev - - gem install dpl - - dpl --provider=heroku --app=gitlab-ci-python-test-staging --api-key=$HEROKU_STAGING_API_KEY + - apt-get update -qy + - apt-get install -y ruby-dev + - gem install dpl + - dpl --provider=heroku --app=gitlab-ci-python-test-staging --api-key=$HEROKU_STAGING_API_KEY only: - - master + - master production: stage: deploy script: - - apt-get update -qy - - apt-get install -y ruby-dev - - gem install dpl - - dpl --provider=heroku --app=gitlab-ci-python-test-prod --api-key=$HEROKU_PRODUCTION_API_KEY + - apt-get update -qy + - apt-get install -y ruby-dev + - gem install dpl + - dpl --provider=heroku --app=gitlab-ci-python-test-prod --api-key=$HEROKU_PRODUCTION_API_KEY only: - - tags + - tags ``` This project has three jobs: diff --git a/doc/ci/img/metrics_reports_advanced_v13_0.png b/doc/ci/img/metrics_reports_advanced_v13_0.png Binary files differnew file mode 100644 index 00000000000..c092c6a84e4 --- /dev/null +++ b/doc/ci/img/metrics_reports_advanced_v13_0.png diff --git a/doc/ci/interactive_web_terminal/index.md b/doc/ci/interactive_web_terminal/index.md index be3741332e0..c79fb5bc926 100644 --- a/doc/ci/interactive_web_terminal/index.md +++ b/doc/ci/interactive_web_terminal/index.md @@ -38,10 +38,12 @@ but support [is planned](https://gitlab.com/gitlab-org/charts/gitlab-runner/-/is ## Debugging a running job -NOTE: **Note:** Not all executors are +NOTE: **Note:** +Not all executors are [supported](https://docs.gitlab.com/runner/executors/#compatibility-chart). -NOTE: **Note:** The `docker` executor does not keep running +NOTE: **Note:** +The `docker` executor does not keep running after the build script is finished. At that point, the terminal will automatically disconnect and will not wait for the user to finish. Please follow [this issue](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/3605) for updates on @@ -67,6 +69,6 @@ close the terminal window.  -## Interactive Web Terminals for the Web IDE **(ULTIMATE ONLY)** +## Interactive Web Terminals for the Web IDE -Read the Web IDE docs to learn how to run [Interactive Terminals through the Web IDE](../../user/project/web_ide/index.md). +Read the Web IDE docs to learn how to run [Interactive Terminals through the Web IDE](../../user/project/web_ide/index.md#interactive-web-terminals-for-the-web-ide). diff --git a/doc/ci/introduction/index.md b/doc/ci/introduction/index.md index 7ba24f9c861..db18624d4e9 100644 --- a/doc/ci/introduction/index.md +++ b/doc/ci/introduction/index.md @@ -12,9 +12,9 @@ In this document, we'll present an overview of the concepts of Continuous Integr Continuous Delivery, and Continuous Deployment, as well as an introduction to GitLab CI/CD. -NOTE: **Out-of-the-box management systems can decrease hours spent on maintaining toolchains by 10% or more.** -Watch our -["Mastering continuous software development"](https://about.gitlab.com/webcast/mastering-ci-cd/) +NOTE: **Note:** +Out-of-the-box management systems can decrease hours spent on maintaining toolchains by 10% or more. +Watch our ["Mastering continuous software development"](https://about.gitlab.com/webcast/mastering-ci-cd/) webcast to learn about continuous methods and how GitLab’s built-in CI can help you simplify and scale software development. ## Introduction to CI/CD methodologies @@ -188,8 +188,9 @@ according to each stage (Verify, Package, Release). 1. **Verify**: - Automatically build and test your application with Continuous Integration. - - Analyze your source code quality with [GitLab Code Quality](../../user/project/merge_requests/code_quality.md). **(STARTER)** - - Determine the performance impact of code changes with [Browser Performance Testing](../../user/project/merge_requests/browser_performance_testing.md). **(PREMIUM)** + - Analyze your source code quality with [GitLab Code Quality](../../user/project/merge_requests/code_quality.md). + - Determine the browser performance impact of code changes with [Browser Performance Testing](../../user/project/merge_requests/browser_performance_testing.md). **(PREMIUM)** + - Determine the server performance impact of code changes with [Load Performance Testing](../../user/project/merge_requests/load_performance_testing.md). **(PREMIUM)** - Perform a series of tests, such as [Container Scanning](../../user/application_security/container_scanning/index.md) **(ULTIMATE)**, [Dependency Scanning](../../user/application_security/dependency_scanning/index.md) **(ULTIMATE)**, and [JUnit tests](../junit_test_reports.md). - Deploy your changes with [Review Apps](../review_apps/index.md) to preview the app changes on every branch. 1. **Package**: @@ -202,7 +203,7 @@ according to each stage (Verify, Package, Release). - Continuous Delivery, manually click to deploy your app to production. - Deploy static websites with [GitLab Pages](../../user/project/pages/index.md). - Ship features to only a portion of your pods and let a percentage of your user base to visit the temporarily deployed feature with [Canary Deployments](../../user/project/canary_deployments.md). **(PREMIUM)** - - Deploy your features behind [Feature Flags](../../user/project/operations/feature_flags.md). **(PREMIUM)** + - Deploy your features behind [Feature Flags](../../operations/feature_flags.md). **(PREMIUM)** - Add release notes to any Git tag with [GitLab Releases](../../user/project/releases/index.md). - View of the current health and status of each CI environment running on Kubernetes with [Deploy Boards](../../user/project/deploy_boards.md). **(PREMIUM)** - Deploy your application to a production environment in a Kubernetes cluster with [Auto Deploy](../../topics/autodevops/stages.md#auto-deploy). @@ -226,7 +227,7 @@ To get started with GitLab CI/CD, you need to familiarize yourself with the [`.gitlab-ci.yml`](../yaml/README.md) configuration file syntax and with its attributes. -This document [introduces the concepts of GitLab CI/CD in the scope of GitLab Pages](../../user/project/pages/getting_started_part_four.md), for deploying static websites. +This document [introduces the concepts of GitLab CI/CD in the scope of GitLab Pages](../../user/project/pages/getting_started/pages_from_scratch.md), for deploying static websites. Although it's meant for users who want to write their own Pages script from scratch, it also serves as an introduction to the setup process for GitLab CI/CD. It covers the very first general steps of writing a CI/CD configuration diff --git a/doc/ci/jenkins/index.md b/doc/ci/jenkins/index.md index 37e0522e74d..1d029dcdd14 100644 --- a/doc/ci/jenkins/index.md +++ b/doc/ci/jenkins/index.md @@ -12,16 +12,25 @@ A lot of GitLab users have successfully migrated to GitLab CI/CD from Jenkins. T easier if you're just getting started, we've collected several resources here that you might find useful before diving in. Think of this page as a "GitLab CI/CD for Jenkins Users" guide. -First of all, our [Quick Start Guide](../quick_start/README.md) contains a good overview of how GitLab CI/CD works. -You may also be interested in [Auto DevOps](../../topics/autodevops/index.md) which can potentially be used to build, test, -and deploy your applications with little to no configuration needed at all. +The following list of recommended steps was created after observing organizations +that were able to quickly complete this migration: + +1. Start by reading the GitLab CI/CD [Quick Start Guide](../quick_start/README.md) and [important product differences](#important-product-differences). +1. Learn the importance of [managing the organizational transition](#managing-the-organizational-transition). +1. [Add Runners](../runners/README.md) to your GitLab instance. +1. Educate and enable your developers to independently perform the following steps in their projects: + 1. Review the [Quick Start Guide](../quick_start/README.md) and [Pipeline Configuration Reference](../yaml/README.md). + 1. Use the [Jenkins Wrapper](#jenkinsfile-wrapper) to temporarily maintain fragile Jenkins jobs. + 1. Migrate the build and CI jobs and configure them to show results directly in your merge requests. They can use [Auto DevOps](../../topics/autodevops/index.md) as a starting point, and [customize](../../topics/autodevops/customize.md) or [decompose](../../topics/autodevops/customize.md#using-components-of-auto-devops) the configuration as needed. + 1. Add [Review Apps](../review_apps/index.md). + 1. Migrate the deployment jobs using [cloud deployment templates](../cloud_deployment/index.md), adding [environments](../environments/index.md), and [deploy boards](../..//user/project/deploy_boards.md). + 1. Work to unwrap any jobs still running with the use of the Jenkins wrapper. +1. Take stock of any common CI/CD job definitions then create and share [templates](#templates) for them. For an example of how to convert a Jenkins pipeline into a GitLab CI/CD pipeline, or how to use Auto DevOps to test your code automatically, watch the [Migrating from Jenkins to GitLab](https://www.youtube.com/watch?v=RlEVGOpYF5Y) video. -For advanced CI/CD teams, [templates](#templates) can enable the reuse of pipeline configurations. - Otherwise, read on for important information that will help you get the ball rolling. Welcome to GitLab! @@ -53,8 +62,7 @@ We are building a [JenkinsFile Wrapper](https://gitlab.com/gitlab-org/jfr-contai you to run a complete Jenkins instance inside of a GitLab job, including plugins. This can help ease the process of transition, by letting you delay the migration of less urgent pipelines for a period of time. -If you are interested, join our [public testing issue](https://gitlab.com/gitlab-org/gitlab/-/issues/215675) to -If you are interested, you might be able to [help GitLab test the wrapper](https://gitlab.com/gitlab-org/gitlab/-/issues/215675). +If you are interested in helping GitLab test the wrapper, join our [public testing issue](https://gitlab.com/gitlab-org/gitlab/-/issues/215675) for instructions and to provide your feedback. ## Important product differences @@ -173,8 +181,8 @@ pdf: script: xelatex mycv.tex artifacts: paths: - - ./mycv.pdf - - ./output/ + - ./mycv.pdf + - ./output/ expire_in: 1 week ``` diff --git a/doc/ci/junit_test_reports.md b/doc/ci/junit_test_reports.md index aa0d40a4d06..aedda3c8341 100644 --- a/doc/ci/junit_test_reports.md +++ b/doc/ci/junit_test_reports.md @@ -73,6 +73,7 @@ For a list of supported languages on JUnit tests, check the To enable the JUnit reports in merge requests, you need to add [`artifacts:reports:junit`](pipelines/job_artifacts.md#artifactsreportsjunit) in `.gitlab-ci.yml`, and specify the path(s) of the generated test reports. +The reports must be `.xml` files, otherwise [GitLab returns an Error 500](https://gitlab.com/gitlab-org/gitlab/-/issues/216575). In the following examples, the job in the `test` stage runs and GitLab collects the JUnit test report from each job. After each job is executed, the @@ -103,7 +104,8 @@ ruby: ### Go example -Use the following job in `.gitlab-ci.yml`: +Use the following job in `.gitlab-ci.yml`, and ensure you use `-set-exit-code`, +otherwise the pipeline will be marked successful, even if the tests fail: ```yaml ## Use https://github.com/jstemmer/go-junit-report to generate a JUnit report with go @@ -111,7 +113,7 @@ golang: stage: test script: - go get -u github.com/jstemmer/go-junit-report - - go test -v 2>&1 | go-junit-report > report.xml + - go test -v 2>&1 | go-junit-report -set-exit-code > report.xml artifacts: reports: junit: report.xml @@ -206,7 +208,7 @@ cunit: junit: ./my-cunit-test.xml ``` -### .Net example +### .NET example The [JunitXML.TestLogger](https://www.nuget.org/packages/JunitXml.TestLogger/) NuGet package can generate test reports for .Net Framework and .Net Core applications. The following @@ -234,7 +236,9 @@ Test: ## Viewing JUnit test reports on GitLab -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/24792) in GitLab 12.5. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/24792) in GitLab 12.5. +> - It's deployed behind a feature flag, disabled by default. +> - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enabling-the-junit-test-reports-feature-core-only). **(CORE ONLY)** If JUnit XML files are generated and uploaded as part of a pipeline, these reports can be viewed inside the pipelines details page. The **Tests** tab on this page will @@ -248,7 +252,7 @@ with failed showing at the top, skipped next and successful cases last. You can also retrieve the reports via the [GitLab API](../api/pipelines.md#get-a-pipelines-test-report). -### Enabling the feature +### Enabling the JUnit test reports feature **(CORE ONLY)** This feature comes with the `:junit_pipeline_view` feature flag disabled by default. This feature is disabled due to some performance issues with very large data sets. @@ -260,13 +264,15 @@ following command: ```ruby Feature.enable(:junit_pipeline_view) -# Enable the feature for a specific project +# Enable the feature for a specific project, GitLab 13.0 and above only. Feature.enable(:junit_pipeline_view, Project.find(<your-project-id-here>)) ``` ## Viewing JUnit screenshots on GitLab -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/202114) in GitLab 13.0. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/202114) in GitLab 13.0. +> - It's deployed behind a feature flag, disabled by default. +> - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enabling-the-junit-screenshots-feature-core-only). **(CORE ONLY)** If JUnit XML files contain an `attachment` tag, GitLab parses the attachment. @@ -280,7 +286,7 @@ Upload your screenshots as [artifacts](pipelines/job_artifacts.md#artifactsrepor When [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/6061) is complete, the attached file will be visible on the pipeline details page. -### Enabling the feature +### Enabling the JUnit screenshots feature **(CORE ONLY)** This feature comes with the `:junit_pipeline_screenshots_view` feature flag disabled by default. diff --git a/doc/ci/merge_request_pipelines/index.md b/doc/ci/merge_request_pipelines/index.md index 7a4ca989fb6..2a6008e6307 100644 --- a/doc/ci/merge_request_pipelines/index.md +++ b/doc/ci/merge_request_pipelines/index.md @@ -24,7 +24,7 @@ can run a pipeline for merge requests.  -NOTE: **Note**: +NOTE: **Note:** If you use this feature with [merge when pipeline succeeds](../../user/project/merge_requests/merge_when_pipeline_succeeds.md), pipelines for merge requests take precedence over the other regular pipelines. @@ -32,7 +32,6 @@ pipelines for merge requests take precedence over the other regular pipelines. To enable pipelines for merge requests: -- You must have maintainer [permissions](../../user/permissions.md). - Your repository must be a GitLab repository, not an [external repository](../ci_cd_for_external_repos/index.md). - [In GitLab 11.10 and later](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/25504), @@ -66,19 +65,19 @@ build: stage: build script: ./build only: - - master + - master test: stage: test script: ./test only: - - merge_requests + - merge_requests deploy: stage: deploy script: ./deploy only: - - master + - master ``` #### Excluding certain jobs @@ -207,7 +206,7 @@ The variable names begin with the `CI_MERGE_REQUEST_` prefix. ### Two pipelines created when pushing to a merge request If you are experiencing duplicated pipelines when using `rules`, take a look at -the [key details when using `rules`](../yaml/README.md#key-details-when-using-rules), +the [important differences between `rules` and `only`/`except`](../yaml/README.md#differences-between-rules-and-onlyexcept), which will help you get your starting configuration correct. If you are seeing two pipelines when using `only/except`, please see the caveats diff --git a/doc/ci/merge_request_pipelines/pipelines_for_merged_results/index.md b/doc/ci/merge_request_pipelines/pipelines_for_merged_results/index.md index a43085c8e7a..84fbefb080f 100644 --- a/doc/ci/merge_request_pipelines/pipelines_for_merged_results/index.md +++ b/doc/ci/merge_request_pipelines/pipelines_for_merged_results/index.md @@ -30,8 +30,7 @@ can still be successfully merged into the target. When the merge request can't be merged, the pipeline runs against the source branch only. For example, when: - The target branch has changes that conflict with the changes in the source branch. -- The merge request is a - [work in progress](../../../user/project/merge_requests/work_in_progress_merge_requests.md). +- The merge request is a [**Draft** merge request](../../../user/project/merge_requests/work_in_progress_merge_requests.md). In these cases, the pipeline runs as a [pipeline for merge requests](../index.md) and is labeled as `detached`. If these cases no longer exist, new pipelines will diff --git a/doc/ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md b/doc/ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md index fc8e6368d72..e12c9d109cc 100644 --- a/doc/ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md +++ b/doc/ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md @@ -146,7 +146,7 @@ is recreated and all pipelines restart. ### Merge request dropped from the merge train immediately -If a merge request is not mergeable (for example, it's WIP, there is a merge +If a merge request is not mergeable (for example, it's a draft merge request, there is a merge conflict, etc.), your merge request will be dropped from the merge train automatically. In these cases, the reason for dropping the merge request is in the **system notes**. diff --git a/doc/ci/metrics_reports.md b/doc/ci/metrics_reports.md index 67f9ad00c90..14669edf7eb 100644 --- a/doc/ci/metrics_reports.md +++ b/doc/ci/metrics_reports.md @@ -49,3 +49,10 @@ metrics: reports: metrics: metrics.txt ``` + +## Advanced Example + +An advanced example of an OpenMetrics text file (from the [Prometheus documentation](https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md#text-format-example)) +renders in the merge request widget as: + + diff --git a/doc/ci/migration/circleci.md b/doc/ci/migration/circleci.md index f6868abc334..625b15ca4fb 100644 --- a/doc/ci/migration/circleci.md +++ b/doc/ci/migration/circleci.md @@ -250,14 +250,14 @@ image: node:latest cache: key: $CI_COMMIT_REF_SLUG paths: - - .npm/ + - .npm/ before_script: - npm ci --cache .npm --prefer-offline test_async: script: - - node ./specs/start.js ./specs/async.spec.js + - node ./specs/start.js ./specs/async.spec.js ``` ## Contexts and variables diff --git a/doc/ci/pipelines/index.md b/doc/ci/pipelines/index.md index 48aaebc3456..18b3fe10bec 100644 --- a/doc/ci/pipelines/index.md +++ b/doc/ci/pipelines/index.md @@ -348,17 +348,17 @@ For example, these three jobs will be in a group named `build ruby`: build ruby 1/3: stage: build script: - - echo "ruby1" + - echo "ruby1" build ruby 2/3: stage: build script: - - echo "ruby2" + - echo "ruby2" build ruby 3/3: stage: build script: - - echo "ruby3" + - echo "ruby3" ``` In the pipeline, the result is a group named `build ruby` with three jobs: diff --git a/doc/ci/pipelines/job_artifacts.md b/doc/ci/pipelines/job_artifacts.md index 1d60f412e5e..c4457d17dc2 100644 --- a/doc/ci/pipelines/job_artifacts.md +++ b/doc/ci/pipelines/job_artifacts.md @@ -33,7 +33,7 @@ pdf: script: xelatex mycv.tex artifacts: paths: - - mycv.pdf + - mycv.pdf expire_in: 1 week ``` @@ -87,8 +87,8 @@ Below is an example of collecting a JUnit XML file from Ruby's RSpec test tool: rspec: stage: test script: - - bundle install - - rspec --format RspecJunitFormatter --out rspec.xml + - bundle install + - rspec --format RspecJunitFormatter --out rspec.xml artifacts: reports: junit: rspec.xml @@ -114,14 +114,15 @@ The `dotenv` report collects a set of environment variables as artifacts. The collected variables are registered as runtime-created variables of the job, which is useful to [set dynamic environment URLs after a job finishes](../environments/index.md#set-dynamic-environment-urls-after-a-job-finishes). -There are a couple of limitations on top of the [original dotenv rules](https://github.com/motdotla/dotenv#rules). +There are a couple of exceptions to the [original dotenv rules](https://github.com/motdotla/dotenv#rules): -- The variable key can contain only letters, digits and underscore ('_'). -- The size of the dotenv file must be smaller than 5 kilobytes. -- The number of variables must be less than 10. -- It does not support variable substitution in the dotenv file itself. -- It does not support empty lines and comments (`#`) in dotenv file. -- It does not support quote escape, spaces in a quote, a new line expansion in a quote, in dotenv file. +- The variable key can contain only letters, digits, and underscores (`_`). +- The maximum size of the `.env` file is 5 KB. +- The maximum number of variables is 10. +- Variable substitution in the `.env` file is not supported. +- The `.env` file can't have empty lines or comments (starting with `#`). +- Key values in the `env` file cannot have spaces or newline characters (`\n`), including when using single or double quotes. +- Quote escaping during parsing (`key = 'value'` -> `{key: "value"}`) is not supported. #### `artifacts:reports:cobertura` @@ -145,9 +146,10 @@ plan report will be uploaded to GitLab as an artifact and will be automatically in merge requests. For more information, see [Output `terraform plan` information into a merge request](../../user/infrastructure/index.md#output-terraform-plan-information-into-a-merge-request). -#### `artifacts:reports:codequality` **(STARTER)** +#### `artifacts:reports:codequality` -> - Introduced in GitLab 11.5. +> - Introduced in [GitLab Starter](https://about.gitlab.com/pricing/) 11.5. +> - Made [available in all tiers](https://gitlab.com/gitlab-org/gitlab/-/issues/212499) in GitLab 13.2. > - Requires GitLab Runner 11.5 and above. The `codequality` report collects [CodeQuality issues](../../user/project/merge_requests/code_quality.md) @@ -250,12 +252,23 @@ dashboards. > - Introduced in GitLab 11.5. > - Requires GitLab Runner 11.5 and above. -The `performance` report collects [Performance metrics](../../user/project/merge_requests/browser_performance_testing.md) +The `performance` report collects [Browser Performance Testing metrics](../../user/project/merge_requests/browser_performance_testing.md) as artifacts. -The collected Performance report will be uploaded to GitLab as an artifact and will +The collected Browser Performance report will be uploaded to GitLab as an artifact and will be automatically shown in merge requests. +#### `artifacts:reports:load_performance` **(PREMIUM)** + +> - Introduced in [GitLab 13.2](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/35260) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.2. +> - Requires GitLab Runner 11.5 and above. + +The `load_performance` report collects [Load Performance Testing metrics](../../user/project/merge_requests/load_performance_testing.md) +as artifacts. + +The report is uploaded to GitLab as an artifact and is +shown in merge requests automatically. + #### `artifacts:reports:metrics` **(PREMIUM)** > Introduced in GitLab 11.10. @@ -407,7 +420,7 @@ information in the UI. ## Erasing artifacts -DANGER: **Warning:** +DANGER: **Danger:** This is a destructive action that leads to data loss. Use with caution. You can erase a single job via the UI, which will also remove the job's diff --git a/doc/ci/pipelines/pipeline_architectures.md b/doc/ci/pipelines/pipeline_architectures.md index 9176444660d..ace765ddb41 100644 --- a/doc/ci/pipelines/pipeline_architectures.md +++ b/doc/ci/pipelines/pipeline_architectures.md @@ -133,28 +133,28 @@ build_b: test_a: stage: test - needs: build_a + needs: [build_a] script: - echo "This test job will start as soon as build_a finishes." - echo "It will not wait for build_b, or other jobs in the build stage, to finish." test_b: stage: test - needs: build_b + needs: [build_b] script: - echo "This test job will start as soon as build_b finishes." - echo "It will not wait for other jobs in the build stage to finish." deploy_a: stage: deploy - needs: test_a + needs: [test_a] script: - echo "Since build_a and test_a run quickly, this deploy job can run much earlier." - echo "It does not need to wait for build_b or test_b." deploy_b: stage: deploy - needs: test_b + needs: [test_b] script: - echo "Since build_b and test_b run slowly, this deploy job will run much later." ``` @@ -228,13 +228,13 @@ build_a: test_a: stage: test - needs: build_a + needs: [build_a] script: - echo "This job tests something." deploy_a: stage: deploy - needs: test_a + needs: [test_a] script: - echo "This job deploys something." ``` @@ -257,13 +257,13 @@ build_b: test_b: stage: test - needs: build_b + needs: [build_b] script: - echo "This job tests something else." deploy_b: stage: deploy - needs: test_b + needs: [test_b] script: - echo "This job deploys something else." ``` diff --git a/doc/ci/pipelines/settings.md b/doc/ci/pipelines/settings.md index 2c9d4ccf2c4..39acef14443 100644 --- a/doc/ci/pipelines/settings.md +++ b/doc/ci/pipelines/settings.md @@ -36,7 +36,7 @@ in `.gitlab-ci.yml`. > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/28919) in GitLab 12.0. -NOTE: **Note**: +NOTE: **Note:** As of GitLab 12.0, newly created projects will automatically have a default `git depth` value of `50`. @@ -283,7 +283,7 @@ Markdown code will embed the test coverage report badge of the `coverage` job into your `README.md`: ```markdown - + ``` ### Badge styles @@ -296,7 +296,7 @@ Pipeline badges can be rendered in different styles by adding the `style=style_n https://example.gitlab.com/<namespace>/<project>/badges/<branch>/coverage.svg?style=flat ``` - + #### Flat square @@ -306,17 +306,19 @@ https://example.gitlab.com/<namespace>/<project>/badges/<branch>/coverage.svg?st https://example.gitlab.com/<namespace>/<project>/badges/<branch>/coverage.svg?style=flat-square ``` - + ### Custom badge text +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/17555) in GitLab 13.1. + The text for a badge can be customized. This can be useful to differentiate between multiple coverage jobs that run in the same pipeline. Customize the badge text and width by adding the `key_text=custom_text` and `key_width=custom_key_width` parameters to the URL: ```plaintext -https://gitlab.com/gitlab-org/gitlab-foss/badges/master/coverage.svg?job=karma&key_text=Frontend+Coverage&key_width=100 +https://gitlab.com/gitlab-org/gitlab/badges/master/coverage.svg?job=karma&key_text=Frontend+Coverage&key_width=100 ``` - + ## Environment Variables diff --git a/doc/ci/review_apps/index.md b/doc/ci/review_apps/index.md index 67fac70c8de..283e4c69941 100644 --- a/doc/ci/review_apps/index.md +++ b/doc/ci/review_apps/index.md @@ -260,10 +260,6 @@ For example, in a Ruby application, you would need to have this script: Then, when your app is deployed via GitLab CI/CD, those variables should get replaced with their real values. -NOTE: **Note:** -Future enhancements [are planned](https://gitlab.com/gitlab-org/gitlab/-/issues/11322) -to make this process even easier. - ### Determining merge request ID The visual review tools retrieve the merge request ID from the `data-merge-request-id` diff --git a/doc/ci/runners/README.md b/doc/ci/runners/README.md index 86fabd5d54f..21c99f928d8 100644 --- a/doc/ci/runners/README.md +++ b/doc/ci/runners/README.md @@ -6,6 +6,8 @@ type: reference --- # Configuring GitLab Runners +<!-- This topic contains several commented-out sections that were accidentally added in 13.2.--> +<!-- The commented-out sections will be added back in 13.3.--> In GitLab CI/CD, Runners run the code defined in [`.gitlab-ci.yml`](../yaml/README.md). A GitLab Runner is a lightweight, highly-scalable agent that picks up a CI job through @@ -35,12 +37,21 @@ Use shared Runners when you have multiple jobs with similar requirements. Rather having multiple Runners idling for many projects, you can have a few Runners that handle multiple projects. -If you are using a self-managed instance of GitLab, your administrator can create -shared Runners and configure them to use the -[executor](https://docs.gitlab.com/runner/executors/README.html) you want. +If you are using a self-managed instance of GitLab: -If you are using GitLab.com, you can select from a list of -[shared Runners that GitLab maintains](../../user/gitlab_com/index.md#shared-runners). +- Your administrator can install and register shared Runners by viewing the instructions + [here](https://docs.gitlab.com/runner/install/index.html). + <!-- going to your project's + <!-- **Settings > CI / CD**, expanding the **Runners** section, and clicking **Show Runner installation instructions**.--> + <!-- These instructions are also available [here](https://docs.gitlab.com/runner/install/index.html).--> +- The administrator can also configure a maximum number of shared Runner [pipeline minutes for + each group](../../user/admin_area/settings/continuous_integration.md#shared-runners-pipeline-minutes-quota-starter-only). + +If you are using GitLab.com: + +- You can select from a list of [shared Runners that GitLab maintains](../../user/gitlab_com/index.md#shared-runners). +- The shared Runners consume the [pipelines minutes](../../subscriptions/index.md#ci-pipeline-minutes) + included with your account. #### How shared Runners pick jobs @@ -95,16 +106,38 @@ The fair usage algorithm assigns jobs in this order: 1. Job 6 is next, because Project 3 is the only project left with no running jobs. 1. Lastly we choose Job 3... because, again, it's the only job left. -#### Enable a shared Runner +#### Enable shared Runners + +On GitLab.com, [shared Runners](#shared-runners) are enabled in all projects by +default. + +On self-managed instances of GitLab, an administrator must [install](https://docs.gitlab.com/runner/install/index.html) +and [register](https://docs.gitlab.com/runner/register/index.html) them. + +You can also enable shared Runners for individual projects. + +To enable shared Runners: + +1. Go to the project's **{settings}** **Settings > CI/CD** and expand the **Runners** section. +1. Click **Allow shared Runners**. -By default, all projects can use shared Runners, and they are enabled by default. +#### Disable shared Runners -However, you can enable or disable shared Runners for individual projects. +You can disable shared Runners for individual projects<!-- or for groups-->. +You must have Owner permissions for the project<!-- or group-->. -To enable or disable a shared Runner: +To disable shared Runners for a project: 1. Go to the project's **{settings}** **Settings > CI/CD** and expand the **Runners** section. -1. Click **Allow shared Runners** or **Disable shared Runners**. +1. In the **Shared Runners** area, click **Disable shared Runners**. + +<!--To disable shared Runners for a group: + +1. Go to the group's **{settings}** **Settings > CI/CD** and expand the **Runners** section. +1. In the **Shared Runners** area, click **Disable shared Runners globally**. +1. Optionally, to allow shared Runners to be enabled for individual projects or subgroups, + click **Allow projects/subgroups to override the global setting**. +--> ### Group Runners @@ -126,14 +159,43 @@ To create a group Runner: 1. Note the URL and token. 1. [Register the Runner](https://docs.gitlab.com/runner/register/). +<!-- #### View and manage group Runners + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/37366/) in GitLab 13.3. + +You can view and manage all Runners for a group, its subgroups, and projects. +You can do this for your self-managed GitLab instance or for GitLab.com. +You must have [Owner permissions](../../user/permissions.md#group-members-permissions) for the group. + +1. Go to the group where you want to view the Runners. +1. Go to **{settings}** **Settings > CI/CD** and expand the **Runners** section. +1. The following fields are displayed. + + | Attribute | Description | + | ------------ | ----------- | + | Type | One or more of the following states: shared, group, specific, locked, or paused | + | Runner token | Token used to identify the Runner, and which the Runner uses to communicate with the GitLab instance | + | Description | Description given to the Runner when it was created | + | Version | GitLab Runner version | + | IP address | IP address of the host on which the Runner is registered | + | Projects | The count of projects to which the Runner is assigned | + | Jobs | Total of jobs run by the Runner | + | Tags | Tags associated with the Runner | + | Last contact | Timestamp indicating when the GitLab instance last contacted the Runner | + +From this page, you can edit, pause, and remove Runners from the group, its subgroups, and projects. --> + #### Pause or remove a group Runner -You can pause or remove a group Runner. +You can pause or remove a group Runner for your self-managed GitLab instance or for GitLab.com. You must have [Owner permissions](../../user/permissions.md#group-members-permissions) for the group. 1. Go to the group you want to remove or pause the Runner for. 1. Go to **{settings}** **Settings > CI/CD** and expand the **Runners** section. 1. Click **Pause** or **Remove Runner**. +<!-- - If you pause a group Runner that is used by multiple projects, the Runner pauses for all projects. --> +<!-- - From the group view, you cannot remove a Runner that is assigned to more than one project. --> +<!-- You must remove it from each project first. --> 1. On the confirmation dialog, click **OK**. ### Specific Runners diff --git a/doc/ci/triggers/README.md b/doc/ci/triggers/README.md index c6ac87ef888..47f11a6228c 100644 --- a/doc/ci/triggers/README.md +++ b/doc/ci/triggers/README.md @@ -62,9 +62,9 @@ and it creates a dependent pipeline relation visible on the build_docs: stage: deploy script: - - curl --request POST --form "token=$CI_JOB_TOKEN" --form ref=master https://gitlab.example.com/api/v4/projects/9/trigger/pipeline + - curl --request POST --form "token=$CI_JOB_TOKEN" --form ref=master https://gitlab.example.com/api/v4/projects/9/trigger/pipeline only: - - tags + - tags ``` Pipelines triggered that way also expose a special variable: @@ -86,11 +86,11 @@ build_submodule: image: debian stage: test script: - - apt update && apt install -y unzip - - curl --location --output artifacts.zip "https://gitlab.example.com/api/v4/projects/1/jobs/artifacts/master/download?job=test&job_token=$CI_JOB_TOKEN" - - unzip artifacts.zip + - apt update && apt install -y unzip + - curl --location --output artifacts.zip "https://gitlab.example.com/api/v4/projects/1/jobs/artifacts/master/download?job=test&job_token=$CI_JOB_TOKEN" + - unzip artifacts.zip only: - - tags + - tags ``` This allows you to use that for multi-project pipelines and download artifacts @@ -179,9 +179,9 @@ need to add in project A's `.gitlab-ci.yml`: build_docs: stage: deploy script: - - "curl --request POST --form token=TOKEN --form ref=master https://gitlab.example.com/api/v4/projects/9/trigger/pipeline" + - "curl --request POST --form token=TOKEN --form ref=master https://gitlab.example.com/api/v4/projects/9/trigger/pipeline" only: - - tags + - tags ``` This means that whenever a new tag is pushed on project A, the job will run and the @@ -235,24 +235,24 @@ variable is non-zero, `make upload` is run. ```yaml stages: -- test -- build -- package + - test + - build + - package run_tests: stage: test script: - - make test + - make test build_package: stage: build script: - - make build + - make build upload_package: stage: package script: - - if [ -n "${UPLOAD_TO_S3}" ]; then make upload; fi + - if [ -n "${UPLOAD_TO_S3}" ]; then make upload; fi ``` You can then trigger a rebuild while you pass the `UPLOAD_TO_S3` variable diff --git a/doc/ci/troubleshooting.md b/doc/ci/troubleshooting.md new file mode 100644 index 00000000000..a019f8232a9 --- /dev/null +++ b/doc/ci/troubleshooting.md @@ -0,0 +1,38 @@ +--- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: reference +--- + +# Troubleshooting CI/CD + +## Merge request pipeline widget + +The merge request pipeline widget shows information about the pipeline status in a Merge Request. It's displayed above the [merge request ability to merge widget](#merge-request-ability-to-merge-widget). + +There are several messages that can be displayed depending on the status of the pipeline. + +### "Checking pipeline status" + +This message is shown when the merge request has no pipeline associated with the latest commit yet and [Pipelines must succeed](../user/project/merge_requests/merge_when_pipeline_succeeds.md#only-allow-merge-requests-to-be-merged-if-the-pipeline-succeeds) is turned on. This might be because: + +- GitLab hasn't finished creating the pipeline yet. +- You are using an external CI service and GitLab hasn't heard back from the service yet. +- You are not using CI/CD pipelines in your project. + +After the pipeline is created, the message will update with the pipeline status. + +Note: Currently if you delete the latest pipeline of a Merge Request, this message will be shown instead of a meaningful error message. This is a known issue and should be resolved soon. + +## Merge request ability to merge widget + +The merge request status widget shows the **Merge** button and whether or not a merge request is ready to merge. If the merge request can't be merged, the reason for this is displayed. + +If the pipeline is still running, the **Merge** button is replaced with the **Merge when pipeline succeeds** button. + +If [**Merge Trains**](merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md) are enabled, the button is either **Add to merge train** or **Add to merge train when pipeline succeeds**. **(PREMIUM)** + +### "A CI/CD pipeline must run and be successful before merge" + +This message is shown if the [Pipelines must succeed](../user/project/merge_requests/merge_when_pipeline_succeeds.md#only-allow-merge-requests-to-be-merged-if-the-pipeline-succeeds) setting is enabled in the project and a pipeline has not yet run successfully. This also applies if the pipeline has not been created yet, or if you are waiting for an external CI service. If you don't use pipelines for your project, then you should disable **Pipelines must succeed** so you can accept merge requests. diff --git a/doc/ci/variables/README.md b/doc/ci/variables/README.md index 541e739082f..4f9a1d8dd27 100644 --- a/doc/ci/variables/README.md +++ b/doc/ci/variables/README.md @@ -227,18 +227,18 @@ To protect a variable: 1. Select the **Protect variable** check box. 1. Click **Update variable**. -The variable is available for all subsequent pipelines. +The variable is available for all subsequent pipelines. Protected variables can only +be updated or viewed by project members with [maintainer permissions](../../user/permissions.md#project-members-permissions). ### Custom variables validated by GitLab Some variables are listed in the UI so you can choose them more quickly. -GitLab validates the values of these variables to ensure they are in the correct format. | Variable | Allowed Values | Introduced in | |-------------------------|----------------------------------------------------|---------------| -| `AWS_ACCESS_KEY_ID` | 20 characters: letters, digits | 12.10 | +| `AWS_ACCESS_KEY_ID` | Any | 12.10 | | `AWS_DEFAULT_REGION` | Any | 12.10 | -| `AWS_SECRET_ACCESS_KEY` | 40 characters: letters, digits, special characters | 12.10 | +| `AWS_SECRET_ACCESS_KEY` | Any | 12.10 | NOTE: **Note:** When you store credentials, there are security implications. If you are using AWS keys, @@ -341,6 +341,7 @@ export CI_PROJECT_DIR="/builds/gitlab-org/gitlab-foss" export CI_PROJECT_NAME="gitlab-foss" export CI_PROJECT_TITLE="GitLab FOSS" export CI_PROJECT_NAMESPACE="gitlab-org" +export CI_PROJECT_ROOT_NAMESPACE="gitlab-org" export CI_PROJECT_PATH="gitlab-org/gitlab-foss" export CI_PROJECT_URL="https://example.com/gitlab-org/gitlab-foss" export CI_REGISTRY="registry.example.com" @@ -457,9 +458,6 @@ The UI interface for Instance-level CI/CD variables is under development but rea It is deployed behind a feature flag that is **enabled by default**. [GitLab administrators with access to the GitLab Rails console](../../administration/feature_flags.md) can opt to disable it for your instance. -NOTE: **Note:** -This feature will not work if the [instance-level CI/CD variables API feature flag is disabled](../../api/instance_level_ci_variables.md#enable-or-disable-instance-level-cicd-variables-core-only). - To disable it: ```ruby @@ -895,8 +893,8 @@ if [[ -d "/builds/gitlab-examples/ci-debug-trace/.git" ]]; then ++ CI_SERVER_VERSION_PATCH=0 ++ export CI_SERVER_REVISION=f4cc00ae823 ++ CI_SERVER_REVISION=f4cc00ae823 -++ export GITLAB_FEATURES=audit_events,burndown_charts,code_owners,contribution_analytics,description_diffs,elastic_search,group_bulk_edit,group_burndown_charts,group_webhooks,issuable_default_templates,issue_weights,jenkins_integration,ldap_group_sync,member_lock,merge_request_approvers,multiple_issue_assignees,multiple_ldap_servers,multiple_merge_request_assignees,protected_refs_for_users,push_rules,related_issues,repository_mirrors,repository_size_limit,scoped_issue_board,usage_quotas,visual_review_app,wip_limits,adjourned_deletion_for_projects_and_groups,admin_audit_log,auditor_user,batch_comments,blocking_merge_requests,board_assignee_lists,board_milestone_lists,ci_cd_projects,cluster_deployments,code_analytics,code_owner_approval_required,commit_committer_check,cross_project_pipelines,custom_file_templates,custom_file_templates_for_namespace,custom_project_templates,custom_prometheus_metrics,cycle_analytics_for_groups,db_load_balancing,default_project_deletion_protection,dependency_proxy,deploy_board,design_management,email_additional_text,extended_audit_events,external_authorization_service_api_management,feature_flags,file_locks,geo,github_project_service_integration,group_allowed_email_domains,group_project_templates,group_saml,issues_analytics,jira_dev_panel_integration,ldap_group_sync_filter,merge_pipelines,merge_request_performance_metrics,merge_trains,metrics_reports,multiple_approval_rules,multiple_clusters,multiple_group_issue_boards,object_storage,operations_dashboard,packages,productivity_analytics,project_aliases,protected_environments,reject_unsigned_commits,required_ci_templates,scoped_labels,service_desk,smartcard_auth,group_timelogs,type_of_work_analytics,unprotection_restrictions,ci_project_subscriptions,cluster_health,container_scanning,dast,dependency_scanning,epics,group_ip_restriction,incident_management,insights,license_management,personal_access_token_expiration_policy,pod_logs,prometheus_alerts,pseudonymizer,report_approver_rules,sast,security_dashboard,tracing,web_ide_terminal -++ GITLAB_FEATURES=audit_events,burndown_charts,code_owners,contribution_analytics,description_diffs,elastic_search,group_bulk_edit,group_burndown_charts,group_webhooks,issuable_default_templates,issue_weights,jenkins_integration,ldap_group_sync,member_lock,merge_request_approvers,multiple_issue_assignees,multiple_ldap_servers,multiple_merge_request_assignees,protected_refs_for_users,push_rules,related_issues,repository_mirrors,repository_size_limit,scoped_issue_board,usage_quotas,visual_review_app,wip_limits,adjourned_deletion_for_projects_and_groups,admin_audit_log,auditor_user,batch_comments,blocking_merge_requests,board_assignee_lists,board_milestone_lists,ci_cd_projects,cluster_deployments,code_analytics,code_owner_approval_required,commit_committer_check,cross_project_pipelines,custom_file_templates,custom_file_templates_for_namespace,custom_project_templates,custom_prometheus_metrics,cycle_analytics_for_groups,db_load_balancing,default_project_deletion_protection,dependency_proxy,deploy_board,design_management,email_additional_text,extended_audit_events,external_authorization_service_api_management,feature_flags,file_locks,geo,github_project_service_integration,group_allowed_email_domains,group_project_templates,group_saml,issues_analytics,jira_dev_panel_integration,ldap_group_sync_filter,merge_pipelines,merge_request_performance_metrics,merge_trains,metrics_reports,multiple_approval_rules,multiple_clusters,multiple_group_issue_boards,object_storage,operations_dashboard,packages,productivity_analytics,project_aliases,protected_environments,reject_unsigned_commits,required_ci_templates,scoped_labels,service_desk,smartcard_auth,group_timelogs,type_of_work_analytics,unprotection_restrictions,ci_project_subscriptions,cluster_health,container_scanning,dast,dependency_scanning,epics,group_ip_restriction,incident_management,insights,license_management,personal_access_token_expiration_policy,pod_logs,prometheus_alerts,pseudonymizer,report_approver_rules,sast,security_dashboard,tracing,web_ide_terminal +++ export GITLAB_FEATURES=audit_events,burndown_charts,code_owners,contribution_analytics,description_diffs,elastic_search,group_bulk_edit,group_burndown_charts,group_webhooks,issuable_default_templates,issue_weights,jenkins_integration,ldap_group_sync,member_lock,merge_request_approvers,multiple_issue_assignees,multiple_ldap_servers,multiple_merge_request_assignees,protected_refs_for_users,push_rules,related_issues,repository_mirrors,repository_size_limit,scoped_issue_board,usage_quotas,visual_review_app,wip_limits,adjourned_deletion_for_projects_and_groups,admin_audit_log,auditor_user,batch_comments,blocking_merge_requests,board_assignee_lists,board_milestone_lists,ci_cd_projects,cluster_deployments,code_analytics,code_owner_approval_required,commit_committer_check,cross_project_pipelines,custom_file_templates,custom_file_templates_for_namespace,custom_project_templates,custom_prometheus_metrics,cycle_analytics_for_groups,db_load_balancing,default_project_deletion_protection,dependency_proxy,deploy_board,design_management,email_additional_text,extended_audit_events,external_authorization_service_api_management,feature_flags,file_locks,geo,github_project_service_integration,group_allowed_email_domains,group_project_templates,group_saml,issues_analytics,jira_dev_panel_integration,ldap_group_sync_filter,merge_pipelines,merge_request_performance_metrics,merge_trains,metrics_reports,multiple_approval_rules,multiple_group_issue_boards,object_storage,operations_dashboard,packages,productivity_analytics,project_aliases,protected_environments,reject_unsigned_commits,required_ci_templates,scoped_labels,service_desk,smartcard_auth,group_timelogs,type_of_work_analytics,unprotection_restrictions,ci_project_subscriptions,container_scanning,dast,dependency_scanning,epics,group_ip_restriction,incident_management,insights,license_management,personal_access_token_expiration_policy,pod_logs,prometheus_alerts,pseudonymizer,report_approver_rules,sast,security_dashboard,tracing,web_ide_terminal +++ GITLAB_FEATURES=audit_events,burndown_charts,code_owners,contribution_analytics,description_diffs,elastic_search,group_bulk_edit,group_burndown_charts,group_webhooks,issuable_default_templates,issue_weights,jenkins_integration,ldap_group_sync,member_lock,merge_request_approvers,multiple_issue_assignees,multiple_ldap_servers,multiple_merge_request_assignees,protected_refs_for_users,push_rules,related_issues,repository_mirrors,repository_size_limit,scoped_issue_board,usage_quotas,visual_review_app,wip_limits,adjourned_deletion_for_projects_and_groups,admin_audit_log,auditor_user,batch_comments,blocking_merge_requests,board_assignee_lists,board_milestone_lists,ci_cd_projects,cluster_deployments,code_analytics,code_owner_approval_required,commit_committer_check,cross_project_pipelines,custom_file_templates,custom_file_templates_for_namespace,custom_project_templates,custom_prometheus_metrics,cycle_analytics_for_groups,db_load_balancing,default_project_deletion_protection,dependency_proxy,deploy_board,design_management,email_additional_text,extended_audit_events,external_authorization_service_api_management,feature_flags,file_locks,geo,github_project_service_integration,group_allowed_email_domains,group_project_templates,group_saml,issues_analytics,jira_dev_panel_integration,ldap_group_sync_filter,merge_pipelines,merge_request_performance_metrics,merge_trains,metrics_reports,multiple_approval_rules,multiple_group_issue_boards,object_storage,operations_dashboard,packages,productivity_analytics,project_aliases,protected_environments,reject_unsigned_commits,required_ci_templates,scoped_labels,service_desk,smartcard_auth,group_timelogs,type_of_work_analytics,unprotection_restrictions,ci_project_subscriptions,cluster_health,container_scanning,dast,dependency_scanning,epics,group_ip_restriction,incident_management,insights,license_management,personal_access_token_expiration_policy,pod_logs,prometheus_alerts,pseudonymizer,report_approver_rules,sast,security_dashboard,tracing,web_ide_terminal ++ export CI_PROJECT_ID=17893 ++ CI_PROJECT_ID=17893 ++ export CI_PROJECT_NAME=ci-debug-trace @@ -909,6 +907,8 @@ if [[ -d "/builds/gitlab-examples/ci-debug-trace/.git" ]]; then ++ CI_PROJECT_PATH_SLUG=gitlab-examples-ci-debug-trace ++ export CI_PROJECT_NAMESPACE=gitlab-examples ++ CI_PROJECT_NAMESPACE=gitlab-examples +++ export CI_PROJECT_ROOT_NAMESPACE=gitlab-examples +++ CI_PROJECT_ROOT_NAMESPACE=gitlab-examples ++ export CI_PROJECT_URL=https://gitlab.com/gitlab-examples/ci-debug-trace ++ CI_PROJECT_URL=https://gitlab.com/gitlab-examples/ci-debug-trace ++ export CI_PROJECT_VISIBILITY=public diff --git a/doc/ci/variables/predefined_variables.md b/doc/ci/variables/predefined_variables.md index 8463bbeb582..98a2e7ae22f 100644 --- a/doc/ci/variables/predefined_variables.md +++ b/doc/ci/variables/predefined_variables.md @@ -23,15 +23,17 @@ future GitLab releases.** You can add a command to your `.gitlab-ci.yml` file to [output the values of all variables available for a job](README.md#list-all-environment-variables). +Kubernetes-specific environment variables are detailed in the +[Kubernetes deployment variables](../../user/project/clusters/index.md#deployment-variables) section. + | Variable | GitLab | Runner | Description | |-----------------------------------------------|--------|--------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `ARTIFACT_DOWNLOAD_ATTEMPTS` | 8.15 | 1.9 | Number of attempts to download artifacts running a job | | `CHAT_CHANNEL` | 10.6 | all | Source chat channel which triggered the [ChatOps](../chatops/README.md) command | | `CHAT_INPUT` | 10.6 | all | Additional arguments passed in the [ChatOps](../chatops/README.md) command | | `CI` | all | 0.4 | Mark that job is executed in CI environment | | `CI_API_V4_URL` | 11.7 | all | The GitLab API v4 root URL | | `CI_BUILDS_DIR` | all | 11.10 | Top-level directory where builds are executed. | -| `CI_COMMIT_BEFORE_SHA` | 11.2 | all | The previous latest commit present on a branch before a merge request. Only populated when there is a merge request associated with the pipeline. | +| `CI_COMMIT_BEFORE_SHA` | 11.2 | all | The previous latest commit present on a branch. Is always `0000000000000000000000000000000000000000` in pipelines for merge requests. | | `CI_COMMIT_DESCRIPTION` | 10.8 | all | The description of the commit: the message without first line, if the title is shorter than 100 characters; full message in other case. | | `CI_COMMIT_MESSAGE` | 10.8 | all | The full commit message. | | `CI_COMMIT_REF_NAME` | 9.0 | all | The branch or tag name for which project is built | @@ -69,8 +71,8 @@ You can add a command to your `.gitlab-ci.yml` file to | `CI_JOB_URL` | 11.1 | 0.5 | Job details URL | | `CI_KUBERNETES_ACTIVE` | 13.0 | all | Included with the value `true` only if the pipeline has a Kubernetes cluster available for deployments. Not included if no cluster is available. Can be used as an alternative to [`only:kubernetes`/`except:kubernetes`](../yaml/README.md#onlykubernetesexceptkubernetes) with [`rules:if`](../yaml/README.md#rulesif) | | `CI_MERGE_REQUEST_ASSIGNEES` | 11.9 | all | Comma-separated list of username(s) of assignee(s) for the merge request if [the pipelines are for merge requests](../merge_request_pipelines/index.md). Available only if `only: [merge_requests]` or [`rules`](../yaml/README.md#rules) syntax is used and the merge request is created. | -| `CI_MERGE_REQUEST_ID` | 11.6 | all | The ID of the merge request if [the pipelines are for merge requests](../merge_request_pipelines/index.md). Available only if `only: [merge_requests]` or [`rules`](../yaml/README.md#rules) syntax is used and the merge request is created. | -| `CI_MERGE_REQUEST_IID` | 11.6 | all | The IID of the merge request if [the pipelines are for merge requests](../merge_request_pipelines/index.md). Available only if `only: [merge_requests]` or [`rules`](../yaml/README.md#rules) syntax is used and the merge request is created. | +| `CI_MERGE_REQUEST_ID` | 11.6 | all | The project-level ID of the merge request. Only available if [the pipelines are for merge requests](../merge_request_pipelines/index.md) and the merge request is created. | +| `CI_MERGE_REQUEST_IID` | 11.6 | all | The instance-level IID of the merge request. Only available If [the pipelines are for merge requests](../merge_request_pipelines/index.md) and the merge request is created. | | `CI_MERGE_REQUEST_LABELS` | 11.9 | all | Comma-separated label names of the merge request if [the pipelines are for merge requests](../merge_request_pipelines/index.md). Available only if `only: [merge_requests]` or [`rules`](../yaml/README.md#rules) syntax is used and the merge request is created. | | `CI_MERGE_REQUEST_MILESTONE` | 11.9 | all | The milestone title of the merge request if [the pipelines are for merge requests](../merge_request_pipelines/index.md). Available only if `only: [merge_requests]` or [`rules`](../yaml/README.md#rules) syntax is used and the merge request is created. | | `CI_MERGE_REQUEST_PROJECT_ID` | 11.6 | all | The ID of the project of the merge request if [the pipelines are for merge requests](../merge_request_pipelines/index.md). Available only if `only: [merge_requests]` or [`rules`](../yaml/README.md#rules) syntax is used and the merge request is created. | @@ -92,13 +94,14 @@ You can add a command to your `.gitlab-ci.yml` file to | `CI_PAGES_URL` | 11.8 | all | URL to GitLab Pages-built pages. Always belongs to a subdomain of `CI_PAGES_DOMAIN`. | | `CI_PIPELINE_ID` | 8.10 | all | The unique ID of the current pipeline that GitLab CI/CD uses internally | | `CI_PIPELINE_IID` | 11.0 | all | The unique ID of the current pipeline scoped to project | -| `CI_PIPELINE_SOURCE` | 10.0 | all | Indicates how the pipeline was triggered. Possible options are: `push`, `web`, `trigger`, `schedule`, `api`, `pipeline`, `parent_pipeline`, `external`, `chat`, `merge_request_event`, and `external_pull_request_event`. For pipelines created before GitLab 9.5, this will show as `unknown` | +| `CI_PIPELINE_SOURCE` | 10.0 | all | Indicates how the pipeline was triggered. Possible options are: `push`, `web`, `schedule`, `api`, `external`, `chat`, `webide`, `merge_request_event`, `external_pull_request_event`, `parent_pipeline`, [`trigger`, or `pipeline`](../triggers/README.md#authentication-tokens) (renamed to `cross_project_pipeline` since 13.0). For pipelines created before GitLab 9.5, this will show as `unknown`. | | `CI_PIPELINE_TRIGGERED` | all | all | The flag to indicate that job was [triggered](../triggers/README.md) | | `CI_PIPELINE_URL` | 11.1 | 0.5 | Pipeline details URL | | `CI_PROJECT_DIR` | all | all | The full path where the repository is cloned and where the job is run. If the GitLab Runner `builds_dir` parameter is set, this variable is set relative to the value of `builds_dir`. For more information, see [Advanced configuration](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-runners-section) for GitLab Runner. | | `CI_PROJECT_ID` | all | all | The unique ID of the current project that GitLab CI/CD uses internally | | `CI_PROJECT_NAME` | 8.10 | 0.5 | The name of the directory for the project that is currently being built. For example, if the project URL is `gitlab.example.com/group-name/project-1`, the `CI_PROJECT_NAME` would be `project-1`. | | `CI_PROJECT_NAMESPACE` | 8.10 | 0.5 | The project namespace (username or group name) that is currently being built | +| `CI_PROJECT_ROOT_NAMESPACE` | 13.2 | 0.5 | The **root** project namespace (username or group name) that is currently being built. For example, if `CI_PROJECT_NAME` is `root-group/child-group/grandchild-group`, `CI_PROJECT_ROOT_NAMESPACE` would be `root-group`. | | `CI_PROJECT_PATH` | 8.10 | 0.5 | The namespace with project name | | `CI_PROJECT_PATH_SLUG` | 9.3 | all | `$CI_PROJECT_PATH` lowercased and with everything except `0-9` and `a-z` replaced with `-`. Use in URLs and domain names. | | `CI_PROJECT_REPOSITORY_LANGUAGES` | 12.3 | all | Comma-separated, lowercased list of the languages used in the repository (e.g. `ruby,javascript,html,css`) | @@ -107,8 +110,8 @@ You can add a command to your `.gitlab-ci.yml` file to | `CI_PROJECT_VISIBILITY` | 10.3 | all | The project visibility (internal, private, public) | | `CI_REGISTRY` | 8.10 | 0.5 | If the Container Registry is enabled it returns the address of GitLab's Container Registry. This variable will include a `:port` value if one has been specified in the registry configuration. | | `CI_REGISTRY_IMAGE` | 8.10 | 0.5 | If the Container Registry is enabled for the project it returns the address of the registry tied to the specific project | -| `CI_REGISTRY_PASSWORD` | 9.0 | all | The password to use to push containers to the GitLab Container Registry | -| `CI_REGISTRY_USER` | 9.0 | all | The username to use to push containers to the GitLab Container Registry | +| `CI_REGISTRY_PASSWORD` | 9.0 | all | The password to use to push containers to the GitLab Container Registry, for the current project. | +| `CI_REGISTRY_USER` | 9.0 | all | The username to use to push containers to the GitLab Container Registry, for the current project. | | `CI_REPOSITORY_URL` | 9.0 | all | The URL to clone the Git repository | | `CI_RUNNER_DESCRIPTION` | 8.10 | 0.5 | The description of the runner as saved in GitLab | | `CI_RUNNER_EXECUTABLE_ARCH` | all | 10.6 | The OS/architecture of the GitLab Runner executable (note that this is not necessarily the same as the environment of the executor) | @@ -129,11 +132,9 @@ You can add a command to your `.gitlab-ci.yml` file to | `CI_SERVER_VERSION_MINOR` | 11.4 | all | GitLab version minor component | | `CI_SERVER_VERSION_PATCH` | 11.4 | all | GitLab version patch component | | `CI_SHARED_ENVIRONMENT` | all | 10.1 | Marks that the job is executed in a shared environment (something that is persisted across CI invocations like `shell` or `ssh` executor). If the environment is shared, it is set to true, otherwise it is not defined at all. | -| `GET_SOURCES_ATTEMPTS` | 8.15 | 1.9 | Number of attempts to fetch sources running a job | | `GITLAB_CI` | all | all | Mark that job is executed in GitLab CI/CD environment | | `GITLAB_FEATURES` | 10.6 | all | The comma separated list of licensed features available for your instance and plan | | `GITLAB_USER_EMAIL` | 8.12 | all | The email of the user who started the job | | `GITLAB_USER_ID` | 8.12 | all | The ID of the user who started the job | | `GITLAB_USER_LOGIN` | 10.0 | all | The login username of the user who started the job | | `GITLAB_USER_NAME` | 10.0 | all | The real name of the user who started the job | -| `RESTORE_CACHE_ATTEMPTS` | 8.15 | 1.9 | Number of attempts to restore the cache running a job | diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md index 7ed5a8fec01..e1d1d27efed 100644 --- a/doc/ci/yaml/README.md +++ b/doc/ci/yaml/README.md @@ -5,7 +5,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w type: reference --- -# GitLab CI/CD Pipeline Configuration Reference +# GitLab CI/CD pipeline configuration reference GitLab CI/CD [pipelines](../pipelines/index.md) are configured using a YAML file called `.gitlab-ci.yml` within each project. @@ -117,7 +117,7 @@ The following table lists available parameters for jobs: | [`when`](#when) | When to run job. Also available: `when:manual` and `when:delayed`. | | [`environment`](#environment) | Name of an environment to which the job deploys. Also available: `environment:name`, `environment:url`, `environment:on_stop`, `environment:auto_stop_in` and `environment:action`. | | [`cache`](#cache) | List of files that should be cached between subsequent runs. Also available: `cache:paths`, `cache:key`, `cache:untracked`, and `cache:policy`. | -| [`artifacts`](#artifacts) | List of files and directories to attach to a job on success. Also available: `artifacts:paths`, `artifacts:exclude`, `artifacts:expose_as`, `artifacts:name`, `artifacts:untracked`, `artifacts:when`, `artifacts:expire_in`, `artifacts:reports`, `artifacts:reports:junit`, `artifacts:reports:cobertura`, and `artifacts:reports:terraform`.<br><br>In GitLab [Enterprise Edition](https://about.gitlab.com/pricing/), these are available: `artifacts:reports:codequality`, `artifacts:reports:sast`, `artifacts:reports:dependency_scanning`, `artifacts:reports:container_scanning`, `artifacts:reports:dast`, `artifacts:reports:license_scanning`, `artifacts:reports:license_management` (removed in GitLab 13.0),`artifacts:reports:performance` and `artifacts:reports:metrics`. | +| [`artifacts`](#artifacts) | List of files and directories to attach to a job on success. Also available: `artifacts:paths`, `artifacts:exclude`, `artifacts:expose_as`, `artifacts:name`, `artifacts:untracked`, `artifacts:when`, `artifacts:expire_in`, `artifacts:reports`, `artifacts:reports:codequality`, `artifacts:reports:junit`, `artifacts:reports:cobertura`, and `artifacts:reports:terraform`.<br><br>In GitLab [Enterprise Edition](https://about.gitlab.com/pricing/), these are available: `artifacts:reports:sast`, `artifacts:reports:dependency_scanning`, `artifacts:reports:container_scanning`, `artifacts:reports:dast`, `artifacts:reports:license_scanning`, `artifacts:reports:license_management` (removed in GitLab 13.0), `artifacts:reports:performance`, `artifacts:reports:load_performance`, and `artifacts:reports:metrics`. | | [`dependencies`](#dependencies) | Restrict which artifacts are passed to a specific job by providing a list of jobs to fetch artifacts from. | | [`coverage`](#coverage) | Code coverage settings for a given job. | | [`retry`](#retry) | When and how many times a job can be auto-retried in case of a failure. | @@ -130,6 +130,7 @@ The following table lists available parameters for jobs: | [`variables`](#variables) | Define job variables on a job level. | | [`interruptible`](#interruptible) | Defines if a job can be canceled when made redundant by a newer run. | | [`resource_group`](#resource_group) | Limit job concurrency. | +| [`release`](#release) | Instructs the Runner to generate a [Release](../../user/project/releases/index.md) object. | NOTE: **Note:** Parameters `types` and `type` are [deprecated](#deprecated-parameters). @@ -256,8 +257,8 @@ karma: ### `stages` -`stages` is used to define stages that can be used by jobs and is defined -globally. +`stages` is used to define stages that contain jobs and is defined +globally for the pipeline. The specification of `stages` allows for having flexible multi stage pipelines. The ordering of elements in `stages` defines the ordering of jobs' execution: @@ -297,6 +298,26 @@ determine whether or not a pipeline is created. It currently accepts a single `rules:` key that operates similarly to [`rules:` defined within jobs](#rules), enabling dynamic configuration of the pipeline. +If you are new to GitLab CI/CD and `workflow: rules`, you may find the [`workflow:rules` templates](#workflowrules-templates) useful. + +To define your own `workflow: rules`, the configuration options currently available are: + +- [`if`](#rulesif): Define a rule. +- [`when`](#when): May be set to `always` or `never` only. If not provided, the default value is `always`. + +The list of `if` rules is evaluated until a single one is matched. If none +match, the last `when` will be used: + +```yaml +workflow: + rules: + - if: $CI_COMMIT_REF_NAME =~ /-wip$/ + when: never + - if: $CI_COMMIT_TAG + when: never + - when: always +``` + #### `workflow:rules` templates > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/217732) in GitLab 13.0. @@ -334,24 +355,6 @@ include: - template: 'Workflows/MergeRequest-Pipelines.gitlab-ci.yml' ``` -If you prefer to define your own rules, the configuration options currently available are: - -- [`if`](#rulesif): Define a rule. -- [`when`](#when): May be set to `always` or `never` only. If not provided, the default value is `always`. - -The list of `if` rules is evaluated until a single one is matched. If none -match, the last `when` will be used: - -```yaml -workflow: - rules: - - if: $CI_COMMIT_REF_NAME =~ /-wip$/ - when: never - - if: $CI_COMMIT_TAG - when: never - - when: always -``` - ### `include` > - Introduced in [GitLab Premium](https://about.gitlab.com/pricing/) 10.5. @@ -375,12 +378,14 @@ otherwise the external file won't be included. | [`remote`](#includeremote) | Include a file from a remote URL. Must be publicly accessible. | | [`template`](#includetemplate) | Include templates which are provided by GitLab. | +The `include` methods do not support [variable expansion](../variables/where_variables_can_be_used.md#variables-usage). + NOTE: **Note:** `.gitlab-ci.yml` configuration included by all methods is evaluated at pipeline creation. The configuration is a snapshot in time and persisted in the database. Any changes to referenced `.gitlab-ci.yml` configuration won't be reflected in GitLab until the next pipeline is created. -The files defined in `include` are: +The files defined by `include` are: - Deep merged with those in `.gitlab-ci.yml`. - Always evaluated first and merged with the content of `.gitlab-ci.yml`, @@ -388,11 +393,11 @@ The files defined in `include` are: TIP: **Tip:** Use merging to customize and override included CI/CD configurations with local -definitions. +definitions. Local definitions in `.gitlab-ci.yml` will override included definitions. NOTE: **Note:** -Using YAML aliases across different YAML files sourced by `include` is not -supported. You must only refer to aliases in the same file. Instead +Using [YAML anchors](#anchors) across different YAML files sourced by `include` is not +supported. You must only refer to anchors in the same file. Instead of using YAML anchors, you can use the [`extends` keyword](#extends). #### `include:local` @@ -699,6 +704,101 @@ job: - Write-Host "This text is not colored" ``` +#### Multiline commands + +You can split long commands into multi-line commands to improve readability +using [`|` (literal) and `>` (folded) YAML multiline block scalar indicators](https://yaml-multiline.info/). + +CAUTION: **Warning:** +If multiple commands are combined into one command string, only the last command's +failure or success will be reported, +[incorrectly ignoring failures from earlier commands due to a bug](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/25394). +If the success of the job depends on the success or failure of these commands, +you can run the commands as separate `script:` items, or add `exit 1` commands +as appropriate to the command string where needed. + +You can use the `|` (literal) YAML multiline block scalar indicator to write +commands over multiple lines in the `script` section of a job description. +Each line is treated as a separate command. +Only the first command is repeated in the job log, but additional +commands are still executed: + +```yaml +job: + script: + - | + echo "First command line." + echo "Second command line." + echo "Third command line." +``` + +The example above renders in the job log as: + +```shell +$ echo First command line # collapsed multi-line command +First command line +Second command line. +Third command line. +``` + +The `>` (folded) YAML multiline block scalar indicator treats empty lines between +sections as the start of a new command: + +```yaml +job: + script: + - > + echo "First command line + is split over two lines." + + echo "Second command line." +``` + +This behaves similarly to writing multiline commands without the `>` or `|` block +scalar indicators: + +```yaml +job: + script: + - echo "First command line + is split over two lines." + + echo "Second command line." +``` + +Both examples above render in the job log as: + +```shell +$ echo First command line is split over two lines. # collapsed multi-line command +First command line is split over two lines. +Second command line. +``` + +When the `>` or `|` block scalar indicators are omitted, GitLab will form the command +by concatenating non-empty lines, so make sure the lines can run when concatenated. + +Shell [here documents](https://en.wikipedia.org/wiki/Here_document) work with the +`|` and `>` operators as well. The example below transliterates the lower case letters +to upper case: + +```yaml +job: + script: + - | + tr a-z A-Z << END_TEXT + one two three + four five six + END_TEXT +``` + +Results in: + +```shell +$ tr a-z A-Z << END_TEXT # collapsed multi-line command + ONE TWO THREE + FOUR FIVE SIX +``` + ### `stage` `stage` is defined per-job and relies on [`stages`](#stages) which is defined @@ -874,24 +974,38 @@ spinach: ``` In GitLab 12.0 and later, it's also possible to use multiple parents for -`extends`. The algorithm used for merge is "closest scope wins", so -keys from the last member will always shadow anything defined on other +`extends`. + +#### Merge details + +`extends` is able to merge hashes but not arrays. +The algorithm used for merge is "closest scope wins", so +keys from the last member will always override anything defined on other levels. For example: ```yaml .only-important: + variables: + URL: "http://my-url.internal" + IMPORTANT_VAR: "the details" only: - master - stable tags: - production + script: + - echo "Hello world!" .in-docker: + variables: + URL: "http://docker-url.internal" tags: - docker image: alpine rspec: + variables: + GITLAB: "is-awesome" extends: - .only-important - .in-docker @@ -903,6 +1017,10 @@ This results in the following `rspec` job: ```yaml rspec: + variables: + URL: "http://docker-url.internal" + IMPORTANT_VAR: "the details" + GITLAB: "is-awesome" only: - master - stable @@ -913,6 +1031,15 @@ rspec: - rake rspec ``` +Note that in the example above: + +- `variables` sections have been merged but that `URL: "http://my-url.internal"` +has been overwritten by `URL: "http://docker-url.internal"`. +- `tags: ['production']` has been overwritten by `tags: ['docker']`. +- `script` has not been merged but rather `script: ['echo "Hello world!"']` has + been overwritten by `script: ['rake rspec']`. Arrays can be + merged using [YAML anchors](#anchors). + #### Using `extends` and `include` together `extends` works across configuration files combined with `include`. @@ -942,113 +1069,295 @@ the `.template` job, and uses the `alpine` Docker image as defined in the local > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/27863) in GitLab 12.3. -`rules` allows for a list of individual rule objects to be evaluated -*in order*, until one matches and dynamically provides attributes to the job. +The `rules` keyword can be used to include or exclude jobs in pipelines. + +Rules are evaluated *in order* until the first match. When matched, the job +is either included or excluded from the pipeline, depending on the configuration. +If included, the job also has [certain attributes](#rules-attributes) +added to it. CAUTION: **Caution:** -`rules` can't be used in combination with `only/except` as it is a replacement for that functionality. If you attempt to do this, the linter will return a +`rules` can't be used in combination with [`only/except`](#onlyexcept-basic) because it is a replacement for +that functionality. If you attempt to do this, the linter returns a `key may not be used with rules` error. -#### Key details when using `rules` +#### Rules attributes -A very important difference between `rules` and `only/except`, is that jobs defined -with `rules` trigger merge request pipelines by default, but `only/except` jobs do not. -This may be surprising if migrating from `only` and `except`, so new users of `rules` -can use one of the [`workflow: rules` templates](#workflowrules-templates) to get started. -This will ensure that the behavior is more stable as you start adding additional `rules` -blocks, and will avoid issues like creating a duplicate, merge request (detached) pipeline. +The job attributes allowed by `rules` are: -We don't recommend mixing `only/except` jobs with `rules` jobs in the same pipeline. -It may not cause YAML errors, but debugging the exact execution behavior can be complex -due to the different default behaviors of `only/except` and `rules`. +- [`when`](#when): If not defined, defaults to `when: on_success`. + - If used as `when: delayed`, `start_in` is also required. +- [`allow_failure`](#allow_failure): If not defined, defaults to `allow_failure: false`. + +If a rule evaluates to true, and `when` has any value except `never`, the job is included in the pipeline. + +For example: + +```yaml +docker build: + script: docker build -t my-image:$CI_COMMIT_REF_SLUG . + rules: + - if: '$CI_COMMIT_BRANCH == "master"' + when: delayed + start_in: '3 hours' + allow_failure: true +``` + +Additional job configuration may be added to rules in the future. If something +useful is not available, please [open an issue](https://gitlab.com/gitlab-org/gitlab/-/issues). + +#### Rules clauses + +Available rule clauses are: + +| Clause | Description | +|----------------------------|------------------------------------------------------------------------------------------------------------------------------------| +| [`if`](#rulesif) | Add or exclude jobs from a pipeline by evaluating an `if` statement. Similar to [`only:variables`](#onlyvariablesexceptvariables). | +| [`changes`](#ruleschanges) | Add or exclude jobs from a pipeline based on what files are changed. Same as [`only:changes`](#onlychangesexceptchanges). | +| [`exists`](#rulesexists) | Add or exclude jobs from a pipeline based on the presence of specific files. | -### Rules clauses +Rules are evaluated in order until a match is found. If a match is found, the attributes +are checked to see if the job should be added to the pipeline. If no attributes are defined, +the defaults are: -Available rule clauses include: +- `when: on_success` +- `allow_failure: false` -- [`if`](#rulesif) (similar to [`only:variables`](#onlyvariablesexceptvariables)) -- [`changes`](#ruleschanges) (same as [`only:changes`](#onlychangesexceptchanges)) -- [`exists`](#rulesexists) +The job is added to the pipeline: -For example, using `if`. This configuration specifies that `job` should be built -and run for every pipeline on merge requests targeting `master`, regardless of -the status of other builds: +- If a rule matches and has `when: on_success`, `when: delayed` or `when: always`. +- If no rules match, but the last clause is `when: on_success`, `when: delayed` + or `when: always` (with no rule). + +The job is not added to the pipeline: + +- If no rules match, and there is no standalone `when: on_success`, `when: delayed` or + `when: always`. +- If a rule matches, and has `when: never` as the attribute. + +For example, using `if` clauses to strictly limit when jobs run: ```yaml job: script: "echo Hello, Rules!" rules: - - if: '$CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "master"' - when: always - - if: '$VAR =~ /pattern/' + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' when: manual + allow_failure: true + - if: '$CI_PIPELINE_SOURCE == "schedule"' +``` + +In this example: + +- If the pipeline is for a merge request, the first rule matches, and the job + is added to the [merge request pipeline](../merge_request_pipelines/index.md) + with attributes of: + - `when: manual` (manual job) + - `allow_failure: true` (allows the pipeline to continue running even if the manual job is not run) +- If the pipeline is **not** for a merge request, the first rule doesn't match, and the + second rule is evaluated. +- If the pipeline is a scheduled pipeline, the second rule matches, and the job + is added to the scheduled pipeline. Since no attributes were defined, it is added + with: + - `when: on_success` (default) + - `allow_failure: false` (default) +- In **all other cases**, no rules match, so the job is **not** added to any other pipeline. + +Alternatively, you can define a set of rules to exclude jobs in a few cases, but +run them in all other cases: + +```yaml +job: + script: "echo Hello, Rules!" + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + when: never + - if: '$CI_PIPELINE_SOURCE == "schedule"' + when: never - when: on_success ``` -In this example, if the first rule: +- If the pipeline is for a merge request, the job is **not** be added to the pipeline. +- If the pipeline is a scheduled pipeline, the job is **not** be added to the pipeline. +- In **all other cases**, the job is added to the pipeline, with `when: on_success`. + +CAUTION: **Caution:** +If you use `when: on_success`, `always`, or `delayed` as the final rule, two +simultaneous pipelines may start. Both push pipelines and merge request pipelines can +be triggered by the same event (a push to the source branch for an open merge request). +See the [important differences between `rules` and `only`/`except`](#differences-between-rules-and-onlyexcept) +for more details. + +#### Differences between `rules` and `only`/`except` + +Jobs defined with `only/except` do not trigger merge request pipelines by default. +You must explicitly add `only: merge_requests`. + +Jobs defined with `rules` can trigger all types of pipelines. +You do not have to explicitly configure each type. + +For example: + +```yaml +job: + script: "echo This creates double pipelines!" + rules: + - if: '$CUSTOM_VARIABLE == "false"' + when: never + - when: always +``` + +This job does not run when `$CUSTOM_VARIABLE` is false, but it *does* run in **all** +other pipelines, including **both** push (branch) and merge request pipelines. With +this configuration, every push to an open merge request's source branch +causes duplicated pipelines. Explicitly allowing both push and merge request pipelines +in the same job could have the same effect. + +We recommend using [`workflow: rules`](#workflowrules) to limit which types of pipelines +are permitted. Allowing only merge request pipelines, or only branch pipelines, +eliminates duplicated pipelines. Alternatively, you can rewrite the rules to be +stricter, or avoid using a final `when` (`always`, `on_success` or `delayed`). + +Also, we don't recommend mixing `only/except` jobs with `rules` jobs in the same pipeline. +It may not cause YAML errors, but debugging the exact execution behavior can be complex +due to the different default behaviors of `only/except` and `rules`. + +##### `rules:if` -- Matches, the job will be given the `when:always` attribute. -- Does not match, the second and third rules will be evaluated sequentially - until a match is found. That is, the job will be given either the: - - `when: manual` attribute if the second rule matches. **The stage won't complete until this manual job is triggered and completes successfully.** - - `when: on_success` attribute if the second rule does not match. The third - rule will always match when reached because it has no conditional clauses. +`rules:if` clauses determine whether or not jobs are added to a pipeline by evaluating +a simple `if` statement. If the `if` statement is true, the job is either included +or excluded from a pipeline. In plain English, `if` rules can be interpreted as one of: -#### `rules:if` +- "If this rule evaluates to true, add the job" (default). +- "If this rule evaluates to true, do not add the job" (by adding `when: never`). `rules:if` differs slightly from `only:variables` by accepting only a single -expression string, rather than an array of them. Any set of expressions to be -evaluated should be conjoined into a single expression using `&&` or `||`, and use +expression string per rule, rather than an array of them. Any set of expressions to be +evaluated can be conjoined into a single expression by using `&&` or `||`, and use the [variable matching syntax](../variables/README.md#syntax-of-environment-variable-expressions). +`if:` clauses are evaluated based on the values of [predefined environment variables](../variables/predefined_variables.md) +or [custom environment variables](../variables/README.md#custom-environment-variables). + For example: ```yaml job: script: "echo Hello, Rules!" rules: - - if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME =~ /^feature/ && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "master"' # This rule will be evaluated + - if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME =~ /^feature/ && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "master"' when: always - - if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME =~ /^feature/' # This rule will only be evaluated if the target branch is not "master" + - if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME =~ /^feature/' when: manual - - if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME' # If neither of the first two match but the simple presence does, we set to "on_success" by default + allow_failure: true + - if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME' # Checking for the presence of a variable is possible ``` Some details regarding the logic that determines the `when` for the job: -- If none of the provided rules match, the job is set to `when: never`, and is +- If none of the provided rules match, the job is set to `when: never` and is not included in the pipeline. - A rule without any conditional clause, such as a `when` or `allow_failure` rule without `if` or `changes`, always matches, and is always used if reached. -- If a rule matches and has no `when` defined, the rule will use the `when` +- If a rule matches and has no `when` defined, the rule uses the `when` defined for the job, which defaults to `on_success` if not defined. +- You can define `when` once per rule, or once at the job-level, which applies to + all rules. You can't mix `when` at the job-level with `when` in rules. + +For behavior similar to the [`only`/`except` keywords](#onlyexcept-basic), you can +check the value of the `$CI_PIPELINE_SOURCE` variable. + +| Value | Description | +|-------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `push` | For pipelines triggered by a `git push` event, including for branches and tags. | +| `web` | For pipelines created by using **Run pipeline** button in the GitLab UI, from the project's **CI/CD > Pipelines** section. | +| `trigger` | For pipelines created by using a trigger token. | +| `schedule` | For [scheduled pipelines](../pipelines/schedules.md). | +| `api` | For pipelines triggered by the [pipelines API](../../api/pipelines.md#create-a-new-pipeline). | +| `external` | When using CI services other than GitLab. | +| `pipelines` | For multi-project pipelines created by [using the API with `CI_JOB_TOKEN`](../triggers/README.md#when-used-with-multi-project-pipelines). | +| `chat` | For pipelines created by using a [GitLab ChatOps](../chatops/README.md) command. | +| `webide` | For pipelines created by using the [WebIDE](../../user/project/web_ide/index.md). | +| `merge_request_event` | For pipelines created when a merge request is created or updated. Required to enable [merge request pipelines](../merge_request_pipelines/index.md), [merged results pipelines](../merge_request_pipelines/pipelines_for_merged_results/index.md), and [merge trains](../merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md). | +| `external_pull_request_event` | When an external pull request on GitHub is created or updated. See [Pipelines for external pull requests](../ci_cd_for_external_repos/index.md#pipelines-for-external-pull-requests). | +| `parent_pipeline` | For pipelines triggered by a [parent/child pipeline](../parent_child_pipelines.md) with `rules`, use this in the child pipeline configuration so that it can be triggered by the parent pipeline. | + +For example: + +```yaml +job: + script: "echo Hello, Rules!" + rules: + - if: '$CI_PIPELINE_SOURCE == "schedule"' + when: manual + allow_failure: true + - if: '$CI_PIPELINE_SOURCE == "push"' +``` + +This example runs the job as a manual job in scheduled pipelines or in push +pipelines (to branches or tags), with `when: on_success` (default). It does not +add the job to any other pipeline type. + +Another example: + +```yaml +job: + script: "echo Hello, Rules!" + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + - if: '$CI_PIPELINE_SOURCE == "schedule"' +``` + +This example runs the job as a `when: on_success` job in [merge request pipelines](../merge_request_pipelines/index.md) +and scheduled pipelines. It does not run in any other pipeline type. -#### `rules:changes` +Other commonly used variables for `if` clauses: -`rules: changes` works exactly the same way as `only: changes` and `except: changes`, -accepting an array of paths. Similarly, it will always return true if there is no -Git push event. See [`only/except: changes`](#onlychangesexceptchanges) for more information. +- `if: $CI_COMMIT_TAG`: If changes are pushed for a tag. +- `if: $CI_COMMIT_BRANCH`: If changes are pushed to any branch. +- `if: '$CI_COMMIT_BRANCH == "master"'`: If changes are pushed to `master`. +- `if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'`: If changes are pushed to the default + branch (usually `master`). Useful if reusing the same configuration in multiple + projects with potentially different default branches. +- `if: '$CI_COMMIT_BRANCH =~ /regex-expression/'`: If the commit branch matches a regular expression. +- `if: '$CUSTOM_VARIABLE !~ /regex-expression/'`: If the [custom variable](../variables/README.md#custom-environment-variables) + `CUSTOM_VARIABLE` does **not** match a regular expression. +- `if: '$CUSTOM_VARIABLE == "value1"'`: If the custom variable `CUSTOM_VARIABLE` is + exactly `value1`. + +##### `rules:changes` + +To determine if jobs should be added to a pipeline, `rules: changes` clauses check +the files changed by Git push events. + +`rules: changes` works exactly the same way as [`only: changes` and `except: changes`](#onlychangesexceptchanges), +accepting an array of paths. Similarly, it always returns true if there is no +Git push event. It should only be used for branch pipelines or merge request pipelines. For example: ```yaml +workflow: + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + docker build: script: docker build -t my-image:$CI_COMMIT_REF_SLUG . rules: - - changes: # Will include the job and set to when:manual if any of the follow paths match a modified file. - - Dockerfile + - changes: + - Dockerfile when: manual - - if: '$VAR == "string value"' - when: manual # Will include the job and set to when:manual if the expression evaluates to true, after the `changes:` rule fails to match. - - when: on_success # If neither of the first rules match, set to on_success + allow_failure: true ``` -In this example, a job either set to: +In this example: -- Run manually if `Dockerfile` has changed OR `$VAR == "string value"`. -- `when:on_success` by the last rule, where no earlier clauses evaluate to true. +- [`workflow: rules`](#workflowrules) allows only pipelines for merge requests for all jobs. +- If `Dockerfile` has changed, add the job to the pipeline as a manual job, and allow the pipeline + to continue running even if the job is not triggered (`allow_failure: true`). +- If `Dockerfile` has not changed, do not add job to any pipeline (same as `when: never`). -#### `rules:exists` +##### `rules:exists` > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/24021) in GitLab 12.4. @@ -1062,7 +1371,7 @@ job: script: docker build -t my-image:$CI_COMMIT_REF_SLUG . rules: - exists: - - Dockerfile + - Dockerfile ``` You can also use glob patterns to match multiple files in any directory within @@ -1075,14 +1384,14 @@ job: script: bundle exec rspec rules: - exists: - - spec/**.rb + - spec/**.rb ``` NOTE: **Note:** For performance reasons, using `exists` with patterns is limited to 10000 checks. After the 10000th check, rules with patterned globs will always match. -#### `rules:allow_failure` +##### `rules:allow_failure` > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/30235) in GitLab 12.8. @@ -1122,53 +1431,18 @@ docker build: rules: - if: '$VAR == "string value"' changes: # Will include the job and set to when:manual if any of the follow paths match a modified file. - - Dockerfile - - docker/scripts/* + - Dockerfile + - docker/scripts/* when: manual # - when: never would be redundant here, this is implied any time rules are listed. ``` -The only clauses currently available are: - -- `if` -- `changes` -- `exists` - Keywords such as `branches` or `refs` that are currently available for `only`/`except` are not yet available in `rules` as they are being individually considered for their usage and behavior in this context. Future keyword improvements are being discussed in our [epic for improving `rules`](https://gitlab.com/groups/gitlab-org/-/epics/2783), where anyone can add suggestions or requests. -#### Permitted attributes - -The only job attributes currently set by `rules` are: - -- `when`. -- `start_in`, if `when` is set to `delayed`. -- `allow_failure`. - -A job will be included in a pipeline if `when` is evaluated to any value -except `never`. - -Delayed jobs require a `start_in` value, so rule objects do as well. For -example: - -```yaml -docker build: - script: docker build -t my-image:$CI_COMMIT_REF_SLUG . - rules: - - changes: # Will include the job and delay 3 hours when the Dockerfile has changed - - Dockerfile - when: delayed - start_in: '3 hours' - - when: on_success # Otherwise include the job and set to run normally -``` - -Additional job configuration may be added to rules in the future. If something -useful is not available, please -[open an issue](https://gitlab.com/gitlab-org/gitlab/-/issues). - ### `only`/`except` (basic) NOTE: **Note:** @@ -1193,20 +1467,20 @@ There are a few rules that apply to the usage of job policy: In addition, `only` and `except` allow the use of special keywords: -| **Value** | **Description** | -| --------- | ---------------- | -| `branches` | When a Git reference of a pipeline is a branch. | -| `tags` | When a Git reference of a pipeline is a tag. | -| `api` | When pipeline has been triggered by a second pipelines API (not triggers API). | -| `external` | When using CI services other than GitLab. | -| `pipelines` | For multi-project triggers, created using the API with `CI_JOB_TOKEN`. | -| `pushes` | Pipeline is triggered by a `git push` by the user. | -| `schedules` | For [scheduled pipelines](../pipelines/schedules.md). | -| `triggers` | For pipelines created using a trigger token. | -| `web` | For pipelines created using **Run pipeline** button in GitLab UI (under your project's **Pipelines**). | -| `merge_requests` | When a merge request is created or updated (See [pipelines for merge requests](../merge_request_pipelines/index.md)). | -| `external_pull_requests`| When an external pull request on GitHub is created or updated (See [Pipelines for external pull requests](../ci_cd_for_external_repos/index.md#pipelines-for-external-pull-requests)). | -| `chat` | For jobs created using a [GitLab ChatOps](../chatops/README.md) command. | +| **Value** | **Description** | +|--------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `branches` | When the Git reference for a pipeline is a branch. | +| `tags` | When the Git reference for a pipeline is a tag. | +| `api` | For pipelines triggered by the [pipelines API](../../api/pipelines.md#create-a-new-pipeline). | +| `external` | When using CI services other than GitLab. | +| `pipelines` | For multi-project pipelines created by using the API with `CI_JOB_TOKEN`. | +| `pushes` | For pipelines triggered by a `git push` event, including for branches and tags. | +| `schedules` | For [scheduled pipelines](../pipelines/schedules.md). | +| `triggers` | For pipelines created by using a trigger token. | +| `web` | For pipelines created by using **Run pipeline** button in the GitLab UI, from the project's **CI/CD > Pipelines** section. | +| `merge_requests` | For pipelines created when a merge request is created or updated. Enables [merge request pipelines](../merge_request_pipelines/index.md), [merged results pipelines](../merge_request_pipelines/pipelines_for_merged_results/index.md), and [merge trains](../merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md). | +| `external_pull_requests` | When an external pull request on GitHub is created or updated (See [Pipelines for external pull requests](../ci_cd_for_external_repos/index.md#pipelines-for-external-pull-requests)). | +| `chat` | For pipelines created by using a [GitLab ChatOps](../chatops/README.md) command. | In the example below, `job` will run only for refs that start with `issue-`, whereas all branches will be skipped: @@ -1295,7 +1569,7 @@ and must be surrounded by `/`. So `issue-/.*/` won't work to match all tag names or branch names that begin with `issue-`. -TIP: **Tip** +TIP: **Tip:** Use anchors `^` and `$` to avoid the regular expression matching only a substring of the tag name or branch name. For example, `/^issue-.*$/` is equivalent to `/^issue-/`, @@ -1701,8 +1975,11 @@ Feature::enable(:ci_dag_limit_needs) > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/14311) in GitLab v12.6. -When using `needs`, artifact downloads are controlled with `artifacts: true` or `artifacts: false`. -The `dependencies` keyword should not be used with `needs`, as this is deprecated since GitLab 12.6. +When using `needs`, artifact downloads are controlled with `artifacts: true` (default) or `artifacts: false`. + +Since GitLab 12.6, you can't combine the [`dependencies`](#dependencies) keyword +with `needs` to control artifact downloads in jobs. `dependencies` is still valid +in jobs that do not use `needs`. In the example below, the `rspec` job will download the `build_job` artifacts, while the `rubocop` job won't: @@ -2160,7 +2437,8 @@ review_app: stage: deploy script: make deploy-app environment: - name: review + name: review/$CI_COMMIT_REF_NAME + url: https://$CI_ENVIRONMENT_SLUG.example.com on_stop: stop_review_app stop_review_app: @@ -2170,7 +2448,7 @@ stop_review_app: script: make delete-app when: manual environment: - name: review + name: review/$CI_COMMIT_REF_NAME action: stop ``` @@ -2195,8 +2473,6 @@ The `stop_review_app` job is **required** to have the following keywords defined - `when` - [reference](#when) - `environment:name` - `environment:action` -- `stage` should be the same as the `review_app` in order for the environment - to stop automatically when the branch is deleted Additionally, both jobs should have matching [`rules`](../yaml/README.md#onlyexcept-basic) or [`only/except`](../yaml/README.md#onlyexcept-basic) configuration. In the example @@ -2554,7 +2830,12 @@ be available for download in the GitLab UI. Paths are relative to the project directory (`$CI_PROJECT_DIR`) and can't directly link outside it. Wildcards can be used that follow the [glob](https://en.wikipedia.org/wiki/Glob_(programming)) -patterns and [`filepath.Match`](https://golang.org/pkg/path/filepath/#Match). +patterns and: + +- In [GitLab Runner 13.0](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/2620) and later, +[`doublestar.Glob`](https://pkg.go.dev/github.com/bmatcuk/doublestar@v1.2.2?tab=doc#Match). +- In GitLab Runner 12.10 and earlier, +[`filepath.Match`](https://pkg.go.dev/path/filepath/#Match). To restrict which jobs a specific job will fetch artifacts from, see [dependencies](#dependencies). @@ -2645,10 +2926,10 @@ For example, to match a single file: ```yaml test: - script: [ 'echo 1' ] + script: [ "echo 'test' > file.txt" ] artifacts: expose_as: 'artifact 1' - paths: ['path/to/file.txt'] + paths: ['file.txt'] ``` With this configuration, GitLab will add a link **artifact 1** to the relevant merge request @@ -2658,14 +2939,15 @@ An example that will match an entire directory: ```yaml test: - script: [ 'echo 1' ] + script: [ "mkdir test && echo 'test' > test/file.txt" ] artifacts: expose_as: 'artifact 1' - paths: ['path/to/directory/'] + paths: ['test/'] ``` Note the following: +- Artifacts do not display in the merge request UI when using variables to define the `artifacts:paths`. - A maximum of 10 job artifacts per merge request can be exposed. - Glob patterns are unsupported. - If a directory is specified, the link will be to the job [artifacts browser](../pipelines/job_artifacts.md#browsing-artifacts) if there is more than @@ -2818,7 +3100,7 @@ job: expire and are therefore deleted, counting from the time they are uploaded and stored on GitLab. If the expiry time is not defined, it defaults to the [instance wide setting](../../user/admin_area/settings/continuous_integration.md#default-artifacts-expiration-core-only) -(30 days by default, forever on GitLab.com). +(30 days by default). You can use the **Keep** button on the job page to override expiration and keep artifacts forever. @@ -2846,8 +3128,10 @@ job: ``` NOTE: **Note:** -For artifacts created in [GitLab 13.1](https://gitlab.com/gitlab-org/gitlab/-/issues/16267) -and later, the latest artifact for a ref is always kept, regardless of the expiry time. +Since [GitLab 13.0](https://gitlab.com/gitlab-org/gitlab/-/issues/16267), the latest +artifacts for refs can be locked against deletion, and kept regardless of the expiry time. This feature is disabled +by default and is not ready for production use. It can be enabled for testing by +enabling the `:keep_latest_artifact_for_ref` and `:destroy_only_unlocked_expired_artifacts` [feature flags](../../administration/feature_flags.md). #### `artifacts:reports` @@ -2863,14 +3147,15 @@ These are the available report types: | [`artifacts:reports:dotenv`](../pipelines/job_artifacts.md#artifactsreportsdotenv) | The `dotenv` report collects a set of environment variables. | | [`artifacts:reports:cobertura`](../pipelines/job_artifacts.md#artifactsreportscobertura) | The `cobertura` report collects Cobertura coverage XML files. | | [`artifacts:reports:terraform`](../pipelines/job_artifacts.md#artifactsreportsterraform) | The `terraform` report collects Terraform `tfplan.json` files. | -| [`artifacts:reports:codequality`](../pipelines/job_artifacts.md#artifactsreportscodequality-starter) **(STARTER)** | The `codequality` report collects CodeQuality issues. | +| [`artifacts:reports:codequality`](../pipelines/job_artifacts.md#artifactsreportscodequality) | The `codequality` report collects CodeQuality issues. | | [`artifacts:reports:sast`](../pipelines/job_artifacts.md#artifactsreportssast-ultimate) **(ULTIMATE)** | The `sast` report collects Static Application Security Testing vulnerabilities. | | [`artifacts:reports:dependency_scanning`](../pipelines/job_artifacts.md#artifactsreportsdependency_scanning-ultimate) **(ULTIMATE)** | The `dependency_scanning` report collects Dependency Scanning vulnerabilities. | | [`artifacts:reports:container_scanning`](../pipelines/job_artifacts.md#artifactsreportscontainer_scanning-ultimate) **(ULTIMATE)** | The `container_scanning` report collects Container Scanning vulnerabilities. | | [`artifacts:reports:dast`](../pipelines/job_artifacts.md#artifactsreportsdast-ultimate) **(ULTIMATE)** | The `dast` report collects Dynamic Application Security Testing vulnerabilities. | | [`artifacts:reports:license_management`](../pipelines/job_artifacts.md#artifactsreportslicense_management-ultimate) **(ULTIMATE)** | The `license_management` report collects Licenses (*removed from GitLab 13.0*). | | [`artifacts:reports:license_scanning`](../pipelines/job_artifacts.md#artifactsreportslicense_scanning-ultimate) **(ULTIMATE)** | The `license_scanning` report collects Licenses. | -| [`artifacts:reports:performance`](../pipelines/job_artifacts.md#artifactsreportsperformance-premium) **(PREMIUM)** | The `performance` report collects Performance metrics. | +| [`artifacts:reports:performance`](../pipelines/job_artifacts.md#artifactsreportsperformance-premium) **(PREMIUM)** | The `performance` report collects Browser Performance metrics. | +| [`artifacts:reports:load_performance`](../pipelines/job_artifacts.md#artifactsreportsload_performance-premium) **(PREMIUM)** | The `load_performance` report collects load performance metrics. | | [`artifacts:reports:metrics`](../pipelines/job_artifacts.md#artifactsreportsmetrics-premium) **(PREMIUM)** | The `metrics` report collects Metrics. | #### `dependencies` @@ -3127,6 +3412,10 @@ This keyword allows the creation of two different types of downstream pipelines: - [Multi-project pipelines](../multi_project_pipelines.md#creating-multi-project-pipelines-from-gitlab-ciyml) - [Child pipelines](../parent_child_pipelines.md) +[Since GitLab 13.2](https://gitlab.com/gitlab-org/gitlab/-/issues/197140/), you can +see which job triggered a downstream pipeline by hovering your mouse cursor over +the downstream pipeline job in the [pipeline graph](../pipelines/index.md#visualize-pipelines). + NOTE: **Note:** Using a `trigger` with `when:manual` together results in the error `jobs:#{job-name} when should be on_success, on_failure or always`, because `when:manual` prevents @@ -3271,7 +3560,7 @@ is enabled. When enabled, a pipeline on the same branch will be canceled when: -- it's made redundant by a newer pipeline run. +- It's made redundant by a newer pipeline run. - Either all jobs are set as interruptible, or any uninterruptible jobs haven't started. Pending jobs are always considered interruptible. @@ -3352,6 +3641,182 @@ It can't start or end with `/`. For more information, see [Deployments Safety](../environments/deployment_safety.md). +### `release` + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/merge_requests/19298) in GitLab 13.2. + +`release` indicates that the job creates a [Release](../../user/project/releases/index.md), +and optionally includes URLs for Release assets. + +These methods are supported: + +- [`tag_name`](#releasetag_name) +- [`name`](#releasename) (optional) +- [`description`](#releasedescription) (optional) +- [`ref`](#releaseref) (optional) +- [`milestones`](#releasemilestones) (optional) +- [`released_at`](#releasereleased_at) (optional) + +The Release is created only if the job processes without error. If the Rails API +returns an error during Release creation, the `release` job fails. + +#### `release-cli` Docker image + +The Docker image to use for the `release-cli` must be specified, using the following directive: + +```yaml +image: registry.gitlab.com/gitlab-org/release-cli:latest +``` + +#### Script + +All jobs require a `script` tag at a minimum. A `:release` job can use the output of a +`:script` tag, but if this is not necessary, a placeholder script can be used, for example: + +```yaml +script: + - echo 'release job' +``` + +An [issue](https://gitlab.com/gitlab-org/gitlab/-/issues/223856) exists to remove this requirement in an upcoming version of GitLab. + +A pipeline can have multiple `release` jobs, for example: + +```yaml +ios-release: + script: + - echo 'iOS release job' + release: + tag_name: v1.0.0-ios + description: 'iOS release v1.0.0' + +android-release: + script: + - echo 'Android release job' + release: + tag_name: v1.0.0-android + description: 'Android release v1.0.0' +``` + +#### `release:tag_name` + +The `tag_name` must be specified. It can refer to an existing Git tag or can be specified by the user. + +When the specified tag doesn't exist in the repository, a new tag is created from the associated SHA of the pipeline. + +For example, when creating a Release from a Git tag: + +```yaml +job: + release: + tag_name: $CI_COMMIT_TAG + description: changelog.txt +``` + +It is also possible to create any unique tag, in which case `only: tags` is not mandatory. +A semantic versioning example: + +```yaml +job: + release: + tag_name: ${MAJOR}_${MINOR}_${REVISION} + description: changelog.txt +``` + +- The Release is created only if the job's main script succeeds. +- If the Release already exists, it is not updated and the job with the `release` keyword fails. +- The `release` section executes after the `script` tag and before the `after_script`. + +#### `release:name` + +The Release name. If omitted, it is populated with the value of `release: tag_name`. + +#### `release:description` + +Specifies the longer description of the Release. + +#### `release:ref` + +If the `release: tag_name` doesn’t exist yet, the release is created from `ref`. +`ref` can be a commit SHA, another tag name, or a branch name. + +#### `release:milestones` + +The title of each milestone the release is associated with. + +#### `release:released_at` + +The date and time when the release is ready. Defaults to the current date and time if not +defined. Expected in ISO 8601 format (2019-03-15T08:00:00Z). + +#### Complete example for `release` + +Combining the individual examples given above for `release` results in the following +code snippets. There are two options, depending on how you generate the +tags. These options cannot be used together, so choose one: + +- To create a release when you push a Git tag, or when you add a Git tag + in the UI by going to **Repository > Tags**: + + ```yaml + release_job: + stage: release + image: registry.gitlab.com/gitlab-org/release-cli:latest + rules: + - if: $CI_COMMIT_TAG # Run this job when a tag is created manually + script: + - echo 'running release_job' + release: + name: 'Release $CI_COMMIT_TAG' + description: 'Created using the release-cli $EXTRA_DESCRIPTION' # $EXTRA_DESCRIPTION must be defined + tag_name: '$CI_COMMIT_TAG' # elsewhere in the pipeline. + ref: '$CI_COMMIT_TAG' + milestones: + - 'm1' + - 'm2' + - 'm3' + released_at: '2020-07-15T08:00:00Z' # Optional, will auto generate if not defined, + # or can use a variable. + ``` + +- To create a release automatically when changes are pushed to the default branch, + using a new Git tag that is defined with variables: + + ```yaml + release_job: + stage: release + image: registry.gitlab.com/gitlab-org/release-cli:latest + rules: + - if: $CI_COMMIT_TAG + when: never # Do not run this job when a tag is created manually + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Run this job when the default branch changes + script: + - echo 'running release_job' + release: + name: 'Release $CI_COMMIT_SHA' + description: 'Created using the release-cli $EXTRA_DESCRIPTION' # $EXTRA_DESCRIPTION and the tag_name + tag_name: 'v${MAJOR}.${MINOR}.${REVISION}' # variables must be defined elsewhere + ref: '$CI_COMMIT_SHA' # in the pipeline. + milestones: + - 'm1' + - 'm2' + - 'm3' + released_at: '2020-07-15T08:00:00Z' # Optional, will auto generate if not defined, + # or can use a variable. + ``` + +#### `releaser-cli` command line + +The entries under the `:release` node are transformed into a `bash` command line and sent +to the Docker container, which contains the [release-cli](https://gitlab.com/gitlab-org/release-cli). +You can also call the `release-cli` directly from a `script` entry. + +The YAML described above would be translated into a CLI command like this: + +```shell +release-cli create --name "Release $CI_COMMIT_SHA" --description "Created using the release-cli $EXTRA_DESCRIPTION" --tag-name "v${MAJOR}.${MINOR}.${REVISION}" --ref "$CI_COMMIT_SHA" --released-at "2020-07-15T08:00:00Z" --milestone "m1" --milestone "m2" --milestone "m3" +``` + ### `pages` `pages` is a special job that is used to upload static content to GitLab that @@ -3460,7 +3925,8 @@ variables: GIT_STRATEGY: none ``` -NOTE: **Note:** `GIT_STRATEGY` is not supported for +NOTE: **Note:** +`GIT_STRATEGY` is not supported for [Kubernetes executor](https://docs.gitlab.com/runner/executors/kubernetes.html), but may be in the future. See the [support Git strategy with Kubernetes executor feature proposal](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/3847) for updates. @@ -3625,7 +4091,7 @@ You can set them globally or per-job in the [`variables`](#variables) section. > Introduced in GitLab 8.9 as an experimental feature. -NOTE: **Note**: +NOTE: **Note:** As of GitLab 12.0, newly created projects will automatically have a [default `git depth` value of `50`](../pipelines/settings.md#git-shallow-clone). You can specify the depth of fetching and cloning using `GIT_DEPTH`. This allows @@ -3748,6 +4214,10 @@ of `.gitlab-ci.yml`. Read more about the various [YAML features](https://learnxinyminutes.com/docs/yaml/). +In most cases, the [`extends` keyword](#extends) is more user friendly and should +be used over these special YAML features. YAML anchors may still +need to be used to merge arrays. + ### Anchors > Introduced in GitLab 8.6 and GitLab Runner v1.1.1. @@ -3755,7 +4225,8 @@ Read more about the various [YAML features](https://learnxinyminutes.com/docs/ya YAML has a handy feature called 'anchors', which lets you easily duplicate content across your document. Anchors can be used to duplicate/inherit properties, and is a perfect example to be used with [hidden jobs](#hide-jobs) -to provide templates for your jobs. +to provide templates for your jobs. When there is duplicate keys, GitLab will +perform a reverse deep merge based on the keys. The following example uses anchors and map merging. It will create two jobs, `test1` and `test2`, that will inherit the parameters of `.job_template`, each @@ -3816,6 +4287,8 @@ directive defined in `.postgres_services` and `.mysql_services` respectively: .job_template: &job_definition script: - test project + tags: + - dev .postgres_services: services: &postgres_definition @@ -3830,6 +4303,8 @@ directive defined in `.postgres_services` and `.mysql_services` respectively: test:postgres: <<: *job_definition services: *postgres_definition + tags: + - postgres test:mysql: <<: *job_definition @@ -3842,6 +4317,8 @@ The expanded version looks like this: .job_template: script: - test project + tags: + - dev .postgres_services: services: @@ -3859,6 +4336,8 @@ test:postgres: services: - postgres - ruby + tags: + - postgres test:mysql: script: @@ -3866,13 +4345,19 @@ test:mysql: services: - mysql - ruby + tags: + - dev ``` You can see that the hidden jobs are conveniently used as templates. NOTE: **Note:** +Note that `tags: [dev]` has been overwritten by `tags: [postgres]`. + +NOTE: **Note:** You can't use YAML anchors across multiple files when leveraging the [`include`](#include) -feature. Anchors are only valid within the file they were defined in. +feature. Anchors are only valid within the file they were defined in. Instead +of using YAML anchors, you can use the [`extends` keyword](#extends). #### YAML anchors for `before_script` and `after_script` @@ -3886,11 +4371,11 @@ Example: ```yaml .something_before: &something_before -- echo 'something before' + - echo 'something before' .something_after: &something_after -- echo 'something after' -- echo 'another thing after' + - echo 'something after' + - echo 'another thing after' job_name: before_script: @@ -3912,7 +4397,7 @@ For example: ```yaml .something: &something -- echo 'something' + - echo 'something' job_name: script: diff --git a/doc/ci/yaml/includes.md b/doc/ci/yaml/includes.md index 969b54d5be8..f7ed7248dc4 100644 --- a/doc/ci/yaml/includes.md +++ b/doc/ci/yaml/includes.md @@ -5,7 +5,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w type: reference --- -# GitLab CI/CD YAML includes +# GitLab CI/CD include examples In addition to the [`includes` examples](README.md#include) listed in the [GitLab CI YAML reference](README.md), this page lists more variations of `include` diff --git a/doc/customization/issue_and_merge_request_template.md b/doc/customization/issue_and_merge_request_template.md index ebf711e105b..bab81629221 100644 --- a/doc/customization/issue_and_merge_request_template.md +++ b/doc/customization/issue_and_merge_request_template.md @@ -1,5 +1,5 @@ --- -redirect_to: '../user/project/description_templates.md#setting-a-default-template-for-merge-requests-and-issues--starter' +redirect_to: '../user/project/description_templates.md' --- -This document was moved to [description_templates](../user/project/description_templates.md#setting-a-default-template-for-merge-requests-and-issues--starter). +This document was moved to [description_templates](../user/project/description_templates.md). diff --git a/doc/development/README.md b/doc/development/README.md index d330d6d466e..ab86c252948 100644 --- a/doc/development/README.md +++ b/doc/development/README.md @@ -99,6 +99,9 @@ Complementary reads: - [Code comments](code_comments.md) - [Renaming features](renaming_features.md) - [Windows Development on GCP](windows.md) +- [Code Intelligence](code_intelligence/index.md) +- [Approval Rules](approval_rules.md) +- [Feature categorization](feature_categorization/index.md) ## Performance guides diff --git a/doc/development/api_graphql_styleguide.md b/doc/development/api_graphql_styleguide.md index 43e96340d10..92e6add9f17 100644 --- a/doc/development/api_graphql_styleguide.md +++ b/doc/development/api_graphql_styleguide.md @@ -571,7 +571,8 @@ module Types end ``` -NOTE: **Note:** If the field's type already [has a particular +NOTE: **Note:** +If the field's type already [has a particular authorization](#type-authorization) then there is no need to add that same authorization to the field. @@ -717,11 +718,48 @@ will be returned as the result of the mutation. ### Naming conventions -Always provide a consistent GraphQL-name to the mutation, this name is -used to generate the input types and the field the mutation is mounted -on. The name should look like `<Resource being modified><Mutation -class name>`, for example the `Mutations::MergeRequests::SetWip` -mutation has GraphQL name `MergeRequestSetWip`. +Each mutation must define a `graphql_name`, which is the name of the mutation in the GraphQL schema. + +Example: + +```ruby +class UserUpdateMutation < BaseMutation + graphql_name 'UserUpdate' +end +``` + +Our GraphQL mutation names are historically inconsistent, but new mutation names should follow the +convention `'{Resource}{Action}'` or `'{Resource}{Action}{Attribute}'`. + +Mutations that **create** new resources should use the verb `Create`. + +Example: + +- `CommitCreate` + +Mutations that **update** data should use: + +- The verb `Update`. +- A domain-specific verb like `Set`, `Add`, or `Toggle` if more appropriate. + +Examples: + +- `EpicTreeReorder` +- `IssueSetWeight` +- `IssueUpdate` +- `TodoMarkDone` + +Mutations that **remove** data should use: + +- The verb `Delete` rather than `Destroy`. +- A domain-specific verb like `Remove` if more appropriate. + +Examples: + +- `AwardEmojiRemove` +- `NoteDelete` + +If you need advice for mutation naming, canvass the Slack `#graphql` channel for feedback. ### Arguments @@ -975,6 +1013,34 @@ to make sure the error information we are passing back is useful. See also the [frontend GraphQL guide](../development/fe_guide/graphql.md#handling-errors). +### Aliasing and deprecating mutations + +The `#mount_aliased_mutation` helper allows us to alias a mutation as +another name within `MutationType`. + +For example, to alias a mutation called `FooMutation` as `BarMutation`: + +```ruby +mount_aliased_mutation 'BarMutation', Mutations::FooMutation +``` + +This allows us to rename a mutation and continue to support the old name, +when coupled with the [`deprecated`](#deprecating-fields) argument. + +Example: + +```ruby +mount_aliased_mutation 'UpdateFoo', + Mutations::Foo::Update, + deprecated: { reason: 'Use fooUpdate', milestone: '13.2' } +``` + +Deprecated mutations should be added to `Types::DeprecatedMutations` and +tested for within the unit test of `Types::MutationType`. The merge request +[!34798](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/34798) +can be referred to as an example of this, including the method of testing +deprecated aliased mutations. + ## Validating arguments For validations of single arguments, use the diff --git a/doc/development/api_styleguide.md b/doc/development/api_styleguide.md index 6a044004926..327f919d7f4 100644 --- a/doc/development/api_styleguide.md +++ b/doc/development/api_styleguide.md @@ -98,6 +98,46 @@ For instance: Model.create(foo: params[:foo]) ``` +## Array types + +With Grape v1.3+, Array types must be defined with a `coerce_with` +block, or parameters will fail to validate when passed a string from an +API request. See the [Grape upgrading +documentation](https://github.com/ruby-grape/grape/blob/master/UPGRADING.md#ensure-that-array-types-have-explicit-coercions) +for more details. + +### Automatic coercion of nil inputs + +Prior to Grape v1.3.3, Array parameters with `nil` values would +automatically be coerced to an empty Array. However, due to [this pull +request in v1.3.3](https://github.com/ruby-grape/grape/pull/2040), this +is no longer the case. For example, suppose you define a PUT `/test` +request that has an optional parameter: + +```ruby +optional :user_ids, type: Array[Integer], coerce_with: ::API::Validations::Types::CommaSeparatedToIntegerArray.coerce, desc: 'The user ids for this rule' +``` + +Normally, a request to PUT `/test?user_ids` would cause Grape to pass +`params` of `{ user_ids: nil }`. + +This may introduce errors with endpoints that expect a blank array and +do not handle `nil` inputs properly. To preserve the previous behavior, +there is a helper method `coerce_nil_params_to_array!` that is used +in the `before` block of all API calls: + +```ruby +before do + coerce_nil_params_to_array! +end +``` + +With this change, a request to PUT `/test?user_ids` will cause Grape to +pass `params` to be `{ user_ids: [] }`. + +There is [an open issue in the Grape tracker](https://github.com/ruby-grape/grape/issues/2068) +to make this easier. + ## Using HTTP status helpers For non-200 HTTP responses, use the provided helpers in `lib/api/helpers.rb` to ensure correct behavior (`not_found!`, `no_content!` etc.). These will `throw` inside Grape and abort the execution of your endpoint. diff --git a/doc/development/application_limits.md b/doc/development/application_limits.md index 1f7a9ff09b9..4d296451add 100644 --- a/doc/development/application_limits.md +++ b/doc/development/application_limits.md @@ -11,7 +11,7 @@ coordinate with others to [document](../administration/instance_limits.md) and communicate those limits. There is a guide about [introducing application -limits](https://about.gitlab.com/handbook/product/product-management/process/#introducing-application-limits). +limits](https://about.gitlab.com/handbook/product/product-processes/#introducing-application-limits). ## Development @@ -27,7 +27,8 @@ limit values. It's recommended to create separate migration script files. add_column(:plan_limits, :project_hooks, :integer, default: 100, null: false) ``` - NOTE: **Note:** Plan limits entries set to `0` mean that limits are not + NOTE: **Note:** + Plan limits entries set to `0` mean that limits are not enabled. You should use this setting only in special and documented circumstances. 1. (Optionally) Create the database migration that fine-tunes each level with @@ -57,7 +58,8 @@ limit values. It's recommended to create separate migration script files. end ``` -NOTE: **Note:** Some plans exist only on GitLab.com. This will be no-op +NOTE: **Note:** +Some plans exist only on GitLab.com. This will be no-op for plans that do not exist. ### Plan limits validation @@ -95,7 +97,8 @@ can be used to validate that a model does not exceed the limits. It ensures that the count of the records for the current model does not exceed the defined limit. -NOTE: **Note:** You must specify the limit scope of the object being validated +NOTE: **Note:** +You must specify the limit scope of the object being validated and the limit name if it's different from the pluralized model name. ```ruby @@ -143,4 +146,5 @@ GitLab.com: - `silver` - Namespaces and projects with a Silver subscription - `gold` - Namespaces and projects with a Gold subscription -NOTE: **Note:** The test environment doesn't have any plans. +NOTE: **Note:** +The test environment doesn't have any plans. diff --git a/doc/development/application_secrets.md b/doc/development/application_secrets.md new file mode 100644 index 00000000000..24755586cf8 --- /dev/null +++ b/doc/development/application_secrets.md @@ -0,0 +1,41 @@ +# Application secrets + +This page is a development guide for application secrets. + +## Secret entries + +|Entry |Description | +|--- |--- | +|`secret_key_base` | The base key to be used for generating a various secrets | +| `otp_key_base` | The base key for One Time Passwords, described in [User management](../raketasks/user_management.md#rotate-two-factor-authentication-encryption-key) | +|`db_key_base` | The base key to encrypt the data for `attr_encrypted` columns | +|`openid_connect_signing_key` | The singing key for OpenID Connect | + +## Where the secrets are stored + +|Installation type |Location | +|--- |--- | +|Omnibus |[`/etc/gitlab/gitlab-secrets.json`](https://docs.gitlab.com/omnibus/settings/backups.html#backup-and-restore-omnibus-gitlab-configuration) | +|Cloud Native GitLab Charts |[Kubernets Secrets](https://gitlab.com/gitlab-org/charts/gitlab/-/blob/f65c3d37fc8cf09a7987544680413552fb666aac/doc/installation/secrets.md#gitlab-rails-secret)| +|Source |`<path-to-gitlab-rails>/config/secrets.yml` (Automatically generated by [01_secret_token.rb](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/initializers/01_secret_token.rb)) | + +## Warning: Before you add a new secret to application secrets + +Before you add a new secret to [`config/initializers/01_secret_token.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/initializers/01_secret_token.rb), +make sure you also update Omnibus GitLab or updates will fail. Omnibus is responsible for writing the `secrets.yml` file. +If Omnibus doesn't know about a secret, Rails will attempt to write to the file, but this will fail because Rails doesn't have write access. +The same rules apply to Cloud Native GitLab charts, you must update the charts at first. +In case you need the secret to have same value on each node (which is usually the case) you need to make sure it's configured for all +GitLab.com environments prior to changing this file. + +**Examples** + +- [Change for source installation](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/27581) +- [Change for omnibus installation](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/3267) +- [Change for omnibus installation](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/4158) +- [Change for Cloud Native installation](https://gitlab.com/gitlab-org/charts/gitlab/-/merge_requests/1318) + +## Further iteration + +We might deprecate/remove this automatic secret generation '01_secret_token.rb' in the future. +Please see [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/222690) for more information. diff --git a/doc/development/approval_rules.md b/doc/development/approval_rules.md new file mode 100644 index 00000000000..65df82721de --- /dev/null +++ b/doc/development/approval_rules.md @@ -0,0 +1,280 @@ +# Approval Rules **(STARTER)** + +This document explains the backend design and flow of all related functionality +about [merge request approval rules](../user/project/merge_requests/merge_request_approvals.md). + +This should help contributors to understand the code design easier and to also +help see if there are parts to improve as the feature and its implementation +evolves. + +It's intentional that it doesn't contain too much implementation detail as they +can change often. The code should explain those things better. The components +mentioned here are the major parts of the application for the approval rules +feature to work. + +NOTE: **Note:** +This is a living document and should be updated accordingly when parts +of the codebase touched in this document changed/removed or when new components +are added. + +## Data Model + +```mermaid +erDiagram + Project ||--o{ MergeRequest: " " + Project ||--o{ ApprovalProjectRule: " " + ApprovalProjectRule }o--o{ User: " " + ApprovalProjectRule }o--o{ Group: " " + ApprovalProjectRule }o--o{ ProtectedBranch: " " + MergeRequest ||--|| ApprovalState: " " + ApprovalState ||--o{ ApprovalWrappedRule: " " + MergeRequest ||--o{ Approval: " " + MergeRequest ||--o{ ApprovalMergeRequestRule: " " + ApprovalMergeRequestRule }o--o{ User: " " + ApprovalMergeRequestRule }o--o{ Group: " " + ApprovalMergeRequestRule ||--o| ApprovalProjectRule: " " +``` + +### `Project` and `MergeRequest` + +`Project` and `MergeRequest` models are defined in `ee/app/models/ee/project.rb` +and `ee/app/models/ee/merge_request.rb`. They extend the non-EE versions since +approval rules is an EE only feature. Associations and other related stuff to +merge request approvals are defined here. + +### `ApprovalState` + +```mermaid +erDiagram + MergeRequest ||--|| ApprovalState: " " +``` + +`ApprovalState` class is defined in `ee/app/models/approval_state.rb`. It's not +an actual `ActiveRecord` model. This class encapsulates all logic related to the +state of the approvals for a certain merge request like: + +- Knowing the approval rules that are applicable to the merge request based on + its target branch. +- Knowing the approval rules that are applicable to a certain target branch. +- Checking if all rules were approved. +- Checking if approval is required. +- Knowing how many approvals were given or still required. + +It gets the approval rules data from the project (`ApprovalProjectRule`) or the +merge request (`ApprovalMergeRequestRule`) and wrap it as `ApprovalWrappedRule`. + +### `ApprovalProjectRule` + +```mermaid +erDiagram + Project ||--o{ ApprovalProjectRule: " " + ApprovalProjectRule }o--o{ User: " " + ApprovalProjectRule }o--o{ Group: " " + ApprovalProjectRule }o--o{ ProtectedBranch: " " +``` + +`ApprovalProjectRule` model is defined in `ee/app/models/approval_project_rule.rb`. + +A record is created/updated/deleted when an approval rule is added/edited/removed +via project settings or the [project level approvals API](../api/merge_request_approvals.md#project-level-mr-approvals). +The `ApprovalState` model get these records when approval rules are not +overwritten. + +The `protected_branches` attribute is set and used when a rule is scoped to +protected branches. See [Scoped to Protected Branch doc](../user/project/merge_requests/merge_request_approvals.md#scoped-to-protected-branch-premium) +for more information about the feature. + +### `ApprovalMergeRequestRule` + +```mermaid +erDiagram + MergeRequest ||--o{ ApprovalMergeRequestRule: " " + ApprovalMergeRequestRule }o--o{ User: " " + ApprovalMergeRequestRule }o--o{ Group: " " + ApprovalMergeRequestRule ||--o| ApprovalProjectRule: " " +``` + +`ApprovalMergeRequestRule` model is defined in `ee/app/models/approval_merge_request_rule.rb`. + +A record is created/updated/deleted when a rule is added/edited/removed via merge +request create/edit form or the [merge request level approvals API](../api/merge_request_approvals.md#merge-request-level-mr-approvals). + +The `approval_project_rule` is set when it is based from an existing `ApprovalProjectRule`. + +An `ApprovalMergeRequestRule` doesn't have `protected_branches` as it inherits +them from the `approval_project_rule` if not overridden. + +### `ApprovalWrappedRule` + +```mermaid +erDiagram + ApprovalState ||--o{ ApprovalWrappedRule: " " +``` + +`ApprovalWrappedRule` is defined in `ee/app/modes/approval_wrapped_rule.rb` and +is not an `ActiveRecord` model. It's used to wrap an `ApprovalProjectRule` or +`ApprovalMergeRequestRule` for common interface. It also has the following sub +types: + +- `ApprovalWrappedAnyApprovalRule` - for wrapping an `any_approver` rule. +- `ApprovalWrappedCodeOwnerRule` - for wrapping a `code_owner` rule. + +This class delegates most of the responsibilities to the approval rule it wraps +but it's also responsible for: + +- Checking if the approval rule is approved. +- Knowing how many approvals were given or still required for the approval rule. + +It gets this information from the approval rule and the `Approval` records from +the merge request. + +### `Approval` + +```mermaid +erDiagram + MergeRequest ||--o{ Approval: " " +``` + +`Approval` model is defined in `ee/app/models/approval.rb`. This model is +responsible for storing information about an approval made on a merge request. +Whenever an approval is given/revoked, a record is created/deleted. + +## Controllers and Services + +The following controllers and services below are being utilized for the approval +rules feature to work. + +### `API::ProjectApprovalSettings` + +This private API is defined in `ee/lib/api/project_approval_settings.rb`. + +This is used for the following: + +- Listing the approval rules in project settings. +- Creating/updating/deleting rules in project settings. +- Listing the approval rules on create merge request form. + +### `Projects::MergeRequests::CreationsController` + +This controller is defined in `app/controllers/projects/merge_requests/creations_controller.rb`. + +The `create` action of this controller is used when create merge request form is +submitted. It accepts the `approval_rules_attributes` parameter for creating/updating/deleting +`ApprovalMergeRequestRule` records. It passes the parameter along when it executes +`MergeRequests::CreateService`. + +### `Projects::MergeRequestsController` + +This controller is defined in `app/controllers/projects/merge_requests_controller.rb`. + +The `update` action of this controller is used when edit merge request form is +submitted. It's like `Projects::MergeRequests::CreationsController` but it executes +`MergeRequests::UpdateService` instead. + +### `API::MergeRequestApprovals` + +This API is defined in `ee/lib/api/merge_request_approvals.rb`. + +The [Approvals API endpoint](../api/merge_request_approvals.md#get-configuration-1) +is requested when merge request page loads. + +The `/projects/:id/merge_requests/:merge_request_iid/approval_settings` is a +private API endpoint used for the following: + +- Listing the approval rules on edit merge request form. +- Listing the approval rules on the merge request page. + +When approving/unapproving MR via UI and API, the [Approve Merge Request](../api/merge_request_approvals.md#approve-merge-request) +API endpoint and the [Unapprove Merge Request](../api/merge_request_approvals.md#unapprove-merge-request) +API endpoint are requested. They execute `MergeRequests::ApprovalService` and +`MergeRequests::RemoveApprovalService` accordingly. + +### `API::ProjectApprovalRules` and `API::MergeRequestApprovalRules` + +These APIs are defined in `ee/lib/api/project_approval_rules.rb` and +`ee/lib/api/merge_request_approval_rules.rb`. + +Used to list/create/update/delete project and merge request level rules via +[Merge request approvals API](../api/merge_request_approvals.md). + +Executes `ApprovalRules::CreateService`, `ApprovalRules::UpdateService`, +`ApprovalRules::ProjectRuleDestroyService`, and `ApprovalRules::MergeRequestRuleDestroyService` +accordingly. + +### `ApprovalRules::ParamsFilteringService` + +This service is defined in `ee/app/services/approval_rules/params_filtering_service.rb`. + +It is called only when `MergeRequests::CreateService` and +`MergeRequests::UpdateService` are executed. + +It is responsible for parsing `approval_rules_attributes` parameter to: + +- Remove it when user can't update approval rules. +- Filter the user IDs whether they are members of the project or not. +- Filter the group IDs whether they are visible to user. +- Identify the `any_approver` rule. +- Append hidden groups to it when specified. +- Append user defined inapplicable (rules that does not apply to MR's target + branch) approval rules. + +## Flow + +These flowcharts should help explain the flow from the controllers down to the +models for different functionalities. + +Some CRUD API endpoints are intentionally skipped because they are pretty +straightforward. + +### Creating a merge request with approval rules via web UI + +```mermaid +graph LR + Projects::MergeRequests::CreationsController --> MergeRequests::CreateService + MergeRequests::CreateService --> ApprovalRules::ParamsFilteringService + ApprovalRules::ParamsFilteringService --> MergeRequests::CreateService + MergeRequests::CreateService --> MergeRequest + MergeRequest --> db[(Database)] + MergeRequest --> User + MergeRequest --> Group + MergeRequest --> ApprovalProjectRule + User --> db[(Database)] + Group --> db[(Database)] + ApprovalProjectRule --> db[(Database)] +``` + +When updating, same flow is followed but it starts at `Projects::MergeRequestsController` +and executes `MergeRequests::UpdateService` instead. + +### Viewing the merge request approval rules on an MR page + +```mermaid +graph LR + API::MergeRequestApprovals --> MergeRequest + MergeRequest --> ApprovalState + ApprovalState --> id1{approval rules are overridden} + id1{approval rules are overridden} --> |No| ApprovalProjectRule & ApprovalMergeRequestRule + id1{approval rules are overridden} --> |Yes| ApprovalMergeRequestRule + ApprovalState --> ApprovalWrappedRule + ApprovalWrappedRule --> Approval +``` + +This flow gets initiated by the frontend component. The data returned will +then be used to display information on the MR widget. + +### Approving a merge request + +```mermaid +graph LR + API::MergeRequestApprovals --> MergeRequests::ApprovalService + MergeRequests::ApprovalService --> Approval + Approval --> db[(Database)] +``` + +When unapproving, same flow is followed but the `MergeRequests::RemoveApprovalService` +is executed instead. + +## TODO + +1. Add information related to other rule types (e.g. `code_owner` and `report_approver`). +1. Add information about side effects of approving/unapproving merge request. diff --git a/doc/development/architecture.md b/doc/development/architecture.md index f0ce033587d..8b28dd03017 100644 --- a/doc/development/architecture.md +++ b/doc/development/architecture.md @@ -140,29 +140,29 @@ Table description links: | [Elasticsearch](#elasticsearch) | Improved search within GitLab | ⤓ | ⤓ | ⤓ | ❌ | ⤓ | ⤓ | EE Only | | [Gitaly](#gitaly) | Git RPC service for handling all Git calls made by GitLab | ✅ | ✅ | ✅ | ✅ | ⚙ | ✅ | CE & EE | | [GitLab Exporter](#gitlab-exporter) | Generates a variety of GitLab metrics | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | CE & EE | -| [GitLab Geo Node](#gitlab-geo) | Geographically distributed GitLab nodes | ⚙ | ❌ | ❌ | ✅ | ❌ | ⚙ | EE Only | +| [GitLab Geo Node](#gitlab-geo) | Geographically distributed GitLab nodes | ⚙ | ⚙ | ❌ | ✅ | ❌ | ⚙ | EE Only | | [GitLab Managed Apps](#gitlab-managed-apps) | Deploy Helm, Ingress, Cert-Manager, Prometheus, a Runner, JupyterHub, or Knative to a cluster | ⤓ | ⤓ | ⤓ | ⤓ | ⤓ | ⤓ | CE & EE | | [GitLab Pages](#gitlab-pages) | Hosts static websites | ⚙ | ❌ | ❌ | ✅ | ⚙ | ⚙ | CE & EE | | [GitLab self-monitoring: Alertmanager](#alertmanager) | Deduplicates, groups, and routes alerts from Prometheus | ⚙ | ✅ | ⚙ | ✅ | ❌ | ❌ | CE & EE | -| [GitLab self-monitoring: Grafana](#grafana) | Metrics dashboard | ✅ | ⤓ | ⤓ | ✅ | ❌ | ❌ | CE & EE | -| [GitLab self-monitoring: Jaeger](#jaeger) | View traces generated by the GitLab instance | ❌ | ❌ | ❌ | ❌ | ⤓ | ⚙ | CE & EE | +| [GitLab self-monitoring: Grafana](#grafana) | Metrics dashboard | ✅ | ⚙ | ⤓ | ✅ | ❌ | ❌ | CE & EE | +| [GitLab self-monitoring: Jaeger](#jaeger) | View traces generated by the GitLab instance | ❌ | ⚙ | ❌ | ❌ | ⤓ | ⚙ | CE & EE | | [GitLab self-monitoring: Prometheus](#prometheus) | Time-series database, metrics collection, and query service | ✅ | ✅ | ⚙ | ✅ | ❌ | ❌ | CE & EE | -| [GitLab self-monitoring: Sentry](#sentry) | Track errors generated by the GitLab instance | ⤓ | ❌ | ❌ | ✅ | ⤓ | ⤓ | CE & EE | +| [GitLab self-monitoring: Sentry](#sentry) | Track errors generated by the GitLab instance | ⤓ | ⤓ | ❌ | ✅ | ⤓ | ⤓ | CE & EE | | [GitLab Shell](#gitlab-shell) | Handles `git` over SSH sessions | ✅ | ✅ | ✅ | ✅ | ⚙ | ✅ | CE & EE | | [GitLab Workhorse](#gitlab-workhorse) | Smart reverse proxy, handles large HTTP requests | ✅ | ✅ | ✅ | ✅ | ⚙ | ✅ | CE & EE | -| [Inbound email (SMTP)](#inbound-email) | Receive messages to update issues | ⤓ | ⤓ | ⤓ | ✅ | ⤓ | ⤓ | CE & EE | +| [Inbound email (SMTP)](#inbound-email) | Receive messages to update issues | ⤓ | ⚙ | ⤓ | ✅ | ⤓ | ⤓ | CE & EE | | [Jaeger integration](#jaeger) | Distributed tracing for deployed apps | ⤓ | ⤓ | ⤓ | ⤓ | ⤓ | ⤓ | EE Only | | [LDAP Authentication](#ldap-authentication) | Authenticate users against centralized LDAP directory | ⤓ | ⤓ | ⤓ | ❌ | ⤓ | ⤓ | CE & EE | | [Mattermost](#mattermost) | Open-source Slack alternative | ⚙ | ⤓ | ⤓ | ⤓ | ❌ | ❌ | CE & EE | | [MinIO](#minio) | Object storage service | ⤓ | ✅ | ✅ | ✅ | ❌ | ⚙ | CE & EE | | [NGINX](#nginx) | Routes requests to appropriate components, terminates SSL | ✅ | ✅ | ⚙ | ✅ | ⤓ | ❌ | CE & EE | | [Node Exporter](#node-exporter) | Prometheus endpoint with system metrics | ✅ | N/A | N/A | ✅ | ❌ | ❌ | CE & EE | -| [Outbound email (SMTP)](#outbound-email) | Send email messages to users | ⤓ | ⤓ | ⤓ | ✅ | ⤓ | ⤓ | CE & EE | +| [Outbound email (SMTP)](#outbound-email) | Send email messages to users | ⤓ | ⚙ | ⤓ | ✅ | ⤓ | ⤓ | CE & EE | | [PgBouncer Exporter](#pgbouncer-exporter) | Prometheus endpoint with PgBouncer metrics | ⚙ | ❌ | ❌ | ✅ | ❌ | ❌ | CE & EE | | [PgBouncer](#pgbouncer) | Database connection pooling, failover | ⚙ | ❌ | ❌ | ✅ | ❌ | ❌ | EE Only | | [PostgreSQL Exporter](#postgresql-exporter) | Prometheus endpoint with PostgreSQL metrics | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | CE & EE | | [PostgreSQL](#postgresql) | Database | ✅ | ✅ | ✅ | ✅ | ⤓ | ✅ | CE & EE | -| [Praefect](#praefect) | A transparent proxy between any Git client and Gitaly storage nodes. | ✅ | ❌ | ❌ | ✅ | ⚙ | ✅ | CE & EE | +| [Praefect](#praefect) | A transparent proxy between any Git client and Gitaly storage nodes. | ✅ | ⚙ | ❌ | ✅ | ⚙ | ✅ | CE & EE | | [Redis Exporter](#redis-exporter) | Prometheus endpoint with Redis metrics | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | CE & EE | | [Redis](#redis) | Caching service | ✅ | ✅ | ✅ | ✅ | ⤓ | ✅ | CE & EE | | [Registry](#registry) | Container registry, allows pushing and pulling of images | ⚙ | ✅ | ✅ | ✅ | ⤓ | ⚙ | CE & EE | @@ -196,7 +196,7 @@ GitLab can be considered to have two layers from a process perspective: - Process: `alertmanager` - GitLab.com: [Monitoring of GitLab.com](https://about.gitlab.com/handbook/engineering/monitoring/) -[Alert manager](https://prometheus.io/docs/alerting/alertmanager/) is a tool provided by Prometheus that _"handles alerts sent by client applications such as the Prometheus server. It takes care of deduplicating, grouping, and routing them to the correct receiver integration such as email, PagerDuty, or OpsGenie. It also takes care of silencing and inhibition of alerts."_ You can read more in [issue #45740](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/45740) about what we will be alerting on. +[Alert manager](https://prometheus.io/docs/alerting/latest/alertmanager/) is a tool provided by Prometheus that _"handles alerts sent by client applications such as the Prometheus server. It takes care of deduplicating, grouping, and routing them to the correct receiver integration such as email, PagerDuty, or Opsgenie. It also takes care of silencing and inhibition of alerts."_ You can read more in [issue #45740](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/45740) about what we will be alerting on. #### Certificate management @@ -273,7 +273,7 @@ repository updates to secondary nodes. - Configuration: - [Omnibus](../administration/geo/replication/index.md#setup-instructions) - - [Charts](https://gitlab.com/gitlab-org/charts/gitlab/-/issues/8) + - [Charts](https://docs.gitlab.com/charts/advanced/geo/) - [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit/blob/master/doc/howto/geo.md) - Layer: Core Service (Processor) @@ -349,7 +349,7 @@ GitLab CI/CD is the open-source continuous integration service included with Git - [Project page](https://github.com/grafana/grafana/blob/master/README.md) - Configuration: - [Omnibus](../administration/monitoring/performance/grafana_configuration.md) - - [Charts](https://github.com/helm/charts/tree/master/stable/grafana) + - [Charts](https://docs.gitlab.com/charts/charts/globals#configure-grafana-integration) - Layer: Monitoring - GitLab.com: [GitLab triage Grafana dashboard](https://dashboards.gitlab.com/d/RZmbBr7mk/gitlab-triage?refresh=30s) @@ -360,7 +360,7 @@ Grafana is an open source, feature rich metrics dashboard and graph editor for G - [Project page](https://github.com/jaegertracing/jaeger/blob/master/README.md) - Configuration: - [Omnibus](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/4104) - - [Charts](https://gitlab.com/gitlab-org/charts/gitlab/-/issues/1320) + - [Charts](https://docs.gitlab.com/charts/charts/globals#tracing) - [Source](../development/distributed_tracing.md#enabling-distributed-tracing) - [GDK](../development/distributed_tracing.md#using-jaeger-in-the-gitlab-development-kit) - Layer: Monitoring @@ -369,7 +369,7 @@ Grafana is an open source, feature rich metrics dashboard and graph editor for G Jaeger, inspired by Dapper and OpenZipkin, is a distributed tracing system. It can be used for monitoring microservices-based distributed systems. -For monitoring deployed apps, see [Jaeger tracing documentation](../user/project/operations/tracing.md) +For monitoring deployed apps, see [Jaeger tracing documentation](../operations/tracing.md) #### Logrotate @@ -458,7 +458,7 @@ Prometheus exporter for PgBouncer. Exports metrics at 9127/metrics. - [Project page](https://github.com/postgres/postgres/blob/master/README) - Configuration: - [Omnibus](https://docs.gitlab.com/omnibus/settings/database.html) - - [Charts](https://github.com/helm/charts/tree/master/stable/postgresql) + - [Charts](https://docs.gitlab.com/charts/installation/deployment.html#postgresql) - [Source](../install/installation.md#6-database) - Layer: Core Service (Data) - Process: `postgresql` @@ -471,7 +471,7 @@ GitLab packages the popular Database to provide storage for Application meta dat - [Project page](https://github.com/wrouesnel/postgres_exporter/blob/master/README.md) - Configuration: - [Omnibus](../administration/monitoring/prometheus/postgres_exporter.md) - - [Charts](https://github.com/helm/charts/tree/master/stable/postgresql) + - [Charts](https://docs.gitlab.com/charts/installation/deployment.html#postgresql) - Layer: Monitoring - Process: `postgres-exporter` - GitLab.com: [Monitoring of GitLab.com](https://about.gitlab.com/handbook/engineering/monitoring/) @@ -483,7 +483,7 @@ GitLab packages the popular Database to provide storage for Application meta dat - [Project page](https://github.com/prometheus/prometheus/blob/master/README.md) - Configuration: - [Omnibus](../administration/monitoring/prometheus/index.md) - - [Charts](https://github.com/helm/charts/tree/master/stable/prometheus) + - [Charts](https://docs.gitlab.com/charts/installation/deployment.html#prometheus) - Layer: Monitoring - Process: `prometheus` - GitLab.com: [Prometheus](../user/gitlab_com/index.md#prometheus) @@ -495,7 +495,7 @@ Prometheus is a time-series tool that helps GitLab administrators expose metrics - [Project page](https://github.com/antirez/redis/blob/unstable/README.md) - Configuration: - [Omnibus](https://docs.gitlab.com/omnibus/settings/redis.html) - - [Charts](https://docs.gitlab.com/charts/charts/redis/) + - [Charts](https://docs.gitlab.com/charts/installation/deployment.html#redis) - [Source](../install/installation.md#7-redis) - Layer: Core Service (Data) - Process: `redis` @@ -512,7 +512,7 @@ Redis is packaged to provide a place to store: - [Project page](https://github.com/oliver006/redis_exporter/blob/master/README.md) - Configuration: - [Omnibus](../administration/monitoring/prometheus/redis_exporter.md) - - [Charts](https://docs.gitlab.com/charts/charts/redis/) + - [Charts](https://docs.gitlab.com/charts/installation/deployment.html#redis) - Layer: Monitoring - Process: `redis-exporter` - GitLab.com: [Monitoring of GitLab.com](https://about.gitlab.com/handbook/engineering/monitoring/) @@ -545,7 +545,7 @@ An external registry can also be configured to use GitLab as an auth endpoint. - [Project page](https://github.com/getsentry/sentry/) - Configuration: - [Omnibus](https://docs.gitlab.com/omnibus/settings/configuration.html#error-reporting-and-logging-with-sentry) - - [Charts](https://gitlab.com/gitlab-org/charts/gitlab/-/issues/1319) + - [Charts](https://docs.gitlab.com/charts/charts/globals#sentry-settings) - [Source](https://gitlab.com/gitlab-org/gitlab/blob/master/config/gitlab.yml.example) - [GDK](https://gitlab.com/gitlab-org/gitlab/blob/master/config/gitlab.yml.example) - Layer: Monitoring @@ -567,7 +567,7 @@ For monitoring deployed apps, see the [Sentry integration docs](../user/project/ - [GDK](https://gitlab.com/gitlab-org/gitlab/blob/master/config/gitlab.yml.example) - Layer: Core Service (Processor) - Process: `sidekiq` -- GitLab.com: [Sidekiq](../user/gitlab_com/index.md#Sidekiq) +- GitLab.com: [Sidekiq](../user/gitlab_com/index.md#sidekiq) Sidekiq is a Ruby background job processor that pulls jobs from the Redis queue and processes them. Background jobs allow GitLab to provide a faster request/response cycle by moving work into the background. @@ -576,7 +576,7 @@ Sidekiq is a Ruby background job processor that pulls jobs from the Redis queue - [Project page](https://gitlab.com/gitlab-org/gitlab/blob/master/README.md) - Configuration: - [Omnibus](https://docs.gitlab.com/omnibus/settings/unicorn.html) - - [Charts](https://docs.gitlab.com/charts/charts/gitlab/unicorn/) + - [Charts](https://docs.gitlab.com/charts/charts/gitlab/webservice/) - [Source](../install/installation.md#configure-it) - [GDK](https://gitlab.com/gitlab-org/gitlab/blob/master/config/gitlab.yml.example) - Layer: Core Service (Processor) @@ -773,7 +773,7 @@ response back to the user directly. When referring to `~git` in the pictures it means the home directory of the Git user which is typically `/home/git`. -GitLab is primarily installed within the `/home/git` user home directory as `git` user. Within the home directory is where the gitlabhq server software resides as well as the repositories (though the repository location is configurable). +GitLab is primarily installed within the `/home/git` user home directory as `git` user. Within the home directory is where the GitLab server software resides as well as the repositories (though the repository location is configurable). The bare repositories are located in `/home/git/repositories`. GitLab is a Ruby on rails application so the particulars of the inner workings can be learned by studying how a Ruby on rails application works. @@ -849,7 +849,7 @@ Usage: /etc/init.d/postgresql {start|stop|restart|reload|force-reload|status} [v ### Log locations of the services -gitlabhq (includes Unicorn and Sidekiq logs): +GitLab (includes Unicorn and Sidekiq logs): - `/home/git/gitlab/log/` contains `application.log`, `production.log`, `sidekiq.log`, `unicorn.stdout.log`, `git_json.log` and `unicorn.stderr.log` normally. diff --git a/doc/development/changelog.md b/doc/development/changelog.md index 5a8e05f888c..00a0573a8ba 100644 --- a/doc/development/changelog.md +++ b/doc/development/changelog.md @@ -14,12 +14,12 @@ following format: --- title: "Change[log]s" merge_request: 1972 -author: Black Sabbath +author: Black Sabbath @bsabbath type: added ``` The `merge_request` value is a reference to a merge request that adds this -entry, and the `author` key is used to give attribution to community +entry, and the `author` key (format: `<full name> <GitLab username>`) is used to give attribution to community contributors. **Both are optional**. The `type` field maps the category of the change, valid options are: added, fixed, changed, deprecated, removed, security, performance, other. **Type field is mandatory**. @@ -42,10 +42,9 @@ the `author` field. GitLab team members **should not**. Example: "Fixed a typo on the search results page." - Any docs-only changes **should not** have a changelog entry. - Any change behind a feature flag **should not** have a changelog entry - unless - the feature flag has been defaulted to true. The entry should be added - [in the merge request removing the feature flags](feature_flags/development.md). - If the change includes a database migration (regular, post, or data migration), - there should be a changelog entry for the migration change. + the feature flag has been defaulted to true. +- A change that [removes a feature flag](feature_flags/development.md) **should** have a changelog entry - + only if the feature flag did not default to true already. - A fix for a regression introduced and then fixed in the same release (i.e., fixing a bug introduced during a monthly release candidate) **should not** have a changelog entry. diff --git a/doc/development/chatops_on_gitlabcom.md b/doc/development/chatops_on_gitlabcom.md index 60056c2f65c..a3a4f1d7adc 100644 --- a/doc/development/chatops_on_gitlabcom.md +++ b/doc/development/chatops_on_gitlabcom.md @@ -14,9 +14,11 @@ tasks such as: To request access to Chatops on GitLab.com: 1. Log into <https://ops.gitlab.net/users/sign_in> **using the same username** as for GitLab.com (you may have to rename it). + 1. You could also use the "Sign in with" Google button to sign in, with your GitLab.com email address. 1. Ask in the [#production](https://gitlab.slack.com/messages/production) channel for an existing member to add you to the `chatops` project in Ops. They can do it by running `/chatops run member add <username> gitlab-com/chatops --ops` command in that channel. -NOTE: **Note:** If you had to change your username for GitLab.com on the first step, make sure [to reflect this information](https://gitlab.com/gitlab-com/www-gitlab-com#adding-yourself-to-the-team-page) on [the team page](https://about.gitlab.com/company/team/). +NOTE: **Note:** +If you had to change your username for GitLab.com on the first step, make sure [to reflect this information](https://gitlab.com/gitlab-com/www-gitlab-com#adding-yourself-to-the-team-page) on [the team page](https://about.gitlab.com/company/team/). ## See also diff --git a/doc/development/cicd/img/ci_template_selection_v13_1.png b/doc/development/cicd/img/ci_template_selection_v13_1.png Binary files differnew file mode 100644 index 00000000000..af9f6dd1a90 --- /dev/null +++ b/doc/development/cicd/img/ci_template_selection_v13_1.png diff --git a/doc/development/cicd/index.md b/doc/development/cicd/index.md index 42d4b494544..e0cca00fd69 100644 --- a/doc/development/cicd/index.md +++ b/doc/development/cicd/index.md @@ -2,6 +2,8 @@ Development guides that are specific to CI/CD are listed here. +If you are creating new CI/CD templates, please read [the development guide for GitLab CI/CD templates](templates.md). + ## CI Architecture overview The following is a simplified diagram of the CI architecture. Some details are left out in order to focus on @@ -90,7 +92,8 @@ A job with the `created` state won't be seen by the Runner yet. To make it possi When the Runner is connected, it requests the next `pending` job to run by polling the server continuously. -NOTE: **Note:** API endpoints used by the Runner to interact with GitLab are defined in [`lib/api/runner.rb`](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/api/runner.rb) +NOTE: **Note:** +API endpoints used by the Runner to interact with GitLab are defined in [`lib/api/runner.rb`](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/api/runner.rb) After the server receives the request it selects a `pending` job based on the [`Ci::RegisterJobService` algorithm](#ciregisterjobservice), then assigns and sends the job to the Runner. @@ -124,7 +127,8 @@ There are 3 top level queries that this service uses to gather the majority of t This list of jobs is then filtered further by matching tags between job and Runner tags. -NOTE: **Note:** If a job contains tags, the Runner will not pick the job if it does not match **all** the tags. +NOTE: **Note:** +If a job contains tags, the Runner will not pick the job if it does not match **all** the tags. The Runner may have more tags than defined for the job, but not vice-versa. Finally if the Runner can only pick jobs that are tagged, all untagged jobs are filtered out. diff --git a/doc/development/cicd/templates.md b/doc/development/cicd/templates.md new file mode 100644 index 00000000000..904e7adffe2 --- /dev/null +++ b/doc/development/cicd/templates.md @@ -0,0 +1,66 @@ +# Development guide for GitLab CI/CD templates + +This document explains how to develop [GitLab CI/CD templates](../../ci/examples/README.md). + +## Place the template file in a relevant directory + +All template files reside in the `lib/gitlab/ci/templates` directory, and are categorized by the following sub-directories: + +| Sub-directroy | Content | [Selectable in UI](#make-sure-the-new-template-can-be-selected-in-ui) | +|---------------|--------------------------------------------------------------|-----------------------------------------------------------------------| +| `/Jobs/*` | Auto DevOps related jobs | Yes | +| `/Pages/*` | Static site generators for GitLab Pages (for example Jekyll) | Yes | +| `/Security/*` | Security related jobs | Yes | +| `/Verify/*` | Verify/testing related jobs | Yes | +| `/Worklows/*` | Common uses of the `workflow:` keyword | No | +| `/*` (root) | General templates | Yes | + +## Criteria + +The file must follow the [`.gitlab-ci.yml` syntax](../../ci/yaml/README.md). +Verify it's valid by pasting it into the CI lint tool at `https://gitlab.com/gitlab-org/gitlab/-/ci/lint`. + +Also, all templates must be named with the `*.gitlab-ci.yml` suffix. + +### Backward compatibility + +A template might be dynamically included with the `include:template:` keyword. If +you make a change to an *existing* template, you must make sure that it won't break +CI/CD in existing projects. + +## Testing + +Each CI/CD template must be tested in order to make sure that it's safe to be published. + +### Manual QA + +It's always good practice to test the template in a minimal demo project. +To do so, please follow the following steps: + +1. Create a public sample project on <https://gitlab.com>. +1. Add a `.gitlab-ci.yml` to the project with the proposed template. +1. Run pipelines and make sure that everything runs properly, in all possible cases + (merge request pipelines, schedules, and so on). +1. Link to the project in the description of the merge request that is adding a new template. + +This is useful information for reviewers to make sure the template is safe to be merged. + +### Make sure the new template can be selected in UI + +Templates located under some directories are also [selectable in the **New file** UI](#place-the-template-file-in-a-relevant-directory). +When you add a template into one of those directories, make sure that it correctly appears in the dropdown: + + + +### Write an RSpec test + +You should write an RSpec test to make sure that pipeline jobs will be generated correctly: + +1. Add a test file at `spec/lib/gitlab/ci/templates/<template-category>/<template-name>_spec.rb` +1. Test that pipeline jobs are properly created via `Ci::CreatePipelineService`. + +## Security + +A template could contain malicious code. For example, a template that contains the `export` shell command in a job +might accidentally expose project secret variables in a job log. +If you're unsure if it's secure or not, you need to ask security experts for cross-validation. diff --git a/doc/development/code_intelligence/index.md b/doc/development/code_intelligence/index.md new file mode 100644 index 00000000000..bd11f0bff79 --- /dev/null +++ b/doc/development/code_intelligence/index.md @@ -0,0 +1,110 @@ +# Code Intelligence + +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/1576) in GitLab 13.1. + +This document describes the design behind [Code Intelligence](../../user/project/code_intelligence.md). + +GitLab's built-in Code Intelligence is powered by +[LSIF](https://lsif.dev) and comes down to generating an LSIF document for a +project in a CI job, processing the data, uploading it as a CI artifact and +displaying this information for the files in the project. + +Here is a sequence diagram for uploading an LSIF artifact: + +```mermaid +sequenceDiagram + participant Runner + participant Workhorse + participant Rails + participant Object Storage + + Runner->>+Workhorse: POST /v4/jobs/:id/artifacts + Workhorse->>+Rails: POST /:id/artifacts/authorize + Rails-->>-Workhorse: Respond with ProcessLsif header + Note right of Workhorse: Process LSIF file + Workhorse->>+Object Storage: Put file + Object Storage-->>-Workhorse: request results + Workhorse->>+Rails: POST /:id/artifacts + Rails-->>-Workhorse: request results + Workhorse-->>-Runner: request results +``` + +1. The CI/CD job generates a document in an LSIF format (usually `dump.lsif`) using [an + indexer](https://lsif.dev) for the language of a project. The format + [describes](https://github.com/sourcegraph/sourcegraph/blob/master/doc/user/code_intelligence/writing_an_indexer.md) + interactions between a method or function and its definition(s) or references. The + document is marked to be stored as an LSIF report artifact. + +1. After receiving a request for storing the artifact, Workhorse asks + GitLab Rails to authorize the upload. + +1. GitLab Rails validates whether the artifact can be uploaded and sends + `ProcessLsif: true` header if the lsif artifact can be processed. + +1. Workhorse reads the LSIF document line by line and generates code intelligence + data for each file in the project. The output is a zipped directory of JSON + files which imitates the structure of the project: + + Project: + + ```code + app + controllers + application_controller.rb + models + application.rb + ``` + + Generated data: + + ```code + app + controllers + application_controller.rb.json + models + application.rb.json + ``` + +1. The zipped directory is stored as a ZIP artifact. Workhorse replaces the + original LSIF document with a set of JSON files in the ZIP artifact and + generates metadata for it. The metadata makes it possible to view a single + file in a ZIP file without unpacking or loading the whole file. That allows us + to access code intelligence data for a single file. + +1. When a file is viewed in the GitLab application, frontend fetches code + intelligence data for the file directly from the object storage. The file + contains information about code units in the file. For example: + + ```json + [ + { + "definition_path": "cmd/check/main.go#L4", + "hover": [ + { + "language": "go", + "tokens": [ + [ + { + "class": "kn", + "value": "package" + }, + { + "value": " " + }, + { + "class": "s", + "value": "\"fmt\"" + } + ] + ] + }, + { + "value": "Package fmt implements formatted I/O with functions analogous to C's printf and scanf. The format 'verbs' are derived from C's but are simpler. \n\n### hdr-PrintingPrinting\nThe verbs: \n\nGeneral: \n\n```\n%v\tthe value in a default format\n\twhen printing st..." + } + ], + "start_char": 2, + "start_line": 33 + } + ... + ] + ``` diff --git a/doc/development/code_review.md b/doc/development/code_review.md index 9c6cb1d0237..fd53ce79534 100644 --- a/doc/development/code_review.md +++ b/doc/development/code_review.md @@ -95,17 +95,16 @@ with [domain expertise](#domain-experts). **approved by a [Distribution team member](https://about.gitlab.com/company/team/)**. See how to work with the [Distribution team](https://about.gitlab.com/handbook/engineering/development/enablement/distribution/#how-to-work-with-distribution) for more details. 1. If your merge request includes documentation changes, it must be **approved by a [Technical writer](https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers)**, based on - the appropriate [product category](https://about.gitlab.com/handbook/product/categories/). -1. If your merge request includes Quality and non-Quality-related changes (*3*), it must be **approved + the appropriate [product category](https://about.gitlab.com/handbook/product/product-categories/). +1. If your merge request includes end-to-end **and** non-end-to-end changes (*3*), it must be **approved by a [Software Engineer in Test](https://about.gitlab.com/handbook/engineering/quality/#individual-contributors)**. -1. If your merge request includes _only_ Quality-related changes (*3*), it must be **approved - by a [Quality maintainer](https://about.gitlab.com/handbook/engineering/projects/#gitlab_maintainers_qa)**. +1. If your merge request only includes end-to-end changes (*3*) **or** if the MR author is a [Software Engineer in Test](https://about.gitlab.com/handbook/engineering/quality/#individual-contributors), it must be **approved by a [Quality maintainer](https://about.gitlab.com/handbook/engineering/projects/#gitlab_maintainers_qa)** - (*1*): Please note that specs other than JavaScript specs are considered backend code. - (*2*): We encourage you to seek guidance from a database maintainer if your merge request is potentially introducing expensive queries. It is most efficient to comment on the line of code in question with the SQL queries so they can give their advice. -- (*3*): Quality-related changes include all files within the `qa` directory. +- (*3*): End-to-end changes include all files within the `qa` directory. #### Security requirements @@ -386,9 +385,12 @@ author. - Learning how to find the right balance takes time; that is why we have reviewers that become maintainers after some time spent on reviewing merge requests. -- Finding bugs and improving code style is important, but thinking about good - design is important as well. Building abstractions and good design is what - makes it possible to hide complexity and makes future changes easier. +- Finding bugs is important, but thinking about good design is important as + well. Building abstractions and good design is what makes it possible to hide + complexity and makes future changes easier. +- Enforcing and improving [code style](contributing/style_guides.md) should be primarily done through + [automation](https://about.gitlab.com/handbook/values/#cleanup-over-sign-off) + instead of review comments. - Asking the author to change the design sometimes means the complete rewrite of the contributed code. It's usually a good idea to ask another maintainer or reviewer before doing it, but have the courage to do it when you believe it is diff --git a/doc/development/contributing/issue_workflow.md b/doc/development/contributing/issue_workflow.md index f70299cbfc2..9feaa485bd2 100644 --- a/doc/development/contributing/issue_workflow.md +++ b/doc/development/contributing/issue_workflow.md @@ -81,7 +81,7 @@ already reserved for category labels). The descriptions on the [labels page](https://gitlab.com/groups/gitlab-org/-/labels) explain what falls under each type label. -The GitLab handbook documents [when something is a bug and when it is a feature request](https://about.gitlab.com/handbook/product/product-management/process/feature-or-bug.html). +The GitLab handbook documents [when something is a bug](https://about.gitlab.com/handbook/product/product-processes/#bug-issues) and [when it is a feature request](https://about.gitlab.com/handbook/product/product-processes/#feature-issues). ### Facet labels @@ -104,7 +104,7 @@ Following is a non-exhaustive list of facet labels: ### Stage labels -Stage labels specify which [stage](https://about.gitlab.com/handbook/product/categories/#hierarchy) the issue belongs to. +Stage labels specify which [stage](https://about.gitlab.com/handbook/product/product-categories/#hierarchy) the issue belongs to. #### Naming and color convention @@ -148,7 +148,7 @@ The current group labels can be found by [searching the labels list for `group:: These labels are [scoped labels](../../user/project/labels.md#scoped-labels-premium) and thus are mutually exclusive. -You can find the groups listed in the [Product Stages, Groups, and Categories](https://about.gitlab.com/handbook/product/categories/) page. +You can find the groups listed in the [Product Stages, Groups, and Categories](https://about.gitlab.com/handbook/product/product-categories/) page. We use the term group to map down product requirements from our product stages. As a team needs some way to collect the work their members are planning to be assigned to, we use the `~group::` labels to do so. @@ -167,7 +167,7 @@ Please read [Stage and Group labels in Throughput](https://about.gitlab.com/hand ### Category labels From the handbook's -[Product stages, groups, and categories](https://about.gitlab.com/handbook/product/categories/#hierarchy) +[Product stages, groups, and categories](https://about.gitlab.com/handbook/product/product-categories/#hierarchy) page: > Categories are high-level capabilities that may be a standalone product at @@ -199,7 +199,7 @@ in <https://gitlab.com/gitlab-com/www-gitlab-com/blob/master/data/categories.yml ### Feature labels From the handbook's -[Product stages, groups, and categories](https://about.gitlab.com/handbook/product/categories/#hierarchy) +[Product stages, groups, and categories](https://about.gitlab.com/handbook/product/product-categories/#hierarchy) page: > Features: Small, discrete functionalities. e.g. Issue weights. Some common @@ -312,22 +312,13 @@ We want to avoid a situation when a contributor picks an because we realize that it does not fit our vision, or we want to solve it in a different way. -We add the ~"Accepting merge requests" label to: +We automatically add the ~"Accepting merge requests" label to issues +that match the [triage policy](https://about.gitlab.com/handbook/engineering/quality/triage-operations/#accepting-merge-requests). -- Low priority ~bug issues (i.e. we do not add it to the bugs that we want to - solve in the ~"Next Patch Release") -- Small ~feature -- Small ~"technical debt" issues - -After adding the ~"Accepting merge requests" label, we try to estimate the -[weight](#issue-weight) of the issue. We use issue weight to let contributors -know how difficult the issue is. Additionally: - -- We advertise [`Accepting merge requests` issues with weight < 5](https://gitlab.com/groups/gitlab-org/-/issues?state=opened&label_name[]=Accepting+merge+requests&assignee_id=None&sort=weight) - as suitable for people that have never contributed to GitLab before on the - [Up For Grabs campaign](https://up-for-grabs.net/#/) -- We encourage people that have never contributed to any open source project to - look for [`Accepting merge requests` issues with a weight of 1](https://gitlab.com/groups/gitlab-org/-/issues?state=opened&label_name[]=Accepting+merge+requests&assignee_id=None&sort=weight&weight=1) +We recommend people that have never contributed to any open source project to +look for issues labeled `~"Accepting merge requests"` with a [weight of 1](https://gitlab.com/groups/gitlab-org/-/issues?state=opened&label_name[]=Accepting+merge+requests&assignee_id=None&sort=weight&weight=1) or the `~"Good for 1st time contributors"` [label](https://gitlab.com/gitlab-org/gitlab/-/issues?scope=all&utf8=%E2%9C%93&state=opened&label_name[]=Good%20for%201st%20time%20contributors&assignee_id=None) attached to it. +More experienced contributors are very welcome to tackle +[any of them](https://gitlab.com/groups/gitlab-org/-/issues?state=opened&label_name[]=Accepting+merge+requests&assignee_id=None). If you've decided that you would like to work on an issue, please @-mention the [appropriate product manager](https://about.gitlab.com/handbook/product/#who-to-talk-to-for-what) diff --git a/doc/development/contributing/merge_request_workflow.md b/doc/development/contributing/merge_request_workflow.md index 14c6b0b1073..e5a8bdad7b0 100644 --- a/doc/development/contributing/merge_request_workflow.md +++ b/doc/development/contributing/merge_request_workflow.md @@ -119,8 +119,7 @@ document from the Kubernetes team also has some great points regarding this. When writing commit messages, please follow the guidelines below: - The commit subject must contain at least 3 words. -- The commit subject should ideally contain up to 50 characters, - and must not be longer than 72 characters. +- The commit subject must not be longer than 72 characters. - The commit subject must start with a capital letter. - The commit subject must not end with a period. - The commit subject and body must be separated by a blank line. diff --git a/doc/development/contributing/style_guides.md b/doc/development/contributing/style_guides.md index 9e4870eadc4..ed254052180 100644 --- a/doc/development/contributing/style_guides.md +++ b/doc/development/contributing/style_guides.md @@ -56,6 +56,16 @@ Additionally, we have a dedicated [newlines style guide](../newlines_styleguide.md), as well as dedicated [test-specific style guides and best practices](../testing_guide/index.md). +### Creating new RuboCop cops + +Typically it is better for the linting rules to be enforced programmatically as it +reduces the aforementioned [bike-shedding](https://en.wiktionary.org/wiki/bikeshedding). + +To that end, we encourage creation of new RuboCop rules in the codebase. + +When creating a new cop that could be applied to multiple applications, we encourage you +to add it to our [GitLab Styles](https://gitlab.com/gitlab-org/gitlab-styles) gem. + ## Database migrations See the dedicated [Database Migrations Style Guide](../migration_style_guide.md). @@ -82,7 +92,7 @@ See the dedicated [Shell scripting standards and style guidelines](../shell_scri ## Markdown -We're following [Ciro Santilli's Markdown Style Guide](https://cirosantilli.com/markdown-style-guide). +We're following [Ciro Santilli's Markdown Style Guide](https://cirosantilli.com/markdown-style-guide/). ## Documentation diff --git a/doc/development/dangerbot.md b/doc/development/dangerbot.md index eda9e1e21cc..93434479846 100644 --- a/doc/development/dangerbot.md +++ b/doc/development/dangerbot.md @@ -128,10 +128,18 @@ Here is a (non-exhaustive) list of the kinds of things Danger has been used for at GitLab so far: - Coding style -- Database review workflow -- Documentation review workflow +- Database review +- Documentation review - Merge request metrics -- Reviewer roulette workflow +- Reviewer roulette. Reviewers and maintainers are chosen based on: + - Their roles (backend, frontend, database, etc). + - Their availability: + - No "OOO"/"PTO"/"Parental Leave" in their GitLab or Slack status. + - No `:red_circle:`/`:palm_tree:`/`:beach:`/`:beach_umbrella:`/`:beach_with_umbrella:` emojis in GitLab or Slack status. + - [Experimental] Their timezone: people for which the local hour is between + 6 AM and 2 PM are eligible to be picked. This is to ensure they have a good + chance to get to perform a review during their current work day. The experimentation is tracked in + [this issue](https://gitlab.com/gitlab-org/quality/team-tasks/-/issues/563) - Single codebase effort ## Limitations diff --git a/doc/development/database/add_foreign_key_to_existing_column.md b/doc/development/database/add_foreign_key_to_existing_column.md index b8817eddeec..1b41a52c95e 100644 --- a/doc/development/database/add_foreign_key_to_existing_column.md +++ b/doc/development/database/add_foreign_key_to_existing_column.md @@ -113,7 +113,8 @@ end Validating the foreign key will scan the whole table and make sure that each relation is correct. -NOTE: **Note:** When using [background migrations](../background_migrations.md), foreign key validation should happen in the next GitLab release. +NOTE: **Note:** +When using [background migrations](../background_migrations.md), foreign key validation should happen in the next GitLab release. Migration file for validating the foreign key: diff --git a/doc/development/database/database_reviewer_guidelines.md b/doc/development/database/database_reviewer_guidelines.md new file mode 100644 index 00000000000..894b1ea15f0 --- /dev/null +++ b/doc/development/database/database_reviewer_guidelines.md @@ -0,0 +1,95 @@ +# Database Reviewer Guidelines + +This page includes introductory material for new database reviewers. + +If you are interested in getting an application update reviewed, +check the [database review guidelines](../database_review.md). + +## Scope of work done by a database reviewer + +Database reviewers are domain experts who have substantial experience with databases, +`SQL`, and query performance optimization. + +A database review is required whenever an application update [touches the database](../database_review.md#general-process). + +The database reviewer is tasked with reviewing the database specific updates and +making sure that any queries or modifications will perform without issues +at the scale of GitLab.com. + +For more information on the database review process, check the [database review guidelines](../database_review.md). + +## How to apply for becoming a database reviewer + +Team members are encouraged to self-identify as database domain experts and add it to their [team profile](https://gitlab.com/gitlab-com/www-gitlab-com/-/blob/master/data/team.yml) + +```yaml + projects: + gitlab: + - reviewer database +``` + +Assign the MR which adds your expertise to the `team.yml` file to a database maintainer +or the [Database Team's Engineering Manager](https://about.gitlab.com/handbook/engineering/development/enablement/database/). + +Once the `team.yml` update is merged, the [Reviewer roulette](../code_review.md#reviewer-roulette) +may recommend you as a database reviewer. + +## Resources for database reviewers + +As a database reviewer, join the internal `#database` Slack channel and ask questions or discuss +database related issues with other database reviewers and maintainers. + +There is also an optional database office hours call held bi-weekly, alternating between +European/US and APAC friendly hours. You can join the office hours call and bring topics +that require a more in-depth discussion between the database reviewers and maintainers: + +- [Database Office Hours Agenda](https://docs.google.com/document/d/1wgfmVL30F8SdMg-9yY6Y8djPSxWNvKmhR5XmsvYX1EI/edit). +- [Youtube playlist with past recordings](https://www.youtube.com/playlist?list=PL05JrBw4t0Kp-kqXeiF7fF7cFYaKtdqXM). + +You should also join the [#database-labs](../understanding_explain_plans.md#database-lab) +Slack channel and get familiar with how to use Joe, the slackbot that provides developers +with their own clone of the production database. + +Understanding and efficiently using `EXPLAIN` plans is at the core of the database review process. +The following guides provide a quick introduction and links to follow on more advanced topics: + +- Guide on [understanding EXPLAIN plans](../understanding_explain_plans.md). +- [Explaining the unexplainable series in depesz](https://www.depesz.com/tag/unexplainable/). + +Finally, you can find various guides in the [Database guides](index.md) page that cover more specific +topics and use cases. The most frequently required during database reviewing are the following: + +- [Migrations style guide](../migration_style_guide.md) for creating safe SQL migrations. +- [What requires downtime?](../what_requires_downtime.md). +- [SQL guidelines](../sql.md) for working with SQL queries. + +## How to apply for becoming a database maintainer + +Once a database reviewer feels confident on switching to a database maintainer, +they can update their [team profile](https://gitlab.com/gitlab-com/www-gitlab-com/-/blob/master/data/team.yml) +to a `trainee_maintainer database`: + +```yaml + projects: + gitlab: + - trainee_maintainer database +``` + +The first step is to a create a [Trainee Database Maintainer Issue](https://gitlab.com/gitlab-com/www-gitlab-com/-/issues/new?issuable_template=trainee-database-maintainer). +Use and follow the process described in the 'Trainee database maintainer' template. + +Note that [trainee maintainers](https://about.gitlab.com/handbook/engineering/workflow/code-review/#trainee-maintainer) +are three times as likely to be picked by the [Danger bot](../dangerbot.md) as other reviewers. + +## What to do if you feel overwhelmed + +Similar to all types of reviews, [unblocking others is always a top priority](https://about.gitlab.com/handbook/values/#global-optimization). +Database reviewers are expected to [review assigned merge requests in a timely manner](../code_review.md#review-turnaround-time) +or let the author know as soon as possible and help them find another reviewer or maintainer. + +We are doing reviews to help the rest of the GitLab team and, at the same time, get exposed +to more use cases, get a lot of insights and hone our database and data management skills. + +If you are feeling overwhelmed, think you are at capacity, and are unable to accept any more +reviews until some have been completed, communicate this through your GitLab status by setting +the `:red_circle:` emoji and mentioning that you are at capacity in the status text. diff --git a/doc/development/database/index.md b/doc/development/database/index.md index 665af623059..9ea5b6fcaac 100644 --- a/doc/development/database/index.md +++ b/doc/development/database/index.md @@ -1,5 +1,13 @@ # Database guides +## Database Reviews + +- If you're creating a database MR for review, check out our [Database review guidelines](../database_review.md). + + It provides an introduction on database-related changes, migrations, and complex SQL queries. + +- If you're a database reviewer or want to become one, check out our [introduction to reviewing database changes](database_reviewer_guidelines.md). + ## Tooling - [Understanding EXPLAIN plans](../understanding_explain_plans.md) diff --git a/doc/development/database_debugging.md b/doc/development/database_debugging.md index 629aea5b121..6da0455f392 100644 --- a/doc/development/database_debugging.md +++ b/doc/development/database_debugging.md @@ -1,4 +1,4 @@ -# Database Debugging and Troubleshooting +# Troubleshooting and Debugging Database This section is to help give some copy-pasta you can use as a reference when you run into some head-banging database problems. diff --git a/doc/development/database_review.md b/doc/development/database_review.md index 5405a8808f0..967df411db5 100644 --- a/doc/development/database_review.md +++ b/doc/development/database_review.md @@ -19,6 +19,10 @@ A database review is required for: generally up to the author of a merge request to decide whether or not complex queries are being introduced and if they require a database review. +- Changes in usage data metrics that use `count` and `distinct_count`. + These metrics could have complex queries over large tables. + See the [Telemetry Guide](telemetry/usage_ping.md#implementing-usage-ping) + for implementation details. A database reviewer is expected to look out for obviously complex queries in the change and review those closer. If the author does not @@ -190,7 +194,8 @@ In general, migrations for a single deploy shouldn't take longer than 1 hour for GitLab.com. The following guidelines are not hard rules, they were estimated to keep migration timing to a minimum. -NOTE: **Note:** Keep in mind that all runtimes should be measured against GitLab.com. +NOTE: **Note:** +Keep in mind that all runtimes should be measured against GitLab.com. | Migration Type | Execution Time Recommended | Notes | |----|----|---| diff --git a/doc/development/distributed_tracing.md b/doc/development/distributed_tracing.md index 7fc33380aba..15b3b8ba755 100644 --- a/doc/development/distributed_tracing.md +++ b/doc/development/distributed_tracing.md @@ -66,7 +66,7 @@ GITLAB_TRACING=opentracing://<driver>?<param_name>=<param_value>&<param_name_2>= In this example, we have the following hypothetical values: - `driver`: the driver. [GitLab supports - `jaeger`](../user/project/operations/tracing.md). In future, other + `jaeger`](../operations/tracing.md). In future, other tracing implementations may also be supported. - `param_name`, `param_value`: these are driver specific configuration values. Configuration parameters for Jaeger are documented [further on in this diff --git a/doc/development/documentation/index.md b/doc/development/documentation/index.md index f845a6ec592..2ea26985fcf 100644 --- a/doc/development/documentation/index.md +++ b/doc/development/documentation/index.md @@ -26,7 +26,7 @@ The source of the documentation exists within the codebase of each GitLab applic | --- | --- | | [GitLab](https://gitlab.com/gitlab-org/gitlab/) | [`/doc`](https://gitlab.com/gitlab-org/gitlab/tree/master/doc) | | [GitLab Runner](https://gitlab.com/gitlab-org/gitlab-runner/) | [`/docs`](https://gitlab.com/gitlab-org/gitlab-runner/tree/master/docs) | -| [Omnibus GitLab](https://gitlab.com/gitlab-org/omnibus-gitlab/) | [`/doc`](https://gitlab.com/gitlab-org/gitlab/tree/master/doc) | +| [Omnibus GitLab](https://gitlab.com/gitlab-org/omnibus-gitlab/) | [`/doc`](https://gitlab.com/gitlab-org/omnibus-gitlab/tree/master/doc) | | [Charts](https://gitlab.com/gitlab-org/charts/gitlab) | [`/doc`](https://gitlab.com/gitlab-org/charts/gitlab/tree/master/doc) | Documentation issues and merge requests are part of their respective repositories and all have the label `Documentation`. @@ -70,7 +70,28 @@ See the [Structure](styleguide.md#structure) section of the [Documentation Style ## Metadata To provide additional directives or useful information, we add metadata in YAML -format to the beginning of each product documentation page. +format to the beginning of each product documentation page (YAML front matter). +All values are treated as strings and are only used for the +[docs website](site_architecture/index.md). + +### Stage and group metadata + +Each page should ideally have metadata related to the stage and group it +belongs to, as well as an information block as described below: + +- `stage`: The [Stage](https://about.gitlab.com/handbook/product/product-categories/#devops-stages) + to which the majority of the page's content belongs. +- `group`: The [Group](https://about.gitlab.com/company/team/structure/#product-groups) + to which the majority of the page's content belongs. +- `info`: The following line, which provides direction to contributors regarding + how to contact the Technical Writer associated with the page's Stage and + Group: + + ```plaintext + To determine the technical writer assigned to the Stage/Group + associated with this page, see + https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers + ``` For example, the following metadata would be at the beginning of a product documentation page whose content is primarily associated with the Audit Events @@ -84,24 +105,64 @@ info: To determine the technical writer assigned to the Stage/Group associated w --- ``` -The following list describes the YAML parameters in use: +### Page type metadata + +Originally discussed in [this epic](https://gitlab.com/groups/gitlab-org/-/epics/1280), +each page should have a `type` metadata. It can be one or more of the following: + +- `index`: Index/overview pages. They serve as a list to other pages. Doesn't + necessarily mean the page should be named `index.md`. [Example page](../../install/README.md). +- `concepts`: What you need to know before using product. Informational, not + instructional. For example, abstract ideas, explain meaning or benefit, support + understanding of tasks. They are read for background information, for example + "Why X is important". [Example page](../../topics/autodevops/index.md). +- `howto`: Specific use case instructions. [Example page](../../ssh/README.md). +- `tutorial`: Learn a process/concept by doing. [Example page](../../gitlab-basics/start-using-git.md). +- `reference`: Covers what things are/do. Things like specific settings, facts + without too much explanation that are read for detailed information. + [Example page](../../ci/yaml/README.md). + +### Redirection metadata + +The following metadata should be added when a page is moved to another location: - `redirect_to`: The relative path and filename (with an `.md` extension) of the location to which visitors should be redirected for a moved page. [Learn more](#changing-document-location). -- `stage`: The [Stage](https://about.gitlab.com/handbook/product/categories/#devops-stages) - to which the majority of the page's content belongs. -- `group`: The [Group](https://about.gitlab.com/company/team/structure/#product-groups) - to which the majority of the page's content belongs. -- `info`: The following line, which provides direction to contributors regarding - how to contact the Technical Writer associated with the page's Stage and - Group: `To determine the technical writer assigned to the Stage/Group - associated with this page, see - https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers` - `disqus_identifier`: Identifier for Disqus commenting system. Used to keep comments with a page that's been moved to a new URL. [Learn more](#redirections-for-pages-with-disqus-comments). +### Comments metadata + +The [docs website](site_architecture/index.md) has comments (provided by Disqus) +enabled by default. In case you want to disable them (for example in index pages), +set it to `false`: + +```yaml +--- +comments: false +--- +``` + +### Additional page metadata + +Each page can have additional (optional) metadata (set in the +[default.html](https://gitlab.com/gitlab-org/gitlab-docs/-/blob/fc3577921343173d589dfa43d837b4307e4e620f/layouts/default.html#L30-52) +Nanoc layout), which will be shown to the top of the page if defined: + +- `author`: The name of the author of a page, usually a tutorial. It requires `author_gitlab` in order to be shown. +- `author_gitlab`: The username of the author on GitLab.com. It requires `author` in order to be shown. +- `date`: The date the page was created, usually for tutorials. +- `article_type`: The type of article. Can be either `tutorial` or `user guide`. +- `level`: The level of complexity of a how-to or tutorial. Can be either `beginner`, + `advanced`, or `intermediate`. +- `last_updated`: The date in ISO format when the page was last updated. For example `2020-02-14`. +- `reading_time`: If you want to add an indication of the approximate reading + time of a page, you can set `reading_time` to `true`. This uses a simple + [algorithm](https://gitlab.com/gitlab-org/gitlab-docs/-/blob/master/lib/helpers/reading_time.rb) + to calculate the reading time based on the number of words. + ## Changing document location Changing a document's location requires specific steps to ensure that @@ -226,9 +287,6 @@ it increases the work of the release managers. Every GitLab instance includes the documentation, which is available at `/help` (`https://gitlab.example.com/help`). For example, <https://gitlab.com/help>. -There are [plans](https://gitlab.com/groups/gitlab-org/-/epics/693) to end this -practice and instead link out from the GitLab application to <https://docs.gitlab.com> URLs. - The documentation available online on <https://docs.gitlab.com> is deployed every four hours from the `master` branch of GitLab, Omnibus, and Runner. Therefore, after a merge request gets merged, it will be available online on the same day. However, it will be shipped (and available on `/help`) within the milestone assigned @@ -492,122 +550,138 @@ The output should be similar to: Note that this requires you to either have the required lint tools installed on your machine, or a working Docker installation, in which case an image with these tools pre-installed will be used. -### Local linting +### Local linters To help adhere to the [documentation style guidelines](styleguide.md), and improve the content -added to documentation, consider locally installing and running documentation linters. This will -help you catch common issues before raising merge requests for review of documentation. - -Running the following locally allows you to match the checks in the build pipeline: +added to documentation, [install documentation linters](#install-linters) and +[integrate them with your code editor](#configure-editors). -- [markdownlint](#markdownlint). -- [Vale](#vale). +At GitLab, we mostly use: -NOTE: **Note:** -This list does not limit what other linters you can add to your local documentation writing toolchain. +- [markdownlint](#markdownlint) +- [Vale](#vale) #### markdownlint -[markdownlint](https://github.com/DavidAnson/markdownlint) checks that Markdown -syntax follows [certain rules](https://github.com/DavidAnson/markdownlint/blob/master/doc/Rules.md#rules), -and is used by the [`docs-lint` test](#testing) with a [configuration file](#markdownlint-configuration). -Our [Documentation Style Guide](styleguide.md#markdown) and [Markdown Guide](https://about.gitlab.com/handbook/markdown-guide/) -elaborate on which choices must be made when selecting Markdown syntax for GitLab -documentation. This tool helps catch deviations from those guidelines. - -markdownlint can be used [on the command line](https://github.com/igorshubovych/markdownlint-cli#markdownlint-cli--), -either on a single Markdown file or on all Markdown files in a project. For example, to run -markdownlint on all documentation in the [`gitlab` project](https://gitlab.com/gitlab-org/gitlab), -run the following commands from within your `gitlab` project root directory, which will -automatically detect the [`.markdownlint.json`](#markdownlint-configuration) configuration -file in the root of the project, and test all files in `/doc` and its subdirectories: - -```shell -markdownlint 'doc/**/*.md' -``` +[markdownlint](https://github.com/DavidAnson/markdownlint) checks that Markdown syntax follows +[certain rules](https://github.com/DavidAnson/markdownlint/blob/master/doc/Rules.md#rules), and is +used by the [`docs-lint` test](#testing). -If you wish to use a different configuration file, use the `-c` flag: +Our [Documentation Style Guide](styleguide.md#markdown) and +[Markdown Guide](https://about.gitlab.com/handbook/markdown-guide/) elaborate on which choices must +be made when selecting Markdown syntax for GitLab documentation. This tool helps catch deviations +from those guidelines. -```shell -markdownlint -c <config-file-name> 'doc/**/*.md' -``` +markdownlint configuration is found in the following projects: -##### Run markdownlint in an editor +- [`gitlab`](https://gitlab.com/gitlab-org/gitlab/blob/master/.markdownlint.json) +- [`gitlab-runner`](https://gitlab.com/gitlab-org/gitlab-runner/blob/master/.markdownlint.json) +- [`omnibus-gitlab`](https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/.markdownlint.json) +- [`charts`](https://gitlab.com/gitlab-org/charts/gitlab/-/blob/master/.markdownlint.json) +- [`gitlab-development-kit`](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/master/.markdownlint.json) -markdownlint can also be run as a linter within text editors using [plugins/extensions](https://github.com/DavidAnson/markdownlint#related), -such as: +This configuration is also used within build pipelines. -- [Sublime Text](https://packagecontrol.io/packages/SublimeLinter-contrib-markdownlint) -- [Visual Studio Code](https://marketplace.visualstudio.com/items?itemName=DavidAnson.vscode-markdownlint) -- [Atom](https://atom.io/packages/linter-node-markdownlint) +You can use markdownlint: -It is best to use the [same configuration file](#markdownlint-configuration) as what -is in use in the four repositories that are the sources for <https://docs.gitlab.com>. Each -plugin/extension has different requirements regarding the configuration file, which -is explained in each editor's docs. +- [On the command line](https://github.com/igorshubovych/markdownlint-cli#markdownlint-cli--). +- [Within a code editor](#configure-editors). -##### markdownlint configuration +#### Vale -Each formatting issue that markdownlint checks has an associated -[rule](https://github.com/DavidAnson/markdownlint/blob/master/doc/Rules.md#rules). -These rules are configured in the `.markdownlint.json` files located in the root of -four repositories that are the sources for <https://docs.gitlab.com>: +[Vale](https://errata-ai.gitbook.io/vale/) is a grammar, style, and word usage linter for the +English language. Vale's configuration is stored in the +[`.vale.ini`](https://gitlab.com/gitlab-org/gitlab/blob/master/.vale.ini) file located in the root +directory of projects. -- [`gitlab`](https://gitlab.com/gitlab-org/gitlab/blob/master/.markdownlint.json) -- [`gitlab-runner`](https://gitlab.com/gitlab-org/gitlab-runner/blob/master/.markdownlint.json) -- [`omnibus-gitlab`](https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/.markdownlint.json) -- [`charts`](https://gitlab.com/charts/gitlab/blob/master/.markdownlint.json) +Vale supports creating [custom tests](https://errata-ai.github.io/vale/styles/) that extend any of +several types of checks, which we store in the `.linting/vale/styles/gitlab` directory within the +documentation directory of projects. -By default all rules are enabled, so the configuration file is used to disable unwanted -rules, and also to configure optional parameters for enabled rules as needed. You can -also check [the issue](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/64352) that -tracked the changes required to implement these rules, and details which rules were -on or off when markdownlint was enabled on the docs. +Vale configuration is found in the following projects: -#### Vale +- [`gitlab`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/doc/.vale/gitlab) +- [`gitlab-runner`](https://gitlab.com/gitlab-org/gitlab-runner/-/tree/master/docs/.vale/gitlab) +- [`omnibus-gitlab`](https://gitlab.com/gitlab-org/omnibus-gitlab/-/tree/master/doc/.vale/gitlab) +- [`charts`](https://gitlab.com/gitlab-org/charts/gitlab/-/tree/master/doc/.vale/gitlab) +- [`gitlab-development-kit`](https://gitlab.com/gitlab-org/gitlab-development-kit/-/tree/master/doc/.vale/gitlab) -[Vale](https://errata-ai.gitbook.io/vale/) is a grammar, style, and word usage linter -for the English language. Vale's configuration is stored in the -[`.vale.ini`](https://gitlab.com/gitlab-org/gitlab/blob/master/.vale.ini) file -located in the root directory of the [GitLab repository](https://gitlab.com/gitlab-org/gitlab). +This configuration is also used within build pipelines. -Vale supports creating [custom tests](https://errata-ai.github.io/vale/styles/), -stored in the `doc/.linting/vale/styles/gitlab` directory, that extend any of -several types of checks. +You can use Vale: -To view linting suggestions locally, you must install Vale on your own machine, -and from GitLab's root directory (where `.vale.ini` is located), run: +- [On the command line](https://errata-ai.gitbook.io/vale/getting-started/usage). +- [Within a code editor](#configure-editors). -```shell -vale --glob='*.{md}' doc -``` +#### Install linters + +At a minimum, install [markdownlint](#markdownlint) and [Vale](#vale) to match the checks run in +build pipelines: -Vale's error-level test results [are displayed](#testing) in CI pipelines. +1. Install `markdownlint-cli`, using either: -##### Run Vale in an editor + - `npm`: + + ```shell + npm install -g markdownlint-cli + ``` + + - `yarn`: + + ```shell + yarn global add markdownlint-cli + ``` + + We recommend installing the version of `markdownlint-cli` currently used in the documentation + linting [Docker image](https://gitlab.com/gitlab-org/gitlab-docs/-/blob/master/dockerfiles/Dockerfile.gitlab-docs-lint#L38). + +1. Install [`vale`](https://github.com/errata-ai/vale/releases). For example, to install using + `brew` for macOS, run: + + ```shell + brew install vale + ``` + + We recommend installing the version of Vale currently used in the documentation linting + [Docker image](https://gitlab.com/gitlab-org/gitlab-docs/-/blob/master/dockerfiles/Dockerfile.gitlab-docs-lint#L16). + +In addition to using markdownlint and Vale at the command line, these tools can be +[integrated with your code editor](#configure-editors). + +#### Configure editors + +To configure markdownlint within your editor, install one of the following as appropriate: + +- [Sublime Text](https://packagecontrol.io/packages/SublimeLinter-contrib-markdownlint) +- [Visual Studio Code](https://marketplace.visualstudio.com/items?itemName=DavidAnson.vscode-markdownlint) +- [Atom](https://atom.io/packages/linter-node-markdownlint) -You can run Vale as a linter within your text editor of choice, with: +To configure Vale within your editor, install one of the following as appropriate: - The Sublime Text [`SublimeLinter-contrib-vale` plugin](https://packagecontrol.io/packages/SublimeLinter-contrib-vale) - The Visual Studio Code [`testthedocs.vale` extension](https://marketplace.visualstudio.com/items?itemName=testthedocs.vale) We don't use [Vale Server](https://errata-ai.github.io/vale/#using-vale-with-a-text-editor-or-another-third-party-application). -##### Disable a Vale test +#### Disable Vale tests -You can disable a specific Vale linting rule or all Vale linting rules for any portion of a document: +You can disable a specific Vale linting rule or all Vale linting rules for any portion of a +document: -- To disable a specific rule, add a `<!-- vale gitlab.rulename = NO -->` tag - before the text, and a `<!-- vale gitlab.rulename = YES -->` tag after the text, - replacing `rulename` with the filename of a test in the [GitLab styles](https://gitlab.com/gitlab-org/gitlab/-/tree/master/doc/.linting/vale/styles/gitlab) directory. -- To disable all Vale linting rules, add a `<!-- vale off -->` tag before the text, - and a `<!-- vale on -->` tag after the text. +- To disable a specific rule, add a `<!-- vale gitlab.rulename = NO -->` tag before the text, and a + `<!-- vale gitlab.rulename = YES -->` tag after the text, replacing `rulename` with the filename + of a test in the + [GitLab styles](https://gitlab.com/gitlab-org/gitlab/-/tree/master/doc/.linting/vale/styles/gitlab) + directory. +- To disable all Vale linting rules, add a `<!-- vale off -->` tag before the text, and a + `<!-- vale on -->` tag after the text. -Whenever possible, exclude only the problematic rule and line(s). -In some cases, such as list items, you may need to disable linting for the entire -list until ["Ignore comments are not honored in a Markdown file"](https://github.com/errata-ai/vale/issues/175) is resolved. +Whenever possible, exclude only the problematic rule and line(s). In some cases, such as list items, +you may need to disable linting for the entire list until +[Vale issue #175](https://github.com/errata-ai/vale/issues/175) is resolved. -For more information, see [Vale's documentation](https://errata-ai.gitbook.io/vale/getting-started/markup#markup-based-configuration). +For more information, see +[Vale's documentation](https://errata-ai.gitbook.io/vale/getting-started/markup#markup-based-configuration). ## Danger Bot diff --git a/doc/development/documentation/site_architecture/global_nav.md b/doc/development/documentation/site_architecture/global_nav.md index 71020e6054e..2625fbe0415 100644 --- a/doc/development/documentation/site_architecture/global_nav.md +++ b/doc/development/documentation/site_architecture/global_nav.md @@ -16,6 +16,22 @@ Global navigation (the left-most pane in our three pane documentation) provides: - The ability to refine landing pages, so they don't have to do all the work of surfacing every page contained within the documentation. +## Quick start + +To add a topic to the global nav, go to the directory that contains +[navigation files](https://gitlab.com/gitlab-org/gitlab-docs/blob/master/content/_data/) +and edit the `yaml` file for your product area. You can copy an existing nav entry and +edit it to point to your topic. + +The files are: + +| File | Document | Location | +|-----------------------|--------------------------------------------------------------------|-------------------------------------------------------| +| `charts-nav.yaml` | GitLab cloud native Helm Chart | `https://docs.gitlab.com/charts/` | +| `default-nav.yaml` | GitLab Docs | `https://docs.gitlab.com/ee/` | +| `omnibus-nav.yaml` | Omnibus GitLab Docs | `https://docs.gitlab.com/omnibus/` | +| `runner-nav.yaml` | GitLab Runner Docs | `https://docs.gitlab.com/runner/` | + ## Adding new items All new pages need a new navigation item. Without a navigation, the page becomes "orphaned". That @@ -98,7 +114,7 @@ for clarity. To see the improvements planned, check the [global nav epic](https://gitlab.com/groups/gitlab-com/-/epics/21). -CAUTION: **Attention!** +NOTE: **Note:** **Do not** [add items](#adding-new-items) to the global nav without the consent of one of the technical writers. @@ -275,7 +291,7 @@ and the following syntax rules. an "information" icon on the nav to make the user aware that the feature is EE-only. -DANGER: **Important!** +CAUTION: **Caution:** All links present on the data file must end in `.html`, not `.md`. Do not start any relative link with a forward slash `/`. @@ -290,7 +306,7 @@ Examples: docs: - doc_title: Service Desk doc_url: 'user/project/service_desk.html' - ee_only: true + ee_only: false # note that the URL above ends in html and, as the # document is EE-only, the attribute ee_only is set to true. ``` diff --git a/doc/development/documentation/site_architecture/index.md b/doc/development/documentation/site_architecture/index.md index 942b202a3ec..60e6d4bcb13 100644 --- a/doc/development/documentation/site_architecture/index.md +++ b/doc/development/documentation/site_architecture/index.md @@ -111,7 +111,7 @@ located in the [Dockerfiles directory](https://gitlab.com/gitlab-org/gitlab-docs If you need to rebuild the Docker images immediately (must have maintainer level permissions): -CAUTION: **Caution** +CAUTION: **Caution:** If you change the dockerfile configuration and rebuild the images, you can break the master pipeline in the main `gitlab` repository as well as in `gitlab-docs`. Create an image with a different name first and test it to ensure you do not break the pipelines. @@ -152,9 +152,9 @@ Suppose we have the `content/_data/versions.yaml` file with the content: ```yaml versions: -- 10.6 -- 10.5 -- 10.4 + - 10.6 + - 10.5 + - 10.4 ``` We can then loop over the `versions` array with something like: diff --git a/doc/development/documentation/structure.md b/doc/development/documentation/structure.md index eadcedfaac0..4cfc57aa57b 100644 --- a/doc/development/documentation/structure.md +++ b/doc/development/documentation/structure.md @@ -20,7 +20,7 @@ Every feature or use case document should include the following content in the f with exceptions and details noted below and in the template included on this page. - **Title**: Top-level heading with the feature name, or a use case name, which would start with - a verb, like Configuring, Enabling, and so on. + a verb, like "Configure", "Enable", and so on. - **Introduction**: A couple sentences about the subject matter and what's to be found on this page. Describe what the feature or topic is, what it does, and in what context it should be used. There is no need to add a title called "Introduction" or "Overview," because people rarely @@ -47,10 +47,13 @@ When done, remove all of this commented-out text, except a commented-out Trouble which, if empty, can be left in place to encourage future use.--> --- description: "Short document description." # Up to ~200 chars long. They will be displayed in Google Search snippets. It may help to write the page intro first, and then reuse it here. +stage: "Add the stage name here, and remove the quotation marks" +group: "Add the group name here, and remove the quotation marks" +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # Feature Name or Use Case Name **[TIER]** (1) -<!--If writing about a use case, drop the tier, and start with a verb, e.g. 'Configuring', 'Implementing', + the goal/scenario--> +<!--If writing about a use case, drop the tier, and start with a verb, e.g. "Configure", "Implement", + the goal/scenario--> <!--For pages on newly introduced features, add the following line. If only some aspects of the feature have been introduced, specify what parts of the feature.--> > [Introduced](link_to_issue_or_mr) in GitLab (Tier) X.Y (2). @@ -99,12 +102,12 @@ Larger instruction sets may have subsections covering specific phases of the pro Where appropriate, provide examples of code or configuration files to better clarify intended usage. - Write a step-by-step guide, with no gaps between the steps. -- Include example code or configurations as part of the relevant step. Use appropriate markdown to [wrap code blocks with syntax highlighting](../../user/markdown.md#colored-code-and-syntax-highlighting). +- Include example code or configurations as part of the relevant step. Use appropriate Markdown to [wrap code blocks with syntax highlighting](../../user/markdown.md#colored-code-and-syntax-highlighting). - Start with an h2 (`##`), break complex steps into small steps using subheadings h3 > h4 > h5 > h6. _Never skip a hierarchy level, such as h2 > h4_, as it will break the TOC and may affect the breadcrumbs. - Use short and descriptive headings (up to ~50 chars). You can use one -single heading like `## Configuring X` for instructions when the feature +single heading like `## Configure X` for instructions when the feature is simple and the document is short. <!-- ## Troubleshooting diff --git a/doc/development/documentation/styleguide.md b/doc/development/documentation/styleguide.md index 9a93dbf4f94..f2c90e71bd5 100644 --- a/doc/development/documentation/styleguide.md +++ b/doc/development/documentation/styleguide.md @@ -88,7 +88,7 @@ New information that would be useful toward the future usage or troubleshooting The more we reflexively add useful information to the docs, the more (and more successfully) the docs will be used to efficiently accomplish tasks and solve problems. -If you have questions when considering, authoring, or editing docs, ask the Technical Writing team on Slack in `#docs` or in GitLab by mentioning the writer for the applicable [DevOps stage](https://about.gitlab.com/handbook/product/categories/#devops-stages). Otherwise, forge ahead with your best effort. It does not need to be perfect; the team is happy to review and improve upon your content. Please review the [Documentation guidelines](index.md) before you begin your first documentation MR. +If you have questions when considering, authoring, or editing docs, ask the Technical Writing team on Slack in `#docs` or in GitLab by mentioning the writer for the applicable [DevOps stage](https://about.gitlab.com/handbook/product/product-categories/#devops-stages). Otherwise, forge ahead with your best effort. It does not need to be perfect; the team is happy to review and improve upon your content. Please review the [Documentation guidelines](index.md) before you begin your first documentation MR. Having a knowledge base in any form that is separate from the documentation would be against the docs-first methodology because the content would overlap with the documentation. @@ -325,9 +325,22 @@ tenses, words, and phrases: - Avoid using the word *currently* when talking about the product or its features. The documentation describes the product as it is, and not as it will be at some indeterminate point in the future. +- Avoid the using the word *scalability* with increasing GitLab's performance + for additional users. Using the words *scale* or *scaling* in other cases is + acceptable, but references to increasing GitLab's performance for additional + users should direct readers to the GitLab + [reference architectures](../../administration/reference_architectures/index.md) + page. +- Avoid all forms of the phrases *high availability* and *HA*, and instead + direct readers to the GitLab [reference architectures](../../administration/reference_architectures/index.md) + for information about configuring GitLab to have the performance needed for + additional users over time. - Don't use profanity or obscenities. Doing so may negatively affect other users and contributors, which is contrary to GitLab's value of - [diversity and inclusion](https://about.gitlab.com/handbook/values/#diversity-inclusion). + [Diversity, Inclusion, and Belonging](https://about.gitlab.com/handbook/values/#diversity-inclusion). +- Avoid the use of [racially-insensitive terminology or phrases](https://www.marketplace.org/2020/06/17/tech-companies-update-language-to-avoid-offensive-terms/). For example: + - Use *primary* and *secondary* for database and server relationships. + - Use *allowlist* and *denylist* to describe access control lists. ### Word usage clarifications @@ -662,7 +675,7 @@ For other punctuation rules, please refer to the - Avoid adding things that show ephemeral statuses. For example, if a feature is considered beta or experimental, put this information in a note, not in the heading. - When introducing a new document, be careful for the headings to be - grammatically and syntactically correct. Mention an [assigned technical writer (TW)](https://about.gitlab.com/handbook/product/categories/) + grammatically and syntactically correct. Mention an [assigned technical writer (TW)](https://about.gitlab.com/handbook/product/product-categories/) for review. This is to ensure that no document with wrong heading is going live without an audit, thus preventing dead links and redirection issues when @@ -750,6 +763,15 @@ _Internal_ refers to documentation in the same project. When linking to document separate projects (for example, linking to Omnibus docs from GitLab docs), you must use absolute URLs. +Do not use absolute URLs like `https://docs.gitlab.com/ee/index.html` to crosslink +to other docs within the same project. Use relative links to the file, like `../index.md`. (These are converted to HTML when the site is rendered.) + +Relative linking enables crosslinks to work: + +- in Review Apps, local previews, and `/help`. +- when working on the docs locally, so you can verify that they work as early as possible in the process. +- within the GitLab UI when browsing doc files in their respective repositories. For example, the links displayed at `https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/README.md`. + To link to internal documentation: - Use relative links to Markdown files in the same repository. @@ -778,7 +800,7 @@ To link to internal documentation: - `../../issues/tags.md` - `../../issues/tags.md#stages` -NOTE: **Note**: +NOTE: **Note:** Using the Markdown extension is necessary for the [`/help`](index.md#gitlab-help) section of GitLab. ### Links to external documentation @@ -839,19 +861,42 @@ To indicate the steps of navigation through the UI: ## Images +Images, including screenshots, can help a reader better understand a concept. +However, they can be hard to maintain, and should be used sparingly. + +Before including an image in the documentation, ensure it provides value to the reader. + +### Capture the image + +Use images to help the reader understand where they are in a process, or how they need to +interact with the application. + +When you take screenshots: + +- *Capture the most relevant area of the page.* Don't include unnecessary white + space or areas of the page that don't help illustrate your point. Also, don't + include the entire page if you don't have to, but also ensure the image + contains enough information to allow the user to determine where things are. +- *Be consistent.* Find a browser window size that works for you that also + displays all areas of the product, including the left navigation (usually > + 1200px wide). For consistency, use this browser window size for your + screenshots by installing a browser extension for setting a window to a + specific size (for example, + [Window Resizer](https://chrome.google.com/webstore/detail/window-resizer/kkelicaakdanhinjdeammmilcgefonfh/related?hl=en) + for Google Chrome). + +### Save the image + +- Save the image with a lowercase file name that is descriptive of the feature + or concept in the image. If the image is of the GitLab interface, append the + GitLab version to the file name, based on the following format: + `image_name_vX_Y.png`. For example, for a screenshot taken from the pipelines + page of GitLab 11.1, a valid name is `pipelines_v11_1.png`. If you're adding an + illustration that doesn't include parts of the user interface, add the release + number corresponding to the release the image was added to; for an MR added to + 11.1's milestone, a valid name for an illustration is `devops_diagram_v11_1.png`. - Place images in a separate directory named `img/` in the same directory where the `.md` document that you're working on is located. -- Images should have a specific, non-generic name that will - differentiate and describe them properly. -- For screenshots of GitLab software, append the GitLab version the screenshot was taken from to the - file name. Use the following format: `image_name_vX_Y.png`. -- For example, for a screenshot taken from the pipelines page of - GitLab 11.1, a valid name is `pipelines_v11_1.png`. If you're - adding an illustration that does not include parts of the UI, - add the release number corresponding to the release the image - was added to. Example, for an MR added to 11.1's milestone, - a valid name for an illustration is `devops_diagram_v11_1.png`. -- Keep all file names in lower case. - Consider using PNG images instead of JPEG. - [Compress all PNG images](#compress-images). - Compress gifs with <https://ezgif.com/optimize> or similar tool. @@ -860,15 +905,20 @@ To indicate the steps of navigation through the UI: - Max image size: 100KB (gifs included). - See also how to link and embed [videos](#videos) to illustrate the docs. -Inside the document: +### Add the image link to content + +The Markdown code for including an image in a document is: +`` + +The image description is the alt text for the rendered image on the docs site. +For accessibility and SEO, use [descriptions](https://webaim.org/techniques/alttext/) +that: + +- Are accurate, succinct, and unique. +- Don't use *image of …* or *graphic of…* to describe the image. -- The Markdown way of using an image inside a document is: - `` -- Always use a proper description for what the image is about. That way, when a - browser fails to show the image, this text will be used as an alternative - description. -- If a heading is placed right after an image, always add three dashes (`---`) - between the image and the heading. +Also, if a heading immediately follows an image, be sure to add three dashes +(`---`) between the image and the heading. ### Remove image shadow @@ -1133,8 +1183,8 @@ The following are examples of source Markdown for menu items with their publishe Whenever you need to call special attention to particular sentences, use the following markup for highlighting. -_Note that the alert boxes only work for one paragraph only. Multiple paragraphs, -lists, headers and so on, will not render correctly. For multiple lines, use blockquotes instead._ +Note that the alert boxes only work for one paragraph only. Multiple paragraphs, +lists, headers and so on, will not render correctly. For multiple lines, use [blockquotes](#blockquotes) instead. Alert boxes only render on the GitLab Docs site (<https://docs.gitlab.com>). Within GitLab itself, they will appear as plain Markdown text (like the examples @@ -1271,20 +1321,20 @@ The following are styles to follow when describing UI elements on a screen: ### Verbs for UI elements -The following are recommended verbs for specific uses. +The following are recommended verbs for specific uses with UI elements: -| Recommended | Used for | Alternatives | -|:------------|:---------------------------|:---------------------------| -| "click" | buttons, links, menu items | "hit", "press", "select" | -| "check" | checkboxes | "enable", "click", "press" | -| "select" | dropdowns | "pick" | -| "expand" | expandable sections | "open" | +| Recommended | Used for | Replaces | +|:--------------------|:---------------------------|:---------------------------| +| *click* | buttons, links, menu items | "hit", "press", "select" | +| *select* or *clear* | checkboxes | "enable", "click", "press" | +| *select* | dropdowns | "pick" | +| *expand* | expandable sections | "open" | ### Other Verbs -| Recommended | Used for | Alternatives | -|:------------|:--------------------------------|:-------------------| -| "go" | making a browser go to location | "navigate", "open" | +| Recommended | Used for | Replaces | +|:------------|:--------------------------------|:----------------------| +| *go to* | making a browser go to location | "navigate to", "open" | ## GitLab versions and tiers @@ -1296,6 +1346,11 @@ Tagged and released versions of GitLab documentation are available: The version introducing a new feature is added to the top of the topic in the documentation to provide a helpful link back to how the feature was developed. +TIP: **Tip:** +Whenever you have documentation related to the `gitlab.rb` file, you're working with a self-managed installation. +The section or page is therefore likely to apply only to self-managed instances. +If so, the relevant "`TIER` ONLY" [Product badge](#product-badges) should be included at the highest applicable heading level. + ### Text for documentation requiring version text - For features that need to declare the GitLab version that the feature was introduced. Text similar @@ -1327,6 +1382,22 @@ a helpful link back to how the feature was developed. > - [Moved](<link-to-issue>) to GitLab Core in 12.0. ``` +- If a feature is deprecated, include a link to a replacement (when available): + + ```markdown + > - [Deprecated](<link-to-issue>) in GitLab 11.3. Replaced by [meaningful text](<link-to-appropriate-documentation>). + ``` + + It's also acceptable to describe the replacement in surrounding text, if available. + + If the deprecation is not obvious in existing text, you may want to include a warning such as: + + ```markdown + CAUTION: **Warning:** + This feature was [deprecated](link-to-issue) in GitLab 12.3 + and replaced by [Feature name](link-to-feature-documentation). + ``` + NOTE: **Note:** Version text must be on its own line and surrounded by blank lines to render correctly. @@ -1389,7 +1460,7 @@ lines with an inserted line break. Splitting product or feature names across lines makes searching for these items more difficult, and can cause problems if names change. -For example, the followng Markdown content is *not* formatted correctly: +For example, the following Markdown content is *not* formatted correctly: ```markdown When entering a product or feature name that includes a space (such as GitLab @@ -1430,7 +1501,7 @@ For GitLab.com only tiers (when the feature is not available for self-managed in The tier should be ideally added to headers, so that the full badge will be displayed. However, it can be also mentioned from paragraphs, list items, and table cells. For these cases, -the tier mention will be represented by an orange question mark that will show the tiers on hover. +the tier mention will be represented by an orange info icon **(information)** that will show the tiers on hover. Use the lowest tier at the page level, even if higher-level tiers exist on the page. For example, you might have a page that is marked as Starter but a section badged as Premium. @@ -1542,6 +1613,9 @@ can facilitate this by making sure the troubleshooting content addresses: 1. How the user can confirm they have the problem. 1. Steps the user can take towards resolution of the problem. +If the contents of each category can be summarized in one line and a list of steps aren't required, consider setting up a +[table](#tables) with headers of *Problem* \| *Cause* \| *Solution* (or *Workaround* if the fix is temporary), or *Error message* \| *Solution*. + ## Feature flags Learn how to [document features deployed behind flags](feature_flags.md). @@ -1607,6 +1681,13 @@ and email address of a user. Don't use real user information in API calls: - **Names**: Use strings like `Example Username`. Alternatively, use diverse or non-gendered names with common surnames, such as `Sidney Jones`, `Zhang Wei`. or `Maria Garcia`. +### Fake URLs + +When including sample URLs in the documentation, use: + +- `example.com` when the domain name is generic. +- `gitlab.example.com` when referring to self-managed instances of GitLab. + ### Fake tokens There may be times where a token is needed to demonstrate an API call using diff --git a/doc/development/ee_features.md b/doc/development/ee_features.md index e22e96b6f06..ac544113cbd 100644 --- a/doc/development/ee_features.md +++ b/doc/development/ee_features.md @@ -512,12 +512,12 @@ do that, so we'll follow regular object-oriented practices that we define the interface first here. For example, suppose we have a few more optional parameters for EE. We can move the -parameters out of the `Grape::API` class to a helper module, so we can inject it +parameters out of the `Grape::API::Instance` class to a helper module, so we can inject it before it would be used in the class. ```ruby module API - class Projects < Grape::API + class Projects < Grape::API::Instance helpers Helpers::ProjectsHelpers end end @@ -578,7 +578,7 @@ class definition to make it easy and clear: ```ruby module API - class JobArtifacts < Grape::API + class JobArtifacts < Grape::API::Instance # EE::API::JobArtifacts would override the following helpers helpers do def authorize_download_artifacts! @@ -622,7 +622,7 @@ route. Something like this: ```ruby module API - class MergeRequests < Grape::API + class MergeRequests < Grape::API::Instance helpers do # EE::API::MergeRequests would override the following helpers def update_merge_request_ee(merge_request) @@ -691,7 +691,7 @@ least argument. We would approach this as follows: ```ruby # api/merge_requests/parameters.rb module API - class MergeRequests < Grape::API + class MergeRequests < Grape::API::Instance module Parameters def self.update_params_at_least_one_of %i[ @@ -707,7 +707,7 @@ API::MergeRequests::Parameters.prepend_if_ee('EE::API::MergeRequests::Parameters # api/merge_requests.rb module API - class MergeRequests < Grape::API + class MergeRequests < Grape::API::Instance params do at_least_one_of(*Parameters.update_params_at_least_one_of) end @@ -900,27 +900,79 @@ export default { </template> ``` -#### For JS code that is EE only, like props, computed properties, methods, etc, we will keep the current approach +#### For JS code that is EE only, like props, computed properties, methods, etc -- Since we [can't async load a mixin](https://github.com/vuejs/vue-loader/issues/418#issuecomment-254032223) we will use the [`ee_else_ce`](../development/ee_features.md#javascript-code-in-assetsjavascripts) alias we already have for webpack. - - This means all the EE specific props, computed properties, methods, etc that are EE only should be in a mixin in the `ee/` folder and we need to create a CE counterpart of the mixin +- Please do not use mixins unless ABSOLUTELY NECESSARY. Please try to find an alternative pattern. -##### Example +##### Reccomended alternative approach (named/scoped slots) -```javascript -import mixin from 'ee_else_ce/path/mixin'; +- We can use slots and/or scoped slots to achieve the same thing as we did with mixins. If you only need an EE component there is no need to create the CE component. + +1. First, we have a CE component that can render a slot incase we need EE template and functionality to be decorated on top of the CE base. + +```vue +// ./ce/my_component.vue + +<script> +export default { + props: { + tooltipDefaultText: { + type: String, + }, + }, + computed: { + tooltipText() { + return this.tooltipDefaultText || "5 issues please"; + } + }, +} +</script> + +<template> + <span v-gl-tooltip :title="tooltipText" class="ce-text">Community Edition Only Text</span> + <slot name="ee-specific-component"> +</template> +``` + +1. Next, we render the EE component, and inside of the EE component we render the CE component and add additional content in the slot. -{ - mixins: [mixin] +```vue +// ./ee/my_component.vue + +<script> +export default { + computed: { + tooltipText() { + if (this.weight) { + return "5 issues with weight 10"; + } + } + }, + methods: { + submit() { + // do something. + } + }, } +</script> + +<template> + <my-component :tooltipDefaultText="tooltipText"> + <template #ee-specific-component> + <span class="some-ee-specific">EE Specific Value</span> + <button @click="submit">Click Me</button> + </template> + </my-component> +</template> ``` -- Computed Properties/methods and getters only used in the child import still need a counterpart in CE +1. Finally, wherever the component is needed we can require it like so + +`import MyComponent from 'ee_else_ce/path/my_component'.vue` -- For store modules, we will need a CE counterpart too. -- You can see an MR with an example [here](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/9762) +- this way the correct component will be included for either the ce or ee implementation -#### `template` tag +**For EE components that need different results for the same computed values, we can pass in props to the CE wrapper as seen in the example.** - **EE Child components** - Since we are using the async loading to check which component to load, we'd still use the component's name, check [this example](#child-component-only-used-in-ee). diff --git a/doc/development/elasticsearch.md b/doc/development/elasticsearch.md index 9f54386f1af..90debab3b5c 100644 --- a/doc/development/elasticsearch.md +++ b/doc/development/elasticsearch.md @@ -60,7 +60,7 @@ The `whitespace` tokenizer was selected in order to have more control over how t Please see the `code` filter for an explanation on how tokens are split. -NOTE: **Known Issues**: +NOTE: **Note:** Currently the [Elasticsearch code_analyzer doesn't account for all code cases](../integration/elasticsearch.md#known-issues). #### `code_search_analyzer` @@ -111,11 +111,8 @@ Patterns: - `'"((?:\\"|[^"]|\\")*)"'`: captures terms inside quotes, removing the quotes - `"'((?:\\'|[^']|\\')*)'"`: same as above, for single-quotes - `'\.([^.]+)(?=\.|\s|\Z)'`: separate terms with periods in-between -- `'\/?([^\/]+)(?=\/|\b)'`: separate path terms `like/this/one` - -#### `edgeNGram_filter` - -Uses an [Edge NGram token filter](https://www.elastic.co/guide/en/elasticsearch/reference/5.5/analysis-edgengram-tokenfilter.html) to allow inputs with only parts of a token to find the token. For example it would turn `glasses` into permutations starting with `gl` and ending with `glasses`, which would allow a search for "`glass`" to find the original token `glasses` +- `'([\p{L}_.-]+)'`: some common chars in file names to keep the whole filename intact (eg. `my_file-ñame.txt`) +- `'([\p{L}\d_]+)'`: letters, numbers and underscores are the most common tokens in programming. Always capture them greedily regardless of context. ## Gotchas @@ -160,7 +157,8 @@ The global configurations per version are now in the `Elastic::(Version)::Config ### Creating new version of schema -NOTE: **Note:** this is not applicable yet as multiple indices functionality is not fully implemented. +NOTE: **Note:** +This is not applicable yet as multiple indices functionality is not fully implemented. Folders like `ee/lib/elastic/v12p1` contain snapshots of search logic from different versions. To keep a continuous Git history, the latest version lives under `ee/lib/elastic/latest`, but its classes are aliased under an actual version (e.g. `ee/lib/elastic/v12p3`). When referencing these classes, never use the `Latest` namespace directly, but use the actual version (e.g. `V12p3`). @@ -222,6 +220,16 @@ be used both locally in development and on any deployed GitLab instance to diagnose poor search performance. This will show the exact queries being made, which is useful to diagnose why a search might be slow. +### Correlation ID and X-Opaque-Id + +Our [correlation +ID](./distributed_tracing.md#developer-guidelines-for-working-with-correlation-ids) +is forwarded by all requests from Rails to Elasticsearch as the +[`X-Opaque-Id`](https://www.elastic.co/guide/en/elasticsearch/reference/current/tasks.html#_identifying_running_tasks) +header which allows us to track any +[tasks](https://www.elastic.co/guide/en/elasticsearch/reference/current/tasks.html) +in the cluster back the request in GitLab. + ## Troubleshooting ### Getting `flood stage disk watermark [95%] exceeded` diff --git a/doc/development/emails.md b/doc/development/emails.md index 2477a51f78f..cf7f49ee834 100644 --- a/doc/development/emails.md +++ b/doc/development/emails.md @@ -115,7 +115,7 @@ Examples of valid email keys: - `1234567890abcdef1234567890abcdef-unsubscribe` (unsubscribe from a conversation) - `1234567890abcdef1234567890abcdef` (reply to a conversation) -Please note that the action `-issue-` is used in GitLab Premium as the handler for the Service Desk feature. +Please note that the action `-issue-` is used in GitLab as the handler for the Service Desk feature. ### Legacy format @@ -127,7 +127,7 @@ These are the only valid legacy formats for an email handler: - `namespace` - `namespace+action` -Please note that `path/to/project` is used in GitLab Premium as handler for the Service Desk feature. +Please note that `path/to/project` is used in GitLab as the handler for the Service Desk feature. --- diff --git a/doc/development/experiment_guide/index.md b/doc/development/experiment_guide/index.md index f0e05139cba..b1a15c7749f 100644 --- a/doc/development/experiment_guide/index.md +++ b/doc/development/experiment_guide/index.md @@ -1,12 +1,12 @@ # Experiment Guide -Experiments will be conducted by teams from the [Growth Section](https://about.gitlab.com/handbook/engineering/development/growth/) and are not tied to releases, because they will primarily target GitLab.com. +Experiments can be conducted by any GitLab team, most often the teams from the [Growth Sub-department](https://about.gitlab.com/handbook/engineering/development/growth/). Experiments are not tied to releases because they will primarily target GitLab.com. Experiments will be run as an A/B test and will be behind a feature flag to turn the test on or off. Based on the data the experiment generates, the team will decide if the experiment had a positive impact and will be the new default or rolled back. -## Follow-up issue +## Experiment tracking issue -Each experiment requires a follow-up issue to resolve the experiment. Immediately after an experiment is deployed, the deadline of the issue needs to be set (this depends on the experiment but can be up to a few weeks in the future). +Each experiment should have an [Experiment tracking](https://gitlab.com/groups/gitlab-org/-/issues?scope=all&utf8=%E2%9C%93&state=opened&label_name[]=growth%20experiment&search=%22Experiment+tracking%22) issue to track the experiment from roll-out through to cleanup/removal. Immediately after an experiment is deployed, the due date of the issue should be set (this depends on the experiment but can be up to a few weeks in the future). After the deadline, the issue needs to be resolved and either: - It was successful and the experiment will be the new default. @@ -17,10 +17,10 @@ In either case, an outcome of the experiment should be posted to the issue with ## Code reviews Since the code of experiments will not be part of the codebase for a long time and we want to iterate fast to retrieve data,the code quality of experiments might sometimes not fulfill our standards but should not negatively impact the availability of GitLab whether the experiment is running or not. -Experiments will only be deployed to a fraction of users but we still want a flawless experience for those users. Therefore, experiments still require tests. +Initially experiments will only be deployed to a fraction of users but we still want a flawless experience for those users. Therefore, experiments still require tests. For reviewers and maintainers: if you find code that would usually not make it through the review, but is temporarily acceptable, please mention your concerns but note that it's not necessary to change. -The author then adds a comment to this piece of code and adds a link to the issue that resolves the experiment. +The author then adds a comment to this piece of code and adds a link to the issue that resolves the experiment. If the experiment is successful and becomes part of the product these follow up issues should be addressed. ## How to create an A/B test @@ -42,7 +42,7 @@ The author then adds a comment to this piece of code and adds a link to the issu - Use the experiment in a controller: ```ruby - class RegistrationController < Applicationcontroller + class RegistrationController < ApplicationController def show # experiment_enabled?(:feature_name) is also available in views and helpers if experiment_enabled?(:signup_flow) diff --git a/doc/development/fe_guide/accessibility.md b/doc/development/fe_guide/accessibility.md index 998c71135fb..669c93eb251 100644 --- a/doc/development/fe_guide/accessibility.md +++ b/doc/development/fe_guide/accessibility.md @@ -1,4 +1,4 @@ -# Accessibility +# Accessibility & Readability ## Resources diff --git a/doc/development/fe_guide/development_process.md b/doc/development/fe_guide/development_process.md index 892e931bf5b..ff36b8b5c6a 100644 --- a/doc/development/fe_guide/development_process.md +++ b/doc/development/fe_guide/development_process.md @@ -73,8 +73,8 @@ With the purpose of being [respectful of others' time](https://about.gitlab.com/ - Before assigning to a maintainer, assign to a reviewer. - If you assigned a merge request or pinged someone directly, be patient because we work in different timezones and asynchronously. Unless the merge request is urgent (like fixing a broken master), please don't DM or reassign the merge request before waiting for a 24-hour window. - If you have a question regarding your merge request/issue, make it on the merge request/issue. When we DM each other, we no longer have a SSOT and [no one else is able to contribute](https://about.gitlab.com/handbook/values/#public-by-default). -- When you have a big WIP merge request with many changes, you're advised to get the review started before adding/removing significant code. Make sure it is assigned well before the release cut-off, as the reviewer(s)/maintainer(s) would always prioritize reviewing finished MRs before WIP ones. -- Make sure to remove the WIP title before the last round of review. +- When you have a big **Draft** merge request with many changes, you're advised to get the review started before adding/removing significant code. Make sure it is assigned well before the release cut-off, as the reviewer(s)/maintainer(s) would always prioritize reviewing finished MRs before the **Draft** ones. +- Make sure to remove the `Draft:` title before the last round of review. ### Share your work early diff --git a/doc/development/fe_guide/frontend_faq.md b/doc/development/fe_guide/frontend_faq.md index 4f814f3cdde..3c0845a9aaa 100644 --- a/doc/development/fe_guide/frontend_faq.md +++ b/doc/development/fe_guide/frontend_faq.md @@ -146,3 +146,20 @@ export const fetchFoos = ({ state }) => { return axios.get(state.settings.fooPath); }; ``` + +### 7. How can I test the production build locally? + +Sometimes it's necessary to test locally what the frontend production build would produce, to do so the steps are: + +1. Stop webpack: `gdk stop webpack`. +1. Open `gitlab.yaml` located in your `gitlab` installation folder, scroll down to the `webpack` section and change `dev_server` to `enabled: false`. +1. Run `yarn webpack-prod && gdk restart rails-web`. + +The production build takes a few minutes to be completed; any code change at this point will be +displayed only after executing the item 3 above again. +To return to the normal development mode: + +1. Open `gitlab.yaml` located in your `gitlab` installation folder, scroll down to the `webpack` section and change back `dev_server` to `enabled: true`. +1. Run `yarn clean` to remove the production assets and free some space (optional). +1. Start webpack again: `gdk start webpack`. +1. Restart GDK: `gdk-restart rails-web`. diff --git a/doc/development/fe_guide/graphql.md b/doc/development/fe_guide/graphql.md index 191ebd2ff58..3d74ec94ae4 100644 --- a/doc/development/fe_guide/graphql.md +++ b/doc/development/fe_guide/graphql.md @@ -646,7 +646,7 @@ defaultClient.query({ query }) .then(result => console.log(result)); ``` -When [using Vuex](#Using-with-Vuex), disable the cache when: +When [using Vuex](#using-with-vuex), disable the cache when: - The data is being cached elsewhere - The use case does not need caching diff --git a/doc/development/fe_guide/icons.md b/doc/development/fe_guide/icons.md index 131324e6479..7078b5e9b2f 100644 --- a/doc/development/fe_guide/icons.md +++ b/doc/development/fe_guide/icons.md @@ -61,6 +61,7 @@ export default { <gl-icon name="issues" :size="24" + class="class-name" /> </template> ``` @@ -68,7 +69,7 @@ export default { - **name** Name of the Icon in the SVG Sprite ([Overview is available here](https://gitlab-org.gitlab.io/gitlab-svgs)). - **size (optional)** Number value for the size which is then mapped to a specific CSS class (Available Sizes: 8, 12, 16, 18, 24, 32, 48, 72 are mapped to `sXX` CSS classes) -- **css-classes (optional)** Additional CSS Classes to add to the SVG tag. +- **class (optional)** Additional CSS Classes to add to the SVG tag. ### Usage in HTML/JS diff --git a/doc/development/fe_guide/index.md b/doc/development/fe_guide/index.md index 8fd6ba459b9..3a2b3cac9bf 100644 --- a/doc/development/fe_guide/index.md +++ b/doc/development/fe_guide/index.md @@ -17,7 +17,9 @@ Working with our frontend assets requires Node (v10.13.0 or greater) and Yarn For our currently-supported browsers, see our [requirements](../../install/requirements.md#supported-web-browsers). -Use [BrowserStack](https://www.browserstack.com/) to test with our supported browsers. Login to BrowserStack with the credentials saved in GitLab's [shared 1Password account](https://about.gitlab.com/handbook/security/#1password-for-teams). +Use [BrowserStack](https://www.browserstack.com/) to test with our supported browsers. +Sign in to BrowserStack with the credentials saved in the **Engineering** vault of GitLab's +[shared 1Password account](https://about.gitlab.com/handbook/security/#1password-guide). ## Initiatives diff --git a/doc/development/fe_guide/tooling.md b/doc/development/fe_guide/tooling.md index 585cd969c96..28deb7d95f9 100644 --- a/doc/development/fe_guide/tooling.md +++ b/doc/development/fe_guide/tooling.md @@ -4,6 +4,44 @@ We use ESLint to encapsulate and enforce frontend code standards. Our configuration may be found in the [`gitlab-eslint-config`](https://gitlab.com/gitlab-org/gitlab-eslint-config) project. +### Yarn Script + +This section describes yarn scripts that are available to validate and apply automatic fixes to files using ESLint. + +To check all currently staged files (based on `git diff`) with ESLint, run the following script: + +```shell +yarn eslint-staged +``` + +_A list of problems found will be logged to the console._ + +To apply automatic ESLint fixes to all currently staged files (based on `git diff`), run the following script: + +```shell +yarn eslint-staged-fix +``` + +_If manual changes are required, a list of changes will be sent to the console._ + +To check **all** files in the repository with ESLint, run the following script: + +```shell +yarn eslint +``` + +_A list of problems found will be logged to the console._ + +To apply automatic ESLint fixes to **all** files in the repository, run the following script: + +```shell +yarn eslint-fix +``` + +_If manual changes are required, a list of changes will be sent to the console._ + +**Caution:** Limit use to global rule updates. Otherwise, the changes can lead to huge Merge Requests. + ### Disabling ESLint in new files Do not disable ESLint when creating new files. Existing files may have existing rules @@ -56,13 +94,15 @@ When declaring multiple globals, always use one `/* global [name] */` line per v ## Formatting with Prettier -Our code is automatically formatted with [Prettier](https://prettier.io) to follow our style guides. Prettier is taking care of formatting `.js`, `.vue`, and `.scss` files based on the standard prettier rules. You can find all settings for Prettier in `.prettierrc`. +> Support for `.graphql` [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/227280) in GitLab 13.2. + +Our code is automatically formatted with [Prettier](https://prettier.io) to follow our style guides. Prettier is taking care of formatting `.js`, `.vue`, `.graphql`, and `.scss` files based on the standard prettier rules. You can find all settings for Prettier in `.prettierrc`. ### Editor The easiest way to include prettier in your workflow is by setting up your preferred editor (all major editors are supported) accordingly. We suggest setting up prettier to run automatically when each file is saved. Find [here](https://prettier.io/docs/en/editors.html) the best way to set it up in your preferred editor. -Please take care that you only let Prettier format the same file types as the global Yarn script does (`.js`, `.vue`, and `.scss`). In VSCode by example you can easily exclude file formats in your settings file: +Please take care that you only let Prettier format the same file types as the global Yarn script does (`.js`, `.vue`, `.graphql`, and `.scss`). In VSCode by example you can easily exclude file formats in your settings file: ```json "prettier.disableLanguages": [ @@ -131,6 +171,9 @@ To select Prettier as a formatter, add the following properties to your User or }, "[vue]": { "editor.defaultFormatter": "esbenp.prettier-vscode" + }, + "[graphql]": { + "editor.defaultFormatter": "esbenp.prettier-vscode" } } ``` @@ -150,5 +193,8 @@ To automatically format your files with Prettier, add the following properties t "[vue]": { "editor.formatOnSave": true }, + "[graphql]": { + "editor.formatOnSave": true + }, } ``` diff --git a/doc/development/fe_guide/vue.md b/doc/development/fe_guide/vue.md index 0d77e4d129b..2a0556c6cda 100644 --- a/doc/development/fe_guide/vue.md +++ b/doc/development/fe_guide/vue.md @@ -53,14 +53,14 @@ Be sure to read about [page-specific JavaScript](./performance.md#page-specific- #### Providing data from HAML to JavaScript -While mounting a Vue application may be a need to provide data from Rails to JavaScript. -To do that, provide the data through `data` attributes in the HTML element and query them while mounting the application. +While mounting a Vue application, you might need to provide data from Rails to JavaScript. +To do that, you can use the `data` attributes in the HTML element and query them while mounting the application. _Note:_ You should only do this while initializing the application, because the mounted element will be replaced with Vue-generated DOM. The advantage of providing data from the DOM to the Vue instance through `props` in the `render` function -instead of querying the DOM inside the main Vue component is that makes tests easier by avoiding the need to -create a fixture or an HTML element in the unit test. See the following example: +instead of querying the DOM inside the main Vue component is avoiding the need to create a fixture or an HTML element in the unit test, +which will make the tests easier. See the following example: ```javascript // haml @@ -134,7 +134,7 @@ This approach has a few benefits: intermediate components being aware of it (c.f. passing the flag down via props). - Good testability, since the flag can be provided to `mount`/`shallowMount` - from `vue-test-utils` as easily as a prop. + from `vue-test-utils` simply as a prop. ```javascript import { shallowMount } from '@vue/test-utils'; @@ -155,7 +155,7 @@ This folder holds all components that are specific of this new feature. If you need to use or create a component that will probably be used somewhere else, please refer to `vue_shared/components`. -A good thumb rule to know when you should create a component is to think if +A good rule of thumb to know when you should create a component is to think if it will be reusable elsewhere. For example, tables are used in a quite amount of places across GitLab, a table @@ -321,7 +321,7 @@ We should verify an event has been fired by asserting against the result of the ## Vue.js Expert Role -One should apply to be a Vue.js expert by opening an MR when the Merge Request's they create and review show: +You should only apply to be a Vue.js expert when your own merge requests and your reviews show: - Deep understanding of Vue and Vuex reactivity - Vue and Vuex code are structured according to both official and our guidelines diff --git a/doc/development/fe_guide/vuex.md b/doc/development/fe_guide/vuex.md index e7be67b8da5..02387c15951 100644 --- a/doc/development/fe_guide/vuex.md +++ b/doc/development/fe_guide/vuex.md @@ -201,6 +201,72 @@ By following this pattern we guarantee: 1. All data in the application follows the same lifecycle pattern 1. Unit tests are easier +#### Updating complex state + +Sometimes, especially when the state is complex, is really hard to traverse the state to precisely update what the mutation needs to update. +Ideally a `vuex` state should be as normalized/decoupled as possible but this is not always the case. + +It's important to remember that the code is much easier to read and maintain when the `portion of the mutated state` is selected and mutated in the mutation itself. + +Given this state: + +```javascript + export default () => ({ + items: [ + { + id: 1, + name: 'my_issue', + closed: false, + }, + { + id: 2, + name: 'another_issue', + closed: false, + } + ] +}); +``` + +It may be tempting to write a mutation like so: + +```javascript +// Bad +export default { + [types.MARK_AS_CLOSED](state, item) { + Object.assign(item, {closed: true}) + } +} +``` + +While this approach works it has several dependencies: + +- Correct selection of `item` in the component/action. +- The `item` property is already declared in the `closed` state. + - A new `confidential` property would not be reactive. +- Noting that `item` is referenced by `items` + +A mutation written like this is harder to maintain and more error prone. We should rather write a mutation like this: + +```javascript +// Good +export default { + [types.MARK_AS_CLOSED](state, itemId) { + const item = state.items.find(i => i.id == itemId); + Vue.set(item, 'closed', true) + + state.items.splice(index, 1, item) + } +} +``` + +This approach is better because: + +- It selects and updates the state in the mutation, which is more maintainable. +- It has no external dependencies, if the correct `itemId` is passed the state is correctly updated. +- It does not have reactivity caveats, as we generate a new `item` to avoid coupling to the initial state. + +A mutation written like this is easier to maintain. In addition, we avoid errors due to the limitation of the reactivity system. + ### `getters.js` Sometimes we may need to get derived state based on store state, like filtering for a specific prop. diff --git a/doc/development/feature_categorization/index.md b/doc/development/feature_categorization/index.md new file mode 100644 index 00000000000..bcd5e16cc2b --- /dev/null +++ b/doc/development/feature_categorization/index.md @@ -0,0 +1,130 @@ +# Feature Categorization + +> [Introduced](https://gitlab.com/groups/gitlab-com/gl-infra/-/epics/269) in GitLab 13.2. + +Each Sidekiq worker, controller action, or (eventually) API endpoint +must declare a `feature_category` attribute. This attribute maps each +of these to a [feature +category](https://about.gitlab.com/handbook/product/product-categories/). This +is done for error budgeting, alert routing, and team attribution. + +The list of feature categories can be found in the file `config/feature_categories.yml`. +This file is generated from the +[`stages.yml`](https://gitlab.com/gitlab-com/www-gitlab-com/blob/master/data/stages.yml) +data file used in the GitLab Handbook and other GitLab resources. + +## Updating `config/feature_categories.yml` + +Occasionally new features will be added to GitLab stages, groups, and +product categories. When this occurs, you can automatically update +`config/feature_categories.yml` by running +`scripts/update-feature-categories`. This script will fetch and parse +[`stages.yml`](https://gitlab.com/gitlab-com/www-gitlab-com/blob/master/data/stages.yml) +and generate a new version of the file, which needs to be committed to +the repository. + +The [Scalabilitity +team](https://about.gitlab.com/handbook/engineering/infrastructure/team/scalability) +currently maintains the `stages.yml` file. They will automatically be +notified on Slack when the file becomes outdated. + +## Sidekiq workers + +The declaration uses the `feature_category` class method, as shown below. + +```ruby +class SomeScheduledTaskWorker + include ApplicationWorker + + # Declares that this worker is part of the + # `continuous_integration` feature category + feature_category :continuous_integration + + # ... +end +``` + +The feature categories specified using `feature_category` should be +defined in +[`config/feature_categories.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/feature_categories.yml). If +not, the specs will fail. + +### Excluding Sidekiq workers from feature categorization + +A few Sidekiq workers, that are used across all features, cannot be mapped to a +single category. These should be declared as such using the `feature_category_not_owned!` +declaration, as shown below: + +```ruby +class SomeCrossCuttingConcernWorker + include ApplicationWorker + + # Declares that this worker does not map to a feature category + feature_category_not_owned! + + # ... +end +``` + +## Rails controllers + +Specifying feature categories on controller actions can be done using +the `feature_category` class method. + +A feature category can be specified on an entire controller +using: + +```ruby +class Projects::MergeRequestsController < ApplicationController + feature_category :source_code_management +end +``` + +The feature category can be limited to a list of actions using the +`only` argument, actions can be excluded using the `except` argument. + +```ruby +class Projects::MergeRequestsController < ApplicationController + feature_category :code_testing, only: [:metrics_reports] + feature_category :source_code_management, except: [:test_reports, :coverage_reports] +end +``` + +`except` and `only` arguments can not be combined. + +When specifying `except` all other actions will get the specified +category assigned. + +The assignment can also be scoped using `if` and `unless` procs: + +```ruby +class Projects::MergeRequestsController < ApplicationController + feature_category :source_code_management, + unless: -> (action) { action.include?("reports") } + if: -> (action) { action.include?("widget") } +end +``` + +In this case, both procs need to be satisfied for the action to get +the category assigned. + +### Excluding controller actions from feature categorization + +In the rare case an action cannot be tied to a feature category this +can be done using the `not_owned` feature category. + +```ruby +class Admin::LogsController < ApplicationController + feature_category :not_owned +end +``` + +### Ensuring feature categories are valid + +The `spec/controllers/every_controller_spec.rb` will iterate over all +defined routes, and check the controller to see if a category is +assigned to all actions. + +The spec also validates if the used feature categories are known. And +if the actions used in `only` and `except` configuration still exist +as routes. diff --git a/doc/development/feature_flags/controls.md b/doc/development/feature_flags/controls.md index d4d1ec74591..9e7ce74cc0c 100644 --- a/doc/development/feature_flags/controls.md +++ b/doc/development/feature_flags/controls.md @@ -210,11 +210,20 @@ actors. Feature.enabled?(:some_feature, group) ``` -NOTE: - +NOTE: **Note:** **Percentage of time** rollout is not a good idea if what you want is to make sure a feature is always on or off to the users. In that case, **Percentage of actors** rollout is a better method. +Lastly, to verify that the feature is deemed stable in as many cases as possible, +you should fully roll out the feature by enabling the flag **globally** by running: + +```shell +/chatops run feature set some_feature true +``` + +This changes the feature flag state to be **enabled** always, which overrides the +existing gates (e.g. `--group=gitlab-org`) in the above processes. + ### Feature flag change logging Any feature flag change that affects GitLab.com (production) will diff --git a/doc/development/feature_flags/development.md b/doc/development/feature_flags/development.md index a44bc70439e..0b918478668 100644 --- a/doc/development/feature_flags/development.md +++ b/doc/development/feature_flags/development.md @@ -32,8 +32,8 @@ request removing the feature flag or the merge request where the default value o the feature flag is set to true. If the feature contains any DB migration it should include a changelog entry for DB changes. -In the rare case that you need the feature flag to be on automatically, use -`default_enabled: true` when checking: +If you need the feature flag to be on automatically, use `default_enabled: true` +when checking: ```ruby Feature.enabled?(:feature_flag, project, default_enabled: true) diff --git a/doc/development/feature_flags/index.md b/doc/development/feature_flags/index.md index bd0bd8f2018..0c1e34edc6f 100644 --- a/doc/development/feature_flags/index.md +++ b/doc/development/feature_flags/index.md @@ -1,6 +1,6 @@ # Feature flags in development of GitLab -[Feature Flags](../../user/project/operations/feature_flags.md) +[Feature Flags](../../operations/feature_flags.md) can be used to gradually roll out changes, be it a new feature, or a performance improvement. By using feature flags, we can comfortably measure the impact of our changes, while still being able to easily diff --git a/doc/development/feature_flags/process.md b/doc/development/feature_flags/process.md index 57360f5b771..b053838964b 100644 --- a/doc/development/feature_flags/process.md +++ b/doc/development/feature_flags/process.md @@ -4,7 +4,7 @@ This document only covers feature flags used in the development of GitLab itself. Feature flags in deployed user applications can be found at -[Feature Flags feature documentation](../../user/project/operations/feature_flags.md). +[Feature Flags feature documentation](../../operations/feature_flags.md). ## Feature flags in GitLab development @@ -21,6 +21,19 @@ should be leveraged: - Merge requests that make changes hidden behind a feature flag, or remove an existing feature flag because a feature is deemed stable must have the ~"feature flag" label assigned. +- When development of a feature will be spread across multiple merge + requests, you can use the following workflow: + + 1. Introduce a feature flag which is **off** by default, in the first merge request. + 1. Submit incremental changes via one or more merge requests, ensuring that any + new code added can only be reached if the feature flag is **on**. + You can keep the feature flag enabled on your local GDK during development. + 1. When the feature is ready to be tested, enable the feature flag for + a specific project and ensure that there are no issues with the implementation. + 1. When the feature is ready to be announced, create a merge request that adds + documentation about the feature, including [documentation for the feature flag itself](../documentation/feature_flags.md), + and a changelog entry. In the same merge request either flip the feature flag to + be **on by default** or remove it entirely in order to enable the new behavior. One might be tempted to think that feature flags will delay the release of a feature by at least one month (= one release). This is not the case. A feature @@ -29,6 +42,8 @@ flag does not have to stick around for a specific amount of time is deemed stable. Stable means it works on GitLab.com without causing any problems, such as outages. +Please also read the [development guide for feature flags](development.md). + ### When to use feature flags Starting with GitLab 11.4, developers are required to use feature flags for @@ -56,7 +71,9 @@ absolutely no way to use the feature until it is enabled. In order to build a final release and present the feature for self-managed users, the feature flag should be at least defaulted to **on**. If the feature is deemed stable and there is confidence that removing the feature flag is safe, -consider removing the feature flag altogether. +consider removing the feature flag altogether. It's _strongly_ recommended that +the feature flag is [enabled **globally** on **production**](./controls.md#enabling-a-feature-for-gitlabcom) for **at least one day** +before making this decision. Unexpected bugs are sometimes discovered during this period. The process for enabling features that are disabled by default can take 5-6 days from when the merge request is first reviewed to when the change is deployed to @@ -71,7 +88,7 @@ Take into consideration that such action can make the feature available on GitLab.com shortly after the change to the feature flag is merged. Changing the default state or removing the feature flag has to be done before -the 22nd of the month, _at least_ 2 working days before, in order for the change +the 22nd of the month, _at least_ 3-4 working days before, in order for the change to be included in the final self-managed release. In addition to this, the feature behind feature flag should: diff --git a/doc/development/foreign_keys.md b/doc/development/foreign_keys.md index 508e5665f08..8a81dc158a7 100644 --- a/doc/development/foreign_keys.md +++ b/doc/development/foreign_keys.md @@ -35,6 +35,17 @@ When adding a foreign key in PostgreSQL the column is not indexed automatically, thus you must also add a concurrent index. Not doing so will result in cascading deletes being very slow. +## Naming foreign keys + +By default Ruby on Rails uses the `_id` suffix for foreign keys. So we should +only use this suffix for associations between two tables. If you want to +reference an ID on a third party platform the `_xid` suffix is recommended. + +The spec `spec/db/schema_spec.rb` will test if all columns with the `_id` suffix +have a foreign key constraint. So if that spec fails, don't add the column to +`IGNORED_FK_COLUMNS`, but instead add the FK constraint, or consider naming it +differently. + ## Dependent Removals Don't define options such as `dependent: :destroy` or `dependent: :delete` when diff --git a/doc/development/geo/framework.md b/doc/development/geo/framework.md index 85fecc41cfb..de4d6fb869d 100644 --- a/doc/development/geo/framework.md +++ b/doc/development/geo/framework.md @@ -1,11 +1,13 @@ # Geo self-service framework (alpha) -NOTE: **Note:** This document might be subjected to change. It's a +NOTE: **Note:** +This document might be subjected to change. It's a proposal we're working on and once the implementation is complete this documentation will be updated. Follow progress in the [epic](https://gitlab.com/groups/gitlab-org/-/epics/2161). -NOTE: **Note:** The Geo self-service framework is currently in +NOTE: **Note:** +The Geo self-service framework is currently in alpha. If you need to replicate a new data type, reach out to the Geo team to discuss the options. You can contact them in `#g_geo` on Slack or mention `@geo-team` in the issue or merge request. @@ -178,13 +180,17 @@ For example, to add support for files referenced by a `Widget` model with a mount_uploader :file, WidgetUploader + def self.replicables_for_geo_node + # Should be implemented. The idea of the method is to restrict + # the set of synced items depending on synchronization settings + end ... end ``` 1. Create `ee/app/replicators/geo/widget_replicator.rb`. Implement the `#carrierwave_uploader` method which should return a `CarrierWave::Uploader`. - And implement the private `#model` method to return the `Widget` class. + And implement the class method `.model` to return the `Widget` class. ```ruby # frozen_string_literal: true @@ -193,14 +199,12 @@ For example, to add support for files referenced by a `Widget` model with a class WidgetReplicator < Gitlab::Geo::Replicator include ::Geo::BlobReplicatorStrategy - def carrierwave_uploader - model_record.file + def self.model + ::Widget end - private - - def model - ::Widget + def carrierwave_uploader + model_record.file end end end @@ -215,7 +219,7 @@ For example, to add support for files referenced by a `Widget` model with a require 'spec_helper' - describe Geo::WidgetReplicator do + RSpec.describe Geo::WidgetReplicator do let(:model_record) { build(:widget) } it_behaves_like 'a blob replicator' @@ -231,20 +235,32 @@ For example, to add support for files referenced by a `Widget` model with a class CreateWidgetRegistry < ActiveRecord::Migration[6.0] DOWNTIME = false - def change - create_table :widget_registry, id: :serial, force: :cascade do |t| - t.integer :widget_id, null: false - t.integer :state, default: 0, null: false - t.integer :retry_count, default: 0 - t.string :last_sync_failure, limit: 255 - t.datetime_with_timezone :retry_at - t.datetime_with_timezone :last_synced_at - t.datetime_with_timezone :created_at, null: false - - t.index :widget_id, name: :index_widget_registry_on_repository_id, using: :btree - t.index :retry_at, name: :index_widget_registry_on_retry_at, using: :btree - t.index :state, name: :index_widget_registry_on_state, using: :btree + disable_ddl_transaction! + + def up + unless table_exists?(:widget_registry) + ActiveRecord::Base.transaction do + create_table :widget_registry, id: :bigserial, force: :cascade do |t| + t.integer :widget_id, null: false + t.integer :state, default: 0, null: false, limit: 2 + t.integer :retry_count, default: 0, limit: 2 + t.text :last_sync_failure + t.datetime_with_timezone :retry_at + t.datetime_with_timezone :last_synced_at + t.datetime_with_timezone :created_at, null: false + + t.index :widget_id + t.index :retry_at + t.index :state + end + end end + + add_text_limit :widget_registry, :last_sync_failure, 255 + end + + def down + drop_table :widget_registry end end ``` @@ -255,21 +271,28 @@ For example, to add support for files referenced by a `Widget` model with a # frozen_string_literal: true class Geo::WidgetRegistry < Geo::BaseRegistry - include Geo::StateMachineRegistry + include Geo::ReplicableRegistry + MODEL_CLASS = ::Widget MODEL_FOREIGN_KEY = :widget_id belongs_to :widget, class_name: 'Widget' end ``` + The method `has_create_events?` should return `true` in most of the cases. + However, if the entity you add doesn't have the create event, don't add the + method at all. + +1. Update `REGISTRY_CLASSES` in `ee/app/workers/geo/secondary/registry_consistency_worker.rb`. + 1. Create `ee/spec/factories/geo/widget_registry.rb`: ```ruby # frozen_string_literal: true FactoryBot.define do - factory :widget_registry, class: 'Geo::WidgetRegistry' do + factory :geo_widget_registry, class: 'Geo::WidgetRegistry' do widget state { Geo::WidgetRegistry.state_value(:pending) } @@ -301,14 +324,18 @@ For example, to add support for files referenced by a `Widget` model with a require 'spec_helper' - describe Geo::WidgetRegistry, :geo, type: :model do - let_it_be(:registry) { create(:widget_registry) } + RSpec.describe Geo::WidgetRegistry, :geo, type: :model do + let_it_be(:registry) { create(:geo_widget_registry) } specify 'factory is valid' do expect(registry).to be_valid end include_examples 'a Geo framework registry' + + describe '.find_registry_differences' do + ... # To be implemented + end end ``` @@ -360,36 +387,58 @@ Widgets should now be replicated by Geo! end ``` -1. Add fields `widget_count`, `widget_checksummed_count`, `widget_checksum_failed_count`, - `widget_synced_count` and `widget_failed_count` - to `GeoNodeStatus#RESOURCE_STATUS_FIELDS` array in `ee/app/models/geo_node_status.rb`. +To do: Add verification on secondaries. This should be done as part of +[Geo: Self Service Framework - First Implementation for Package File verification](https://gitlab.com/groups/gitlab-org/-/epics/1817) + +Widgets should now be verified by Geo! + +#### Metrics + +Metrics are gathered by `Geo::MetricsUpdateWorker`, persisted in +`GeoNodeStatus` for display in the UI, and sent to Prometheus. + +1. Add fields `widget_count`, `widget_checksummed_count`, + `widget_checksum_failed_count`, `widget_synced_count`, + `widget_failed_count`, and `widget_registry_count` to + `GeoNodeStatus#RESOURCE_STATUS_FIELDS` array in + `ee/app/models/geo_node_status.rb`. 1. Add the same fields to `GeoNodeStatus#PROMETHEUS_METRICS` hash in `ee/app/models/geo_node_status.rb`. 1. Add the same fields to `Sidekiq metrics` table in `doc/administration/monitoring/prometheus/gitlab_metrics.md`. -1. Add the same fields to `GET /geo_nodes/status` example response in `doc/api/geo_nodes.md`. -1. Modify `GeoNodeStatus#load_verification_data` to make sure the fields mantioned above - are set: +1. Add the same fields to `GET /geo_nodes/status` example response in + `doc/api/geo_nodes.md`. +1. Add the same fields to `ee/spec/models/geo_node_status_spec.rb` and + `ee/spec/factories/geo_node_statuses.rb`. +1. Set `widget_count` in `GeoNodeStatus#load_data_from_current_node`: + + ```ruby + self.widget_count = Geo::WidgetReplicator.primary_total_count + ``` + +1. Add `GeoNodeStatus#load_widgets_data` to set `widget_synced_count`, + `widget_failed_count`, and `widget_registry_count`: ```ruby - self.widget_count = Geo::WidgetReplicator.model.count - self.widget_checksummed_count = Geo::WidgetReplicator.checksummed.count - self.widget_checksum_failed_count = Geo::WidgetReplicator.checksum_failed.count + def load_widget_data self.widget_synced_count = Geo::WidgetReplicator.synced_count self.widget_failed_count = Geo::WidgetReplicator.failed_count + self.widget_registry_count = Geo::WidgetReplicator.registry_count + end ``` -1. Make sure `Widget` model has `checksummed` and `checksum_failed` scopes. -1. Update `ee/spec/fixtures/api/schemas/public_api/v4/geo_node_status.json` with new fields. -1. Update `GeoNodeStatus#PROMETHEUS_METRICS` hash in `ee/app/models/geo_node_status.rb` with new fields. -1. Update `Sidekiq metrics` table in `doc/administration/monitoring/prometheus/gitlab_metrics.md` with new fields. -1. Update `GET /geo_nodes/status` example response in `doc/api/geo_nodes.md` with new fields. -1. Update `ee/spec/models/geo_node_status_spec.rb` and `ee/spec/factories/geo_node_statuses.rb` with new fields. +1. Call `GeoNodeStatus#load_widgets_data` in + `GeoNodeStatus#load_secondary_data`. -To do: Add verification on secondaries. This should be done as part of -[Geo: Self Service Framework - First Implementation for Package File verification](https://gitlab.com/groups/gitlab-org/-/epics/1817) +1. Set `widget_checksummed_count` and `widget_checksum_failed_count` in + `GeoNodeStatus#load_verification_data`: -Widgets should now be verified by Geo! + ```ruby + self.widget_checksummed_count = Geo::WidgetReplicator.checksummed_count self.widget_checksum_failed_count = Geo::WidgetReplicator.checksum_failed_count + ``` + +Widget replication and verification metrics should now be available in the API, +the Admin Area UI, and Prometheus! #### GraphQL API @@ -428,8 +477,8 @@ Widgets should now be verified by Geo! require 'spec_helper' - describe Resolvers::Geo::WidgetRegistriesResolver do - it_behaves_like 'a Geo registries resolver', :widget_registry + RSpec.describe Resolvers::Geo::WidgetRegistriesResolver do + it_behaves_like 'a Geo registries resolver', :geo_widget_registry end ``` @@ -452,8 +501,8 @@ Widgets should now be verified by Geo! require 'spec_helper' - describe Geo::WidgetRegistryFinder do - it_behaves_like 'a framework registry finder', :widget_registry + RSpec.describe Geo::WidgetRegistryFinder do + it_behaves_like 'a framework registry finder', :geo_widget_registry end ``` @@ -484,7 +533,7 @@ Widgets should now be verified by Geo! require 'spec_helper' - describe GitlabSchema.types['WidgetRegistry'] do + RSpec.describe GitlabSchema.types['WidgetRegistry'] do it_behaves_like 'a Geo registry type' it 'has the expected fields (other than those included in RegistryType)' do @@ -503,7 +552,7 @@ Widgets should now be verified by Geo! it_behaves_like 'gets registries for', { field_name: 'widgetRegistries', registry_class_name: 'WidgetRegistry', - registry_factory: :widget_registry, + registry_factory: :geo_widget_registry, registry_foreign_key_field_name: 'widgetId' } ``` @@ -511,6 +560,13 @@ Widgets should now be verified by Geo! Individual widget synchronization and verification data should now be available via the GraphQL API! +1. Take care of replicating "update" events. Geo Framework does not currently support + replicating "update" events because all entities added to the framework, by this time, + are immutable. If this is the case + for the entity you're going to add, please follow <https://gitlab.com/gitlab-org/gitlab/-/issues/118743> + and <https://gitlab.com/gitlab-org/gitlab/-/issues/118745> as examples to add the new event type. + Please also remove this notice when you've added it. + #### Admin UI To do: This should be done as part of diff --git a/doc/development/gitaly.md b/doc/development/gitaly.md index 417c96a44dd..8b4e5090abb 100644 --- a/doc/development/gitaly.md +++ b/doc/development/gitaly.md @@ -1,7 +1,14 @@ -# GitLab Developers Guide to Working with Gitaly +--- +stage: Create +group: Gitaly +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: reference +--- -[Gitaly](https://gitlab.com/gitlab-org/gitaly) is a high-level Git RPC service used by GitLab CE/EE, -Workhorse and GitLab-Shell. +# Gitaly developers guide + +[Gitaly](https://gitlab.com/gitlab-org/gitaly) is a high-level Git RPC service used by GitLab Rails, +Workhorse and GitLab Shell. ## Deep Dive @@ -114,7 +121,8 @@ bundle exec rake gitlab:features:disable_rugged Most of this code exists in the `lib/gitlab/git/rugged_impl` directory. -NOTE: **Note:** You should NOT need to add or modify code related to +NOTE: **Note:** +You should NOT need to add or modify code related to Rugged unless explicitly discussed with the [Gitaly Team](https://gitlab.com/groups/gl-gitaly/group_members). This code will NOT work on GitLab.com or other GitLab instances that do not use NFS. diff --git a/doc/development/go_guide/index.md b/doc/development/go_guide/index.md index 5a5e163e142..779dcd9b7a2 100644 --- a/doc/development/go_guide/index.md +++ b/doc/development/go_guide/index.md @@ -414,7 +414,7 @@ it will display its help message (if `cli` has been used). With the exception of [GitLab Runner](https://gitlab.com/gitlab-org/gitlab-runner), which publishes its own binaries, our Go binaries are created by projects -managed by the [Distribution group](https://about.gitlab.com/handbook/product/categories/#distribution-group). +managed by the [Distribution group](https://about.gitlab.com/handbook/product/product-categories/#distribution-group). The [Omnibus GitLab](https://gitlab.com/gitlab-org/omnibus-gitlab) project creates a single, monolithic operating system package containing all the binaries, while @@ -453,6 +453,28 @@ are: To reduce unnecessary differences between two distribution methods, Omnibus and CNG **should always use the same Go version**. +### Supporting multiple Go versions + +Individual Golang-projects need to support multiple Go versions for the following reasons: + +1. When a new Go release is out, we should start integrating it into the CI pipelines to verify compatibility with the new compiler. +1. We must support the [Omnibus official Go version](#updating-go-version), which may be behind the latest minor release. +1. When Omnibus switches Go version, we still may need to support the old one for security backports. + +These 3 requirements may easily be satisfied by keeping support for the 3 latest minor versions of Go. + +It's ok to drop support for the oldest Go version and support only 2 latest releases, +if this is enough to support backports to the last 3 GitLab minor releases. + +Example: + +In case we want to drop support for `go 1.11` in GitLab `12.10`, we need to verify which Go versions we are using in `12.9`, `12.8`, and `12.7`. + +We will not consider the active milestone, `12.10`, because a backport for `12.7` will be required in case of a critical security release. + +1. If both [Omnibus and CNG](#updating-go-version) were using Go `1.12` since GitLab `12.7`, then we safely drop support for `1.11`. +1. If Omnibus or CNG were using `1.11` in GitLab `12.7`, then we still need to keep support for Go `1.11` for easier backporting of security fixes. + ## Secure Team standards and style guidelines The following are some style guidelines that are specific to the Secure Team. diff --git a/doc/development/gotchas.md b/doc/development/gotchas.md index 7e08da8162b..34fe3f11489 100644 --- a/doc/development/gotchas.md +++ b/doc/development/gotchas.md @@ -20,7 +20,7 @@ Consider the following API spec: ```ruby require 'spec_helper' -describe API::Labels do +RSpec.describe API::Labels do it 'creates a first label' do create(:label) @@ -71,7 +71,7 @@ Following is the fixed API spec: ```ruby require 'spec_helper' -describe API::Labels do +RSpec.describe API::Labels do it 'creates a first label' do create(:label, title: 'foo') diff --git a/doc/development/i18n/externalization.md b/doc/development/i18n/externalization.md index bdd372e90ed..bce95dfc24e 100644 --- a/doc/development/i18n/externalization.md +++ b/doc/development/i18n/externalization.md @@ -95,7 +95,8 @@ Active Record's `:message` option accepts a `Proc`, so we can do this instead: validates :group_id, uniqueness: { scope: [:project_id], message: -> (object, data) { _("already shared with this group") } } ``` -NOTE: **Note:** Messages in the API (`lib/api/` or `app/graphql`) do +NOTE: **Note:** +Messages in the API (`lib/api/` or `app/graphql`) do not need to be externalised. ### HAML files diff --git a/doc/development/i18n/proofreader.md b/doc/development/i18n/proofreader.md index 8c1f62330f4..935a171f34c 100644 --- a/doc/development/i18n/proofreader.md +++ b/doc/development/i18n/proofreader.md @@ -105,6 +105,7 @@ are very appreciative of the work done by translators and proofreaders! - Proofreaders needed. - Turkish - Ali Demirtaş - [GitLab](https://gitlab.com/alidemirtas), [CrowdIn](https://crowdin.com/profile/alidemirtas) + - Rıfat Ünalmış (Rifat Unalmis) - [GitLab](https://gitlab.com/runalmis), [CrowdIn](https://crowdin.com/profile/runalmis) - Ukrainian - Volodymyr Sobotovych - [GitLab](https://gitlab.com/wheleph), [CrowdIn](https://crowdin.com/profile/wheleph) - Andrew Vityuk - [GitLab](https://gitlab.com/3_1_3_u), [CrowdIn](https://crowdin.com/profile/andruwa13) diff --git a/doc/development/import_export.md b/doc/development/import_export.md index ecdfd8a853e..556fa6c7db4 100644 --- a/doc/development/import_export.md +++ b/doc/development/import_export.md @@ -408,4 +408,5 @@ tree └── 4352.json ``` -CAUTION: **Caution:** When updating these fixtures, please ensure you update both `json` files and `tree` folder, as the tests apply to both. +CAUTION: **Caution:** +When updating these fixtures, please ensure you update both `json` files and `tree` folder, as the tests apply to both. diff --git a/doc/development/import_project.md b/doc/development/import_project.md index f222a6533e8..1fa6ea5d405 100644 --- a/doc/development/import_project.md +++ b/doc/development/import_project.md @@ -17,7 +17,8 @@ The first option is to simply [import the Project tarball file via the GitLab UI It should take up to 15 minutes for the project to fully import. You can head to the project's main page for the current status. -NOTE: **Note:** This method ignores all the errors silently (including the ones related to `GITALY_DISABLE_REQUEST_LIMITS`) and is used by GitLab's users. For development and testing, check the other methods below. +NOTE: **Note:** +This method ignores all the errors silently (including the ones related to `GITALY_DISABLE_REQUEST_LIMITS`) and is used by GitLab's users. For development and testing, check the other methods below. ### Importing via the `import-project` script @@ -47,7 +48,9 @@ This method will take longer to import than the other methods and will depend on ### Importing via a Rake task -[`import.rake`](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/tasks/gitlab/import_export/import.rake) was introduced for importing large GitLab project exports. +> The [Rake task](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/tasks/gitlab/import_export/import.rake) was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/20724) in GitLab 12.6, replacing a GitLab.com Ruby script. + +This script was introduced in GitLab 12.6 for importing large GitLab project exports. As part of this script we also disable direct and background upload to avoid situations where a huge archive is being uploaded to GCS (while being inside a transaction, which can cause idle transaction timeouts). @@ -63,9 +66,53 @@ Parameters: | `archive_path` | string | yes | Path to the exported project tarball you want to import | ```shell -bundle exec rake "gitlab:import_export:import[root, root, testingprojectimport, /path/to/file.tar.gz]" +bundle exec rake "gitlab:import_export:import[root, group/subgroup, testingprojectimport, /path/to/file.tar.gz]" +``` + +If you're running Omnibus, run the following Rake task: + +```shell +gitlab-rake "gitlab:import_export:import[root, group/subgroup, testingprojectimport, /path/to/file.tar.gz]" +``` + +#### Troubleshooting + +Check the common errors listed below, what they mean, and how to fix them. + +##### `Exception: undefined method 'name' for nil:NilClass` + +The `username` is not valid. + +##### `Exception: undefined method 'full_path' for nil:NilClass` + +The `namespace_path` does not exist. +For example, one of the groups or subgroups is mistyped or missing +or you've specified the project name in the path. + +The task will only create the project. +If you want to import it to a new group or subgroup then create it first. + +##### `Exception: No such file or directory @ rb_sysopen - (filename)` + +The specified project export file in `archive_path` is missing. + +##### `Name can contain only letters, digits, emojis ...` + +```plaintext +Name can contain only letters, digits, emojis, '_', '.', dash, space. It must start with letter, +digit, emoji or '_'. and Path can contain only letters, digits, '_', '-' and '.'. Cannot start +with '-', end in '.git' or end in '.atom' ``` +The project name specified in `project_path` is not valid for one of the specified reasons. + +Only put the project name in `project_path`. If, for example, you put a path of subgroups in there +it will fail with this error as `/` is not a valid character in a project name. + +##### `Name has already been taken and Path has already been taken` + +A project with that name already exists. + ### Importing via the Rails console The last option is to import a project using a Rails console: diff --git a/doc/development/insert_into_tables_in_batches.md b/doc/development/insert_into_tables_in_batches.md index d8919789808..f65d2478d2e 100644 --- a/doc/development/insert_into_tables_in_batches.md +++ b/doc/development/insert_into_tables_in_batches.md @@ -121,10 +121,10 @@ These callbacks cannot be used with bulk insertions, since they are meant to be every instance that is saved or created. Since these events do not fire when records are inserted in bulk, we currently disallow their use. -The specifics around which callbacks are disallowed are defined in +The specifics around which callbacks are explicitly allowed are defined in [`BulkInsertSafe`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/models/concerns/bulk_insert_safe.rb). -Consult the module source code for details. If your class uses any of the blacklisted -functionality, and you `include BulkInsertSafe`, the application will fail with an error. +Consult the module source code for details. If your class uses callbacks that are not explicitly designated +safe and you `include BulkInsertSafe` the application will fail with an error. ### `BulkInsertSafe` versus `InsertAll` diff --git a/doc/development/integrations/elasticsearch_for_paid_tiers_on_gitlabcom.md b/doc/development/integrations/elasticsearch_for_paid_tiers_on_gitlabcom.md new file mode 100644 index 00000000000..8289be47253 --- /dev/null +++ b/doc/development/integrations/elasticsearch_for_paid_tiers_on_gitlabcom.md @@ -0,0 +1,28 @@ +# Elasticsearch for paid tiers on GitLab.com + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/220246) in GitLab 13.2 +> - It's deployed behind a feature flag, disabled by default. +> - It's disabled on GitLab.com. +> - It's not recommended for use in GitLab self-managed instances. + +This document describes how to enable Elasticsearch with GitLab for all paid tiers on GitLab.com. Once enabled, +all paid tiers will have access to the [Advanced Global Search feature](../../integration/elasticsearch.md) on GitLab.com. + +## Enable or disable Elasticsearch for all paid tiers on GitLab.com + +Since we're still in the process of rolling this out and want to control the timing this is behind a feature flag +which defaults to off. + +To enable it: + +```ruby +# Instance-wide +Feature.enable(:elasticsearch_index_only_paid_groups) +``` + +To disable it: + +```ruby +# Instance-wide +Feature.disable(:elasticsearch_index_only_paid_groups) +``` diff --git a/doc/development/integrations/jira_connect.md b/doc/development/integrations/jira_connect.md index 8e619c3b0a2..374cc976caa 100644 --- a/doc/development/integrations/jira_connect.md +++ b/doc/development/integrations/jira_connect.md @@ -11,7 +11,7 @@ The following are required to install and test the app: For the app to work, Jira Cloud should be able to connect to the GitLab instance through the internet. To easily expose your local development environment, you can use tools like - [serveo](https://medium.com/@osanda.deshan/how-to-forward-my-local-port-to-public-using-serveo-4979f352a3bf) + [serveo](https://medium.com/testautomator/how-to-forward-my-local-port-to-public-using-serveo-4979f352a3bf) or [ngrok](https://ngrok.com). These also take care of SSL for you because Jira requires all connections to the app host to be over SSL. diff --git a/doc/development/integrations/secure.md b/doc/development/integrations/secure.md index 1737daae0e0..22da57400e0 100644 --- a/doc/development/integrations/secure.md +++ b/doc/development/integrations/secure.md @@ -100,13 +100,12 @@ the project repository contains Java source code and the `dependency_scanning` f ```yaml mysec_dependency_scanning: - except: - variables: - - $DEPENDENCY_SCANNING_DISABLED - only: - variables: - - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && - $CI_PROJECT_REPOSITORY_LANGUAGES =~ /\bjava\b/ + rules: + - if: $DEPENDENCY_SCANNING_DISABLED + when: never + - if: $GITLAB_FEATURES =~ /\bdependency_scanning\b/ + exists: + - '**/*.java' ``` Any additional job policy should only be configured by users based on their needs. @@ -232,6 +231,32 @@ to colorize the messages they write to the Unix standard output and standard err We recommend using red to report errors, yellow for warnings, and green for notices. Also, we recommend prefixing error messages with `[ERRO]`, warnings with `[WARN]`, and notices with `[INFO]`. +#### Logging level + +The scanner should filter out a log message if its log level is lower than the +one set in the `SECURE_LOG_LEVEL` variable. For instance, `info` and `warn` +messages should be skipped when `SECURE_LOG_LEVEL` is set to `error`. Accepted +values are as follows, listed from highest to lowest: + +- `fatal` +- `error` +- `warn` +- `info` +- `debug` + +It is recommended to use the `debug` level for verbose logging that could be +useful when debugging. The default value for `SECURE_LOG_LEVEL` should be set +to `info`. + +#### common logutil package + +If you are using [go](https://golang.org/) and +[common](https://gitlab.com/gitlab-org/security-products/analyzers/common), +then it is suggested that you use [logrus](https://github.com/Sirupsen/logrus) +and [common's logutil package](https://gitlab.com/gitlab-org/security-products/analyzers/common/-/tree/master/logutil) +to configure the formatter for [logrus](https://github.com/Sirupsen/logrus). +See the [logutil README.md](https://gitlab.com/gitlab-org/security-products/analyzers/common/-/tree/master/logutil/README.md) + ## Report The report is a JSON document that combines vulnerabilities with possible remediations. @@ -547,3 +572,15 @@ remediation. `fixes[].id` contains a fixed vulnerability's [unique identifier](# The `diff` field is a base64-encoded remediation code diff, compatible with [`git apply`](https://git-scm.com/docs/git-format-patch#_discussion). This field is required. + +## Limitations + +### Container Scanning + +Container Scanning currently has these limitations: + +- Although the Security Dashboard can display scan results from multiple images, if multiple + vulnerabilities have the same fingerprint, only the first instance of that vulnerability is + displayed. We're working on removing this limitation. You can follow our progress on the issue + [Change location fingerprint for Container Scanning](https://gitlab.com/gitlab-org/gitlab/-/issues/215466). +- Different scanners may each report the same vulnerability, resulting in duplicate findings. diff --git a/doc/development/integrations/secure_partner_integration.md b/doc/development/integrations/secure_partner_integration.md index 22e1f8bf769..19a497641f9 100644 --- a/doc/development/integrations/secure_partner_integration.md +++ b/doc/development/integrations/secure_partner_integration.md @@ -11,6 +11,15 @@ with [onboarding as a partner](https://about.gitlab.com/partners/integrate/). The steps below are a high-level view of what needs to be done to complete an integration as well as linking to more detailed resources for how to do so. +## Integration Tiers + +GitLab's security offerings are designed for GitLab Gold and GitLab Ultimate users, and the +[DevSecOps](https://about.gitlab.com/handbook/use-cases/#4-devsecops-shift-left-security) +use case. All the features are in those tiers. This includes the APIs and standard reporting +framework needed to provide a consistent experience for users to easily bring their preferred +security tools into GitLab. We ask that our integration partners focus their work on those license +tiers so that we can provide the most value to our mutual customers. + ## What is the GitLab Developer Workflow? This workflow is how GitLab users interact with our product and expect it to diff --git a/doc/development/licensing.md b/doc/development/licensing.md index f92151e7a37..8cda3d5f361 100644 --- a/doc/development/licensing.md +++ b/doc/development/licensing.md @@ -12,7 +12,7 @@ Some gems may not include their license information in their `gemspec` file, and ### License Finder commands -> Note: License Finder currently uses GitLab misused terms of whitelist and blacklist. As a result, the commands below references those terms. We've created an [issue on their project](https://github.com/pivotal/LicenseFinder/issues/745) to propose that they rename their commands. +> Note: License Finder currently uses GitLab misused terms of `whitelist` and `blacklist`. As a result, the commands below reference those terms. We've created an [issue on their project](https://github.com/pivotal/LicenseFinder/issues/745) to propose that they rename their commands. There are a few basic commands License Finder provides that you'll need in order to manage license detection. diff --git a/doc/development/migration_style_guide.md b/doc/development/migration_style_guide.md index 7d3d9dac174..d1f956f957e 100644 --- a/doc/development/migration_style_guide.md +++ b/doc/development/migration_style_guide.md @@ -88,7 +88,7 @@ body: For example: ```ruby -class MyMigration < ActiveRecord::Migration[4.2] +class MyMigration < ActiveRecord::Migration[6.0] DOWNTIME = true DOWNTIME_REASON = 'This migration requires downtime because ...' @@ -411,7 +411,7 @@ migration. For this to work your migration needs to include the module `Gitlab::Database::MultiThreadedMigration`: ```ruby -class MyMigration < ActiveRecord::Migration[4.2] +class MyMigration < ActiveRecord::Migration[6.0] include Gitlab::Database::MigrationHelpers include Gitlab::Database::MultiThreadedMigration end @@ -421,7 +421,7 @@ You can then use the method `with_multiple_threads` to perform work in separate threads. For example: ```ruby -class MyMigration < ActiveRecord::Migration[4.2] +class MyMigration < ActiveRecord::Migration[6.0] include Gitlab::Database::MigrationHelpers include Gitlab::Database::MultiThreadedMigration @@ -455,7 +455,7 @@ by calling the method `disable_ddl_transaction!` in the body of your migration class like so: ```ruby -class MyMigration < ActiveRecord::Migration[4.2] +class MyMigration < ActiveRecord::Migration[6.0] include Gitlab::Database::MigrationHelpers disable_ddl_transaction! @@ -495,9 +495,11 @@ by calling the method `disable_ddl_transaction!` in the body of your migration class like so: ```ruby -class MyMigration < ActiveRecord::Migration[4.2] +class MyMigration < ActiveRecord::Migration[6.0] include Gitlab::Database::MigrationHelpers + DOWNTIME = false + disable_ddl_transaction! def up @@ -534,7 +536,7 @@ Here's an example where we add a new column with a foreign key constraint. Note it includes `index: true` to create an index for it. ```ruby -class Migration < ActiveRecord::Migration[4.2] +class Migration < ActiveRecord::Migration[6.0] def change add_reference :model, :other_model, index: true, foreign_key: { on_delete: :cascade } @@ -602,7 +604,8 @@ In this particular case, the default value exists and we're just changing the me `request_access_enabled` column, which does not imply a rewrite of all the existing records in the `namespaces` table. Only when creating a new column with a default, all the records are going be rewritten. -NOTE: **Note:** A faster [ALTER TABLE ADD COLUMN with a non-null default](https://www.depesz.com/2018/04/04/waiting-for-postgresql-11-fast-alter-table-add-column-with-a-non-null-default/) +NOTE: **Note:** +A faster [ALTER TABLE ADD COLUMN with a non-null default](https://www.depesz.com/2018/04/04/waiting-for-postgresql-11-fast-alter-table-add-column-with-a-non-null-default/) was introduced on PostgresSQL 11.0, removing the need of rewriting the table when a new column with a default value is added. For the reasons mentioned above, it's safe to use `change_column_default` in a single-transaction migration @@ -808,6 +811,14 @@ class BuildMetadata end ``` +When using a `JSONB` column, use the [JsonSchemaValidator](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/validators/json_schema_validator.rb) to keep control of the data being inserted over time. + +```ruby +class BuildMetadata + validates :config_options, json_schema: { filename: 'build_metadata_config_option' } +end +``` + ## Testing See the [Testing Rails migrations](testing_guide/testing_migrations_guide.md) style guide. @@ -844,16 +855,33 @@ If you need more complex logic, you can define and use models local to a migration. For example: ```ruby -class MyMigration < ActiveRecord::Migration[4.2] +class MyMigration < ActiveRecord::Migration[6.0] class Project < ActiveRecord::Base self.table_name = 'projects' end + + def up + # Reset the column information of all the models that update the database + # to ensure the Active Record's knowledge of the table structure is current + Project.reset_column_information + + # ... ... + end end ``` When doing so be sure to explicitly set the model's table name, so it's not derived from the class name or namespace. +Finally, make sure that `reset_column_information` is run in the `up` method of +the migration for all local Models that update the database. + +The reason for that is that all migration classes are loaded at the beginning +(when `db:migrate` starts), so they can get out of sync with the table schema +they map to in case another migration updates that schema. That makes the data +migration fail when trying to insert or make updates to the underlying table, +as the new columns are reported as `unknown attribute` by `ActiveRecord`. + ### Renaming reserved paths When a new route for projects is introduced, it could conflict with any diff --git a/doc/development/multi_version_compatibility.md b/doc/development/multi_version_compatibility.md index 001517d44ea..aedd5c1ffb7 100644 --- a/doc/development/multi_version_compatibility.md +++ b/doc/development/multi_version_compatibility.md @@ -45,7 +45,7 @@ and set this column to `false`. The old servers were still updating the old colu that updated the new column from the old one. For the new servers though, they were only updating the new column and that same trigger was now working against us and setting it back to the wrong value. -For more information, see this [confidential issue](../user/project/issues/confidential_issues.md) `https://gitlab.com/gitlab-com/gl-infra/infrastructure/-/issues/9176`. +For more information, see [the relevant issue](https://gitlab.com/gitlab-com/gl-infra/infrastructure/-/issues/9176). ### Sidebar wasn't loading for some users diff --git a/doc/development/new_fe_guide/development/accessibility.md b/doc/development/new_fe_guide/development/accessibility.md index f76fd72d4dc..b9ee5c3a549 100644 --- a/doc/development/new_fe_guide/development/accessibility.md +++ b/doc/development/new_fe_guide/development/accessibility.md @@ -43,4 +43,4 @@ In forms we should use the `for` attribute in the label statement: - [Chrome Accessibility Developer Tools](https://github.com/GoogleChrome/accessibility-developer-tools) for testing accessibility - [Audit Rules Page](https://github.com/GoogleChrome/accessibility-developer-tools/wiki/Audit-Rules) for best practices -- [Lighthouse Accessibility Score](https://developers.google.com/web/tools/lighthouse/scoring#a11y) for accessibility audits +- [Lighthouse Accessibility Score](https://web.dev/performance-scoring/) for accessibility audits diff --git a/doc/development/ordering_table_columns.md b/doc/development/ordering_table_columns.md index b68602ea30d..18788d0b86e 100644 --- a/doc/development/ordering_table_columns.md +++ b/doc/development/ordering_table_columns.md @@ -1,5 +1,10 @@ # Ordering Table Columns in PostgreSQL +For GitLab we require that columns of new tables are ordered to use the +least amount of space. An easy way of doing this is to order them based on the +type size in descending order with variable sizes (`text`, `varchar`, arrays, +`json`, `jsonb`, and so on) at the end. + Similar to C structures the space of a table is influenced by the order of columns. This is because the size of columns is aligned depending on the type of the following column. Let's consider an example: @@ -40,10 +45,9 @@ In these examples, the `id` and `user_id` columns are packed together, which means we only need 8 bytes to store _both_ of them. This in turn means each row will require 8 bytes less space. -For GitLab we require that columns of new tables are ordered based to use the -least amount of space. An easy way of doing this is to order them based on the -type size in descending order with variable sizes (`text`, `varchar`, arrays, -`json`, `jsonb`, and so on) at the end. +Note: **NOTE:** +Since Ruby on Rails 5.1, the default data type for IDs is `bigint`, which uses 8 bytes. +We are using `integer` in the examples to showcase a more realistic reordering scenario. ## Type Sizes diff --git a/doc/development/packages.md b/doc/development/packages.md index edf01255e84..ae67af10d88 100644 --- a/doc/development/packages.md +++ b/doc/development/packages.md @@ -68,7 +68,8 @@ The current state of existing package registries availability is: | Go | Yes | No - [open issue](https://gitlab.com/gitlab-org/gitlab/-/issues/213900) | No - [open-issue](https://gitlab.com/gitlab-org/gitlab/-/issues/213902) | | Composer | Yes | Yes | No | -NOTE: **Note:** NPM is currently a hybrid of the instance level and group level. +NOTE: **Note:** +NPM is currently a hybrid of the instance level and group level. It is using the top-level group or namespace as the defining portion of the name (for example, `@my-group-name/my-package-name`). diff --git a/doc/development/performance.md b/doc/development/performance.md index 69ad524675d..16ea1aa27ff 100644 --- a/doc/development/performance.md +++ b/doc/development/performance.md @@ -36,7 +36,6 @@ graphs/dashboards. GitLab provides built-in tools to help improve performance and availability: - [Profiling](profiling.md). - - [Sherlock](profiling.md#sherlock). - [Distributed Tracing](distributed_tracing.md) - [GitLab Performance Monitoring](../administration/monitoring/performance/index.md). - [Request Profiling](../administration/monitoring/performance/request_profiling.md). @@ -108,16 +107,24 @@ In short: ## Profiling By collecting snapshots of process state at regular intervals, profiling allows -you to see where time is spent in a process. The [StackProf](https://github.com/tmm1/stackprof) -gem is included in GitLab's development environment, allowing you to investigate -the behavior of suspect code in detail. +you to see where time is spent in a process. The +[Stackprof](https://github.com/tmm1/stackprof) gem is included in GitLab, +allowing you to profile which code is running on CPU in detail. -It's important to note that profiling an application *alters its performance*, -and will generally be done *in an unrepresentative environment*. In particular, -a method is not necessarily troublesome just because it's executed many times, -or takes a long time to execute. Profiles are tools you can use to better -understand what is happening in an application - using that information wisely -is up to you! +It's important to note that profiling an application *alters its performance*. +Different profiling strategies have different overheads. Stackprof is a sampling +profiler. It will sample stack traces from running threads at a configurable +frequency (e.g. 100hz, that is 100 stacks per second). This type of profiling +has quite a low (albeit non-zero) overhead and is generally considered to be +safe for production. + +### Development + +A profiler can be a very useful tool during development, even if it does run *in +an unrepresentative environment*. In particular, a method is not necessarily +troublesome just because it's executed many times, or takes a long time to +execute. Profiles are tools you can use to better understand what is happening +in an application - using that information wisely is up to you! Keeping that in mind, to create a profile, identify (or create) a spec that exercises the troublesome code path, then run it using the `bin/rspec-stackprof` @@ -166,11 +173,30 @@ dot -Tsvg project_policy_spec.dot > project_policy_spec.svg To load the profile in [kcachegrind](https://kcachegrind.github.io/): ```shell -stackprof tmp/project_policy_spec.dump --callgrind > project_policy_spec.callgrind +stackprof tmp/project_policy_spec.rb.dump --callgrind > project_policy_spec.callgrind kcachegrind project_policy_spec.callgrind # Linux qcachegrind project_policy_spec.callgrind # Mac ``` +For flamegraphs, enable raw collection first. Note that raw +collection can generate a very large file, so increase the `INTERVAL`, or +run on a smaller number of specs for smaller file size: + +```shell +RAW=true bin/rspec-stackprof spec/policies/group_member_policy_spec.rb +``` + +You can then generate, and view the resultant flamegraph. It might take a +while to generate based on the output file size: + +```shell +# Generate +stackprof --flamegraph tmp/group_member_policy_spec.rb.dump > group_member_policy_spec.flame + +# View +stackprof --flamegraph-viewer=group_member_policy_spec.flame +``` + It may be useful to zoom in on a specific method, for example: ```shell @@ -211,11 +237,57 @@ application code, these profiles can be used to investigate slow tests as well. However, for smaller runs (like this example), this means that the cost of setting up the test suite will tend to dominate. -It's also possible to modify the application code in-place to output profiles -whenever a particular code path is triggered without going through the test -suite first. See the -[StackProf documentation](https://github.com/tmm1/stackprof/blob/master/README.md) -for details. +### Production + +Stackprof can also be used to profile production workloads. + +In order to enable production profiling for Ruby processes, you can set the `STACKPROF_ENABLED` environment variable to `true`. + +The following configuration options can be configured: + +- `STACKPROF_ENABLED`: Enables stackprof signal handler on SIGUSR2 signal. + Defaults to `false`. +- `STACKPROF_INTERVAL_US`: Sampling interval in microseconds. Defaults to + `10000` μs (100hz). +- `STACKPROF_FILE_PREFIX`: File path prefix where profiles are stored. Defaults + to `$TMPDIR` (often corresponds to `/tmp`). +- `STACKPROF_TIMEOUT_S`: Profiling timeout in seconds. Profiling will + automatically stop after this time has elapsed. Defaults to `30`. +- `STACKPROF_RAW`: Whether to collect raw samples or only aggregates. Raw + samples are needed to generate flamegraphs, but they do have a higher memory + and disk overhead. Defaults to `true`. + +Once enabled, profiling can be triggered by sending a `SIGUSR2` signal to the +Ruby process. The process will begin sampling stacks. Profiling can be stopped +by sending another `SIGUSR2`. Alternatively, it will automatically stop after +the timeout. + +Once profiling stops, the profile is written out to disk at +`$STACKPROF_FILE_PREFIX/stackprof.$PID.$RAND.profile`. It can then be inspected +further via the `stackprof` command line tool, as described in the previous +section. + +Currently supported profiling targets are: + +- Puma worker +- Sidekiq + +NOTE: **Note:** +The Puma master process is not supported. Neither is Unicorn. +Sending SIGUSR2 to either of those will trigger restarts. In the case of Puma, +take care to only send the signal to Puma workers. + +This can be done via `pkill -USR2 puma:`. The `:` disambiguates between `puma +4.3.3.gitlab.2 ...` (the master process) from `puma: cluster worker 0: ...` (the +worker processes), selecting the latter. + +Production profiles can be especially noisy. It can be helpful to visualize them +as a [flamegraph](https://github.com/brendangregg/FlameGraph). This can be done +via: + +```shell +bundle exec stackprof --stackcollapse /tmp/stackprof.55769.c6c3906452.profile | flamegraph.pl > flamegraph.svg +``` ## RSpec profiling @@ -254,6 +326,13 @@ These results can also be placed into a PostgreSQL database by setting the `RSPEC_PROFILING_POSTGRES_URL` variable. This is used to profile the test suite when running in the CI environment. +We store these results also when running CI jobs on the default branch on +`gitlab.com`. Statistics of these profiling data are [available +online](https://gitlab-org.gitlab.io/rspec_profiling_stats/). For example, +you can find which tests take longest to run or which execute the most +queries. This can be handy for optimizing our tests or identifying performance +issues in our code. + ## Memory profiling One of the reasons of the increased memory footprint could be Ruby memory fragmentation. diff --git a/doc/development/permissions.md b/doc/development/permissions.md index 06a4a03de38..e930345caec 100644 --- a/doc/development/permissions.md +++ b/doc/development/permissions.md @@ -13,9 +13,16 @@ Groups and projects can have the following visibility levels: - internal (`10`) - an entity is visible to logged in users - private (`0`) - an entity is visible only to the approved members of the entity +By default, subgroups can **not** have higher visibility levels. +For example, if you create a new private group, it can not include a public subgroup. + The visibility level of a group can be changed only if all subgroups and -sub-projects have the same or lower visibility level. (e.g., a group can be set -to internal only if all subgroups and projects are internal or private). +sub-projects have the same or lower visibility level. For example, a group can be set +to internal only if all subgroups and projects are internal or private. + +CAUTION: **Warning:** +If you migrate an existing group to a lower visibility level, that action does not migrate subgroups +in the same way. This is a [known issue](https://gitlab.com/gitlab-org/gitlab/-/issues/22406). Visibility levels can be found in the `Gitlab::VisibilityLevel` module. @@ -48,10 +55,10 @@ levels are available (defined in the `Gitlab::Access` module): - Maintainer (`40`) - Owner (`50`) -If a user is the member of both a project and the project parent group, the +If a user is the member of both a project and the project parent group(s), the higher permission is taken into account for the project. -If a user is the member of a project, but not the parent group (or groups), they +If a user is the member of a project, but not the parent group(s), they can still view the groups and their entities (like epics). Project membership (where the group membership is already taken into account) diff --git a/doc/development/pipelines.md b/doc/development/pipelines.md index 05b80cdb4a6..d3623529cd4 100644 --- a/doc/development/pipelines.md +++ b/doc/development/pipelines.md @@ -370,24 +370,28 @@ and are as follows: | MRs | 11 | | `master` (non-scheduled pipelines) | 11 | | 2-hourly scheduled pipelines | 11 | +| `nightly` scheduled pipelines | 11, 12 | #### Long-term plan We follow the [PostgreSQL versions shipped with Omnibus GitLab](https://docs.gitlab.com/omnibus/package-information/postgresql_versions.html): -| PostgreSQL version | 12.10 (April 2020) | 13.0 (May 2020) | 13.1 (June 2020) | 13.2 (July 2020) | 13.3 (August 2020) | 13.4, 13.5 | 13.6 (November 2020) | 14.0 (May 2021?) | -| ------ | ------------------ | --------------- | ---------------- | ---------------- | ------------------ | ------------ | -------------------- | ---------------- | -| PG9.6 | MRs/`master`/`2-hour`/`nightly` | - | - | - | - | - | - | - | -| PG10 | `nightly` | - | - | - | - | - | - | - | -| PG11 | `master`/`2-hour` | MRs/`master`/`2-hour`/`nightly` | MRs/`master`/`2-hour`/`nightly` | MRs/`master`/`2-hour`/`nightly` | MRs/`master`/`2-hour`/`nightly` | MRs/`master`/`2-hour`/`nightly` | `nightly` | - | -| PG12 | - | - | - | - | `master`/`2-hour` | `master`/`2-hour` | MRs/`master`/`2-hour`/`nightly` | `master`/`2-hour` | -| PG13 | - | - | - | - | - | - | - | MRs/`master`/`2-hour`/`nightly` | +| PostgreSQL version | 13.0 (May 2020) | 13.1 (June 2020) | 13.2 (July 2020) | 13.3 (August 2020) | 13.4, 13.5 | 13.6 (November 2020) | 14.0 (May 2021?) | +| ------ | --------------- | ---------------- | ---------------- | ------------------ | ------------ | -------------------- | ---------------- | +| PG11 | MRs/`master`/`2-hour`/`nightly` | MRs/`master`/`2-hour`/`nightly` | MRs/`master`/`2-hour`/`nightly` | MRs/`master`/`2-hour`/`nightly` | MRs/`master`/`2-hour`/`nightly` | `nightly` | - | +| PG12 | - | - | `nightly` | `2-hour`/`nightly` | `2-hour`/`nightly` | MRs/`2-hour`/`nightly` | `2-hour`/`nightly` | +| PG13 | - | - | - | - | - | - | MRs/`2-hour`/`nightly` | ### Test jobs Consult [GitLab tests in the Continuous Integration (CI) context](testing_guide/ci.md) for more information. +We have dedicated jobs for each [testing level](testing_guide/testing_levels.md) and each job runs depending on the +changes made in your merge request. +If you want to force all the RSpec jobs to run regardless of your changes, you can include `RUN ALL RSPEC` in your merge +request title. + ### Review app jobs Consult the [Review Apps](testing_guide/review_apps.md) dedicated page for more information. diff --git a/doc/development/policies.md b/doc/development/policies.md index 62442de825a..8f05948cb41 100644 --- a/doc/development/policies.md +++ b/doc/development/policies.md @@ -158,6 +158,89 @@ end will include all rules from `ProjectPolicy`. The delegated conditions will be evaluated with the correct delegated subject, and will be sorted along with the regular rules in the policy. Note that only the relevant rules for a particular ability will actually be considered. +### Overrides + +We allow policies to opt-out of delegated abilities. + +Delegated policies may define some abilities in a way that is incorrect for the +delegating policy. Take for example a child/parent relationship, where some +abilities can be inferred, and some cannot: + +```ruby +class ParentPolicy < BasePolicy + condition(:speaks_spanish) { @subject.spoken_languages.include?(:es) } + condition(:has_license) { @subject.driving_license.present? } + condition(:enjoys_broccoli) { @subject.enjoyment_of(:broccoli) > 0 } + + rule { speaks_spanish }.enable :read_spanish + rule { has_license }.enable :drive_car + rule { enjoys_broccoli }.enable :eat_broccoli + rule { ~enjoys_broccoli }.prevent :eat_broccoli +end +``` + +Here, if we delegated the child policy to the parent policy, some values would be +incorrect - we might correctly infer that the child can speak their parent's +language, but it would be incorrect to infer that the child can drive or would +eat broccoli just because the parent can and does. + +Some of these things we can deal with - we can forbid driving universally in the +child policy, for example: + +```ruby +class ChildPolicy < BasePolicy + delegate { @subject.parent } + + rule { default }.prevent :drive_car +end +``` + +But the food preferences one is harder - because of the `prevent` call in the +parent policy, if the parent dislikes it, even calling `enable` in the child +will not enable `:eat_broccoli`. + +We could remove the `prevent` call in the parent policy, but that still doesn't +help us, since the rules are different: parents get to eat what they like, and +children eat what they are given, provided they are well behaved. Allowing +delegation would end up with only children whose parents enjoy green vegetables +eating it. But a parent may well give their child broccoli, even if they dislike +it themselves, because it is good for their child. + +The solution it to override the `:eat_broccoli` ability in the child policy: + +```ruby +class ChildPolicy < BasePolicy + delegate { @subject.parent } + + overrides :eat_broccoli + + condition(:good_kid) { @subject.behavior_level >= Child::GOOD } + + rule { good_kid }.enable :eat_broccoli +end +``` + +With this definition, the `ChildPolicy` will _never_ look in the `ParentPolicy` to +satisfy `:eat_broccoli`, but it _will_ use it for any other abilities. The child +policy can then define `:eat_broccoli` in a way that makes sense for `Child` and not +`Parent`. + +### Alternatives to using `overrides` + +Overriding policy delegation is complex, for the same reason delegation is +complex - it involves reasoning about logical inference, and being clear about +semantics. Misuse of `override` has the potential to duplicate code, and +potentially introduce security bugs, allowing things that should be prevented. +For this reason, it should be used only when other approaches are not feasible. + +Other approaches can include for example using different ability names. Choosing +to eat a food and eating foods you are given are semantically distinct, and they +could be named differently (perhaps `chooses_to_eat_broccoli` and +`eats_what_is_given` in this case). It can depend on how polymorphic the call +site is. If you know that we will always check the policy with a `Parent` or a +`Child`, then we can choose the appropriate ability name. If the call site is +polymorphic, then we cannot do that. + ## Specifying Policy Class You can also override the Policy used for a given subject: diff --git a/doc/development/profiling.md b/doc/development/profiling.md index 2cab6750b9b..f7ead94d725 100644 --- a/doc/development/profiling.md +++ b/doc/development/profiling.md @@ -10,7 +10,8 @@ There is a `Gitlab::Profiler.profile` method, and corresponding `bin/profile-url` script, that enable profiling a GET or POST request to a specific URL, either as an anonymous user (the default) or as a specific user. -NOTE: **Note:** The first argument to the profiler is either a full URL +NOTE: **Note:** +The first argument to the profiler is either a full URL (including the instance hostname) or an absolute path, including the leading slash. diff --git a/doc/development/prometheus_metrics.md b/doc/development/prometheus_metrics.md index 004b1884bf0..024da5cc943 100644 --- a/doc/development/prometheus_metrics.md +++ b/doc/development/prometheus_metrics.md @@ -11,16 +11,16 @@ The requirement for adding a new metric is to make each query to have an unique ```yaml - group: Response metrics (NGINX Ingress) metrics: - - title: "Throughput" - y_axis: - name: "Requests / Sec" - format: "number" - precision: 2 - queries: - - id: response_metrics_nginx_ingress_throughput_status_code - query_range: 'sum(rate(nginx_upstream_responses_total{upstream=~"%{kube_namespace}-%{ci_environment_slug}-.*"}[2m])) by (status_code)' - unit: req / sec - label: Status Code + - title: "Throughput" + y_axis: + name: "Requests / Sec" + format: "number" + precision: 2 + queries: + - id: response_metrics_nginx_ingress_throughput_status_code + query_range: 'sum(rate(nginx_upstream_responses_total{upstream=~"%{kube_namespace}-%{ci_environment_slug}-.*"}[2m])) by (status_code)' + unit: req / sec + label: Status Code ``` ### Update existing metrics diff --git a/doc/development/query_recorder.md b/doc/development/query_recorder.md index b28fdb1252b..f7d2c23e28d 100644 --- a/doc/development/query_recorder.md +++ b/doc/development/query_recorder.md @@ -115,6 +115,6 @@ There are multiple ways to find the source of queries. ## See also -- [Bullet](profiling.md#Bullet) For finding `N+1` query problems +- [Bullet](profiling.md#bullet) For finding `N+1` query problems - [Performance guidelines](performance.md) - [Merge request performance guidelines](merge_request_performance_guidelines.md#query-counts) diff --git a/doc/development/rake_tasks.md b/doc/development/rake_tasks.md index 3fa6ba40e6c..fd5dee69fc3 100644 --- a/doc/development/rake_tasks.md +++ b/doc/development/rake_tasks.md @@ -232,6 +232,9 @@ To see the full list of API routes, you can run: bundle exec rake grape:path_helpers ``` +The generated list includes a full list of API endpoints and functional +RESTful API verbs. + For the Rails controllers, run: ```shell diff --git a/doc/development/redis.md b/doc/development/redis.md index 6782ea96448..693b9e1ad0d 100644 --- a/doc/development/redis.md +++ b/doc/development/redis.md @@ -33,6 +33,8 @@ stop being consulted if the project is renamed. If the contents of the key are invalidated by a name change, it is better to include a hook that will expire the entry, instead of relying on the key changing. +### Multi-key commands + We don't use [Redis Cluster](https://redis.io/topics/cluster-tutorial) at the moment, but may wish to in the future: [#118820](https://gitlab.com/gitlab-org/gitlab/-/issues/118820). @@ -41,3 +43,8 @@ operations that require several keys to be held on the same Redis server - for instance, diffing two sets held in Redis - the keys should ensure that by enclosing the changeable parts in curly braces, such as, `project:{1}:set_a` and `project:{1}:set_b`. + +Currently, we validate this in the development and test environments +with the [`RedisClusterValidator`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/instrumentation/redis_cluster_validator.rb), +which is enabled for the `cache` and `shared_state` +[Redis instances](https://docs.gitlab.com/omnibus/settings/redis.html#running-with-multiple-redis-instances).. diff --git a/doc/development/scalability.md b/doc/development/scalability.md index c0c26df88b5..0fb54d89913 100644 --- a/doc/development/scalability.md +++ b/doc/development/scalability.md @@ -115,8 +115,7 @@ that backup, the database can apply the WAL logs in order until the database has reached the target time. On GitLab.com, Consul and Patroni work together to coordinate failovers with -the read replicas. [Omnibus ships with repmgr instead of -Patroni](../administration/postgresql/replication_and_failover.md). +the read replicas. [Omnibus ships with both repmgr and Patroni](../administration/postgresql/replication_and_failover.md). #### Load-balancing diff --git a/doc/development/secure_coding_guidelines.md b/doc/development/secure_coding_guidelines.md index 912b8fbf043..65953620ce6 100644 --- a/doc/development/secure_coding_guidelines.md +++ b/doc/development/secure_coding_guidelines.md @@ -213,7 +213,7 @@ the mitigations for a new feature. #### Feature-specific Mitigations -For situations in which an allowlist or GitLab:HTTP cannot be used, it will be necessary to implement mitigations directly in the feature. It is best to validate the destination IP addresses themselves, not just domain names, as DNS can be controlled by the attacker. Below are a list of mitigations that should be implemented. +For situtions in which an allowlist or GitLab:HTTP cannot be used, it will be necessary to implement mitigations directly in the feature. It is best to validate the destination IP addresses themselves, not just domain names, as DNS can be controlled by the attacker. Below are a list of mitigations that should be implemented. **Important Note:** There are many tricks to bypass common SSRF validations. If feature-specific mitigations are necessary, they should be reviewed by the AppSec team, or a developer who has worked on SSRF mitigations previously. @@ -278,6 +278,7 @@ For any and all input fields, ensure to define expectations on the type/format o - Validate the [input size limits](https://youtu.be/2VFavqfDS6w?t=7582). - Validate the input using an [allowlist approach](https://youtu.be/2VFavqfDS6w?t=7816) to only allow characters through which you are expecting to receive for the field. - Input which fails validation should be **rejected**, and not sanitized. +- When adding redirects or links to a user-controlled URL, ensure that the scheme is HTTP or HTTPS. Allowing other schemes like `javascript://` can lead to XSS and other security issues. Note that denylists should be avoided, as it is near impossible to block all [variations of XSS](https://owasp.org/www-community/xss-filter-evasion-cheatsheet). @@ -292,40 +293,60 @@ Once you've [determined when and where](#setting-expectations) the user submitte ### Additional info -#### Mitigating XSS in Rails +#### XSS mitigation and prevention in Rails + +By default, Rails automatically escapes strings when they are inserted into HTML templates. Avoid the +methods used to keep Rails from escaping strings, especially those related to user-controlled values. +Specifically, the following options are dangerous because they mark strings as trusted and safe: + +| Method | Avoid these options | +|----------------------|-------------------------------| +| HAML templates | `html_safe`, `raw`, `!=` | +| Embedded Ruby (ERB) | `html_safe`, `raw`, `<%== %>` | +In case you want to sanitize user-controlled values against XSS vulnerabilities, you can use +[`ActionView::Helpers::SanitizeHelper`](https://api.rubyonrails.org/classes/ActionView/Helpers/SanitizeHelper.html). +Calling `link_to` and `redirect_to` with user-controlled parameters can also lead to cross-site scripting. + +Do also sanitize and validate URL schemes. + +References: - [XSS Defense in Rails](https://youtu.be/2VFavqfDS6w?t=2442) - [XSS Defense with HAML](https://youtu.be/2VFavqfDS6w?t=2796) - [Validating Untrusted URLs in Ruby](https://youtu.be/2VFavqfDS6w?t=3936) - [RoR Model Validators](https://youtu.be/2VFavqfDS6w?t=7636) +#### XSS mitigation and prevention in JavaScript and Vue + +- When updating the content of an HTML element using JavaScript, mark user-controlled values as `textContent` or `nodeValue` instead of `innerHTML`. +- Avoid using `v-html` with user-controlled data, use [`v-safe-html`](https://gitlab-org.gitlab.io/gitlab-ui/?path=/story/directives-safe-html-directive--default) instead. +- Consider using [`gl-sprintf`](../../ee/development/i18n/externalization.md#interpolation) to interpolate translated strings securely. +- Avoid `__()` with translations that contain user-controlled values. +- When working with `postMessage`, ensure the `origin` of the message is allowlisted. +- Consider using the [Safe Link Directive](https://gitlab-org.gitlab.io/gitlab-ui/?path=/story/directives-safe-link-directive--default) to generate secure hyperlinks by default. + #### GitLab specific libraries for mitigating XSS ##### Vue - [isSafeURL](https://gitlab.com/gitlab-org/gitlab/-/blob/v12.7.5-ee/app/assets/javascripts/lib/utils/url_utility.js#L190-207) +- [GlSprintf](https://gitlab-org.gitlab.io/gitlab-ui/?path=/story/utilities-sprintf--default) #### Content Security Policy - [Content Security Policy](https://www.youtube.com/watch?v=2VFavqfDS6w&t=12991s) - [Use nonce-based Content Security Policy for inline JavaScript](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/65330) -#### Free form input fields - -##### Sanitization - -- [HTML Sanitization](https://youtu.be/2VFavqfDS6w?t=5075) -- [DOMPurify](https://youtu.be/2VFavqfDS6w?t=5381) - -##### `iframe` sandboxes - -- [iframe sandboxing](https://youtu.be/2VFavqfDS6w?t=7043) +#### Free form input field ### Select examples of past XSS issues affecting GitLab -- [Stored XSS in user status](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/55320) +- [Stored XSS in user status](https://gitlab.com/gitlab-org/gitlab-foss/issues/55320) +- [XSS vulnerability on custom project templates form](https://gitlab.com/gitlab-org/gitlab/issues/197302) +- [Stored XSS in branch names](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/55320) +- [Stored XSS in merge request pages](https://gitlab.com/gitlab-org/gitlab/-/issues/35096) -### Developer Training +### Internal Developer Training - [Introduction to XSS](https://www.youtube.com/watch?v=PXR8PTojHmc&t=7785s) - [Reflected XSS](https://youtu.be/2VFavqfDS6w?t=603s) @@ -347,3 +368,29 @@ Once you've [determined when and where](#setting-expectations) the user submitte - [RoR model validators](https://youtu.be/2VFavqfDS6w?t=7636) - [Allowlist input validation](https://youtu.be/2VFavqfDS6w?t=7816) - [Content Security Policy](https://www.youtube.com/watch?v=2VFavqfDS6w&t=12991s) + +## Path Traversal guidelines + +### Description + +Path Traversal vulnerabilities grant attackers access to arbitrary directories and files on the server that is executing an application, including data, code or credentials. + +### Impact + +Path Traversal attacks can lead to multiple critical and high severity issues, like arbitrary file read, remote code execution or information disclosure. + +### When to consider + +When working with user-controlled filenames/paths and filesystem APIs. + +### Mitigation and prevention + +In order to prevent Path Traversal vulnerabilities, user-controlled filenames or paths should be validated before being processed. + +- Comparing user input against an allowlist of allowed values or verifying that it only contains allowed characters. +- After validating the user supplied input, it should be appended to the base directory and the path should be canonicalized using the filesystem API. + +#### GitLab specific validations + +- [`Gitlab::Utils.check_path_traversal`](https://gitlab.com/gitlab-org/security/gitlab/-/blob/master/lib/gitlab/utils.rb#L12-24) can be used to validate user input against Path Traversal vulnerabilities. Remember to add further validation when setting the `allowed_absolute` option to `true`. +- [`file_path` API validator](https://gitlab.com/gitlab-org/security/gitlab/-/blob/master/lib/api/validations/validators/file_path.rb) to validate user input when working with the Grape gem. diff --git a/doc/development/sidekiq_style_guide.md b/doc/development/sidekiq_style_guide.md index 7ae3c9e9de2..2793756ff64 100644 --- a/doc/development/sidekiq_style_guide.md +++ b/doc/development/sidekiq_style_guide.md @@ -175,9 +175,9 @@ Jobs can have an `urgency` attribute set, which can be `:high`, | **Urgency** | **Queue Scheduling Target** | **Execution Latency Requirement** | |--------------|-----------------------------|------------------------------------| -| `:high` | 100 milliseconds | p50 of 1 second, p99 of 10 seconds | -| `:low` | 1 minute | Maximum run time of 1 hour | -| `:throttled` | None | Maximum run time of 1 hour | +| `:high` | 10 seconds | p50 of 1 second, p99 of 10 seconds | +| `:low` | 1 minute | Maximum run time of 5 minutes | +| `:throttled` | None | Maximum run time of 5 minutes | To set a job's urgency, use the `urgency` class method: @@ -225,6 +225,47 @@ work between two different workers, one with `urgency :high` code that executes quickly, and the other with `urgency :low`, which has no execution latency requirements (but also has lower scheduling targets). +### Changing a queue's urgency + +On GitLab.com, we run Sidekiq in several +[shards](https://dashboards.gitlab.net/d/sidekiq-shard-detail/sidekiq-shard-detail), +each of which represents a particular type of workload. + +When changing a queue's urgency, or adding a new queue, we need to take +into account the expected workload on the new shard. Note that, if we're +changing an existing queue, there is also an effect on the old shard, +but that will always be a reduction in work. + +To do this, we want to calculate the expected increase in total execution time +and RPS (throughput) for the new shard. We can get these values from: + +- The [Queue Detail + dashboard](https://dashboards.gitlab.net/d/sidekiq-queue-detail/sidekiq-queue-detail) + has values for the queue itself. For a new queue, we can look for + queues that have similar patterns or are scheduled in similar + circumstances. +- The [Shard Detail + dashboard](https://dashboards.gitlab.net/d/sidekiq-shard-detail/sidekiq-shard-detail) + has Total Execution Time and Throughput (RPS). The Shard Utilization + panel will show if there is currently any excess capacity for this + shard. + +We can then calculate the RPS * average runtime (estimated for new jobs) +for the queue we're changing to see what the relative increase in RPS and +execution time we expect for the new shard: + +```ruby +new_queue_consumption = queue_rps * queue_duration_avg +shard_consumption = shard_rps * shard_duration_avg + +(new_queue_consumption / shard_consumption) * 100 +``` + +If we expect an increase of **less than 5%**, then no further action is needed. + +Otherwise, please ping `@gitlab-org/scalability` on the merge request and ask +for a review. + ## Jobs with External Dependencies Most background jobs in the GitLab application communicate with other GitLab @@ -262,7 +303,8 @@ class ExternalDependencyWorker end ``` -NOTE: **Note:** Note that a job cannot be both high urgency and have +NOTE: **Note:** +Note that a job cannot be both high urgency and have external dependencies. ## CPU-bound and Memory-bound Workers @@ -337,55 +379,10 @@ We use the following approach to determine whether a worker is CPU-bound: - Note that these values should not be used over small sample sizes, but rather over fairly large aggregates. -## Feature Categorization - -Each Sidekiq worker, or one of its ancestor classes, must declare a -`feature_category` attribute. This attribute maps each worker to a feature -category. This is done for error budgeting, alert routing, and team attribution -for Sidekiq workers. - -The declaration uses the `feature_category` class method, as shown below. - -```ruby -class SomeScheduledTaskWorker - include ApplicationWorker - - # Declares that this worker is part of the - # `continuous_integration` feature category - feature_category :continuous_integration - - # ... -end -``` - -The list of value values can be found in the file `config/feature_categories.yml`. -This file is, in turn generated from the [`stages.yml` from the GitLab Company Handbook -source](https://gitlab.com/gitlab-com/www-gitlab-com/blob/master/data/stages.yml). - -### Updating `config/feature_categories.yml` - -Occasionally new features will be added to GitLab stages. When this occurs, you -can automatically update `config/feature_categories.yml` by running -`scripts/update-feature-categories`. This script will fetch and parse -[`stages.yml`](https://gitlab.com/gitlab-com/www-gitlab-com/blob/master/data/stages.yml) -and generate a new version of the file, which needs to be checked into source control. - -### Excluding Sidekiq workers from feature categorization +## Feature category -A few Sidekiq workers, that are used across all features, cannot be mapped to a -single category. These should be declared as such using the `feature_category_not_owned!` - declaration, as shown below: - -```ruby -class SomeCrossCuttingConcernWorker - include ApplicationWorker - - # Declares that this worker does not map to a feature category - feature_category_not_owned! - - # ... -end -``` +All Sidekiq workers must define a known [feature +category](feature_categorization/index.md#sidekiq-workers). ## Job weights @@ -401,6 +398,8 @@ default weight, which is 1. ## Worker context +> - [Introduced](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/9) in GitLab 12.8. + To have some more information about workers in the logs, we add [metadata to the jobs in the form of an `ApplicationContext`](logging.md#logging-context-metadata-through-rails-or-grape-requests). @@ -417,27 +416,27 @@ need to do anything. There are however some instances when there would be no context present when the job is scheduled, or the context that is present is -likely to be incorrect. For these instances we've added rubocop-rules +likely to be incorrect. For these instances, we've added Rubocop rules to draw attention and avoid incorrect metadata in our logs. As with most our cops, there are perfectly valid reasons for disabling them. In this case it could be that the context from the request is correct. Or maybe you've specified a context already in a way that -isn't picked up by the cops. In any case, please leave a code-comment +isn't picked up by the cops. In any case, leave a code comment pointing to which context will be used when disabling the cops. -When you do provide objects to the context, please make sure that the -route for namespaces and projects is pre-loaded. This can be done using +When you do provide objects to the context, make sure that the +route for namespaces and projects is pre-loaded. This can be done by using the `.with_route` scope defined on all `Routable`s. -### Cron-Workers +### Cron workers -The context is automatically cleared for workers in the cronjob-queue -(which `include CronjobQueue`), even when scheduling them from +The context is automatically cleared for workers in the Cronjob queue +(`include CronjobQueue`), even when scheduling them from requests. We do this to avoid incorrect metadata when other jobs are -scheduled from the cron-worker. +scheduled from the cron worker. -Cron-Workers themselves run instance wide, so they aren't scoped to +Cron workers themselves run instance wide, so they aren't scoped to users, namespaces, projects, or other resources that should be added to the context. @@ -449,46 +448,46 @@ somewhere within the worker: 1. Wrap the code that schedules jobs in the `with_context` helper: -```ruby - def perform - deletion_cutoff = Gitlab::CurrentSettings - .deletion_adjourned_period.days.ago.to_date - projects = Project.with_route.with_namespace - .aimed_for_deletion(deletion_cutoff) + ```ruby + def perform + deletion_cutoff = Gitlab::CurrentSettings + .deletion_adjourned_period.days.ago.to_date + projects = Project.with_route.with_namespace + .aimed_for_deletion(deletion_cutoff) - projects.find_each(batch_size: 100).with_index do |project, index| - delay = index * INTERVAL + projects.find_each(batch_size: 100).with_index do |project, index| + delay = index * INTERVAL - with_context(project: project) do - AdjournedProjectDeletionWorker.perform_in(delay, project.id) - end - end - end -``` + with_context(project: project) do + AdjournedProjectDeletionWorker.perform_in(delay, project.id) + end + end + end + ``` 1. Use the a batch scheduling method that provides context: -```ruby - def schedule_projects_in_batch(projects) - ProjectImportScheduleWorker.bulk_perform_async_with_contexts( - projects, - arguments_proc: -> (project) { project.id }, - context_proc: -> (project) { { project: project } } - ) - end -``` - -or when scheduling with delays: - -```ruby - diffs.each_batch(of: BATCH_SIZE) do |diffs, index| - DeleteDiffFilesWorker - .bulk_perform_in_with_contexts(index * 5.minutes, - diffs, - arguments_proc: -> (diff) { diff.id }, - context_proc: -> (diff) { { project: diff.merge_request.target_project } }) - end -``` + ```ruby + def schedule_projects_in_batch(projects) + ProjectImportScheduleWorker.bulk_perform_async_with_contexts( + projects, + arguments_proc: -> (project) { project.id }, + context_proc: -> (project) { { project: project } } + ) + end + ``` + + Or, when scheduling with delays: + + ```ruby + diffs.each_batch(of: BATCH_SIZE) do |diffs, index| + DeleteDiffFilesWorker + .bulk_perform_in_with_contexts(index * 5.minutes, + diffs, + arguments_proc: -> (diff) { diff.id }, + context_proc: -> (diff) { { project: diff.merge_request.target_project } }) + end + ``` ### Jobs scheduled in bulk @@ -512,11 +511,11 @@ For example: Each object from the enumerable in the first argument is yielded into 2 blocks: -The `arguments_proc` which needs to return the list of arguments the -job needs to be scheduled with. +- The `arguments_proc` which needs to return the list of arguments the + job needs to be scheduled with. -The `context_proc` which needs to return a hash with the context -information for the job. +- The `context_proc` which needs to return a hash with the context + information for the job. ## Arguments logging @@ -597,7 +596,6 @@ There are two options for safely adding new arguments to Sidekiq workers: 1. Set up a [multi-step deployment](#multi-step-deployment) in which the new argument is first added to the worker 1. Use a [parameter hash](#parameter-hash) for additional arguments. This is perhaps the most flexible option. -1. Use a parameter hash for additional arguments. This is perhaps the most flexible option. ##### Multi-step deployment diff --git a/doc/development/telemetry/index.md b/doc/development/telemetry/index.md index aee16e4049a..0000e7e9e4f 100644 --- a/doc/development/telemetry/index.md +++ b/doc/development/telemetry/index.md @@ -20,6 +20,7 @@ Telemetry Guide: 1. [Our tracking tools](#our-tracking-tools) 1. [What data can be tracked](#what-data-can-be-tracked) 1. [Telemetry systems overview](#telemetry-systems-overview) + 1. [Snowflake data warehouse](#snowflake-data-warehouse) [Usage Ping Guide](usage_ping.md) @@ -44,9 +45,9 @@ Telemetry Guide: More useful links: - [Telemetry Direction](https://about.gitlab.com/direction/telemetry/) -- [Data Analysis Process](https://about.gitlab.com/handbook/business-ops/data-team/#-data-analysis-process) -- [Data for Product Managers](https://about.gitlab.com/handbook/business-ops/data-team/data-for-product-managers/) -- [Data Infrastructure](https://about.gitlab.com/handbook/business-ops/data-team/data-infrastructure/) +- [Data Analysis Process](https://about.gitlab.com/handbook/business-ops/data-team/#data-analysis-process/) +- [Data for Product Managers](https://about.gitlab.com/handbook/business-ops/data-team/programs/data-for-product-managers/) +- [Data Infrastructure](https://about.gitlab.com/handbook/business-ops/data-team/platform/infrastructure/) ## Our tracking tools @@ -68,7 +69,7 @@ For more details, read the [Usage Ping](usage_ping.md) guide. ### Database import -Database imports are full imports of data into GitLab's data warehouse. For GitLab.com, the PostgreSQL database is loaded into Snowflake data warehouse every 6 hours. For more details, see the [data team handbook](https://about.gitlab.com/handbook/business-ops/data-team/#extract-and-load). +Database imports are full imports of data into GitLab's data warehouse. For GitLab.com, the PostgreSQL database is loaded into Snowflake data warehouse every 6 hours. For more details, see the [data team handbook](https://about.gitlab.com/handbook/business-ops/data-team/platform/#extract-and-load). ### Log system @@ -144,7 +145,7 @@ The systems overview is a simplified diagram showing the interactions between Gi For Telemetry purposes, GitLab Inc has three major components: -1. [Data Infrastructure](https://about.gitlab.com/handbook/business-ops/data-team/data-infrastructure/): This contains everything managed by our data team including Sisense Dashboards for visualization, Snowflake for Data Warehousing, incoming data sources such as PostgreSQL Pipeline and S3 Bucket, and lastly our data collectors [GitLab.com's Snowplow Collector](https://about.gitlab.com/handbook/engineering/infrastructure/library/snowplow/) and GitLab's Versions Application. +1. [Data Infrastructure](https://about.gitlab.com/handbook/business-ops/data-team/platform/infrastructure/): This contains everything managed by our data team including Sisense Dashboards for visualization, Snowflake for Data Warehousing, incoming data sources such as PostgreSQL Pipeline and S3 Bucket, and lastly our data collectors [GitLab.com's Snowplow Collector](https://about.gitlab.com/handbook/engineering/infrastructure/library/snowplow/) and GitLab's Versions Application. 1. GitLab.com: This is the production GitLab application which is made up of a Client and Server. On the Client or browser side, a Snowplow JS Tracker (Frontend) is used to track client-side events. On the Server or application side, a Snowplow Ruby Tracker (Backend) is used to track server-side events. The server also contains Usage Ping which leverages a PostgreSQL database and a Redis in-memory data store to report on usage data. Lastly, the server also contains System Logs which are generated from running the GitLab application. 1. [Monitoring infrastructure](https://about.gitlab.com/handbook/engineering/monitoring/): This is the infrastructure used to ensure GitLab.com is operating smoothly. System Logs are sent from GitLab.com to our monitoring infrastructure and collected by a FluentD collector. From FluentD, logs are either sent to long term Google Cloud Services cold storage via Stackdriver, or, they are sent to our Elastic Cluster via Cloud Pub/Sub which can be explored in real-time using Kibana. @@ -169,3 +170,19 @@ The differences between GitLab.com and self-managed are summarized below: | Self-Managed | **{dotted-circle}**(1) | **{dotted-circle}**(1) | **{check-circle}** | **{dotted-circle}** | **{dotted-circle}** | Note (1): Snowplow JS and Snowplow Ruby are available on self-managed, however, the Snowplow Collector endpoint is set to a self-managed Snowplow Collector which GitLab Inc does not have access to. + +## Snowflake data warehouse + +The Snowflake data warehouse is where we keep all of GitLab Inc's data. + +### Data sources + +There are several data sources available in Snowflake and Sisense each representing a different view of the data along the transformation pipeline. + +| Source | Description | Access | +| ------ | ------ | ------ | +| raw | These tables are the raw data source | Access via Snowflake | +| analytics_staging | These tables have undergone little to no data transformation, meaning they're basically clones of the raw data source | Access via Snowflake or Sisense | +| analytics | These tables have typically undergone more data transformation. They will typically end in `_xf` to represent the fact that they are transformed | Access via Snowflake or Sisense | + +If you are a Product Manager interested in the raw data, you will likely focus on the `analytics` and `analytics_staging` sources. The raw source is limited to the data and infrastructure teams. For more information, please see [Data For Product Managers: What's the difference between analytics_staging and analytics?](https://about.gitlab.com/handbook/business-ops/data-team/programs/data-for-product-managers/#whats-the-difference-between-analytics_staging-and-analytics) diff --git a/doc/development/telemetry/snowplow.md b/doc/development/telemetry/snowplow.md index b7090ee4d20..f03742afe2d 100644 --- a/doc/development/telemetry/snowplow.md +++ b/doc/development/telemetry/snowplow.md @@ -6,15 +6,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Snowplow Guide -This guide provides a details about how Snowplow works. It includes the following sections: - -1. [What is Snowplow](#what-is-snowplow) -1. [Snowplow schema](#snowplow-schema) -1. [Enabling Snowplow](#enabling-snowplow) -1. [Snowplow request flow](#snowplow-request-flow) -1. [Implementing Snowplow JS (Frontend) tracking](#implementing-snowplow-js-frontend-tracking) -1. [Implementing Snowplow Ruby (Backend) tracking](#implementing-snowplow-ruby-backend-tracking) -1. [Developing and testing Snowplow](#developing-and-testing-snowplow) +This guide provides an overview of how Snowplow works, and implementation details. For more information about Telemetry, see: @@ -24,32 +16,31 @@ For more information about Telemetry, see: More useful links: - [Telemetry Direction](https://about.gitlab.com/direction/telemetry/) -- [Data Analysis Process](https://about.gitlab.com/handbook/business-ops/data-team/#-data-analysis-process) -- [Data for Product Managers](https://about.gitlab.com/handbook/business-ops/data-team/data-for-product-managers/) -- [Data Infrastructure](https://about.gitlab.com/handbook/business-ops/data-team/data-infrastructure/) +- [Data Analysis Process](https://about.gitlab.com/handbook/business-ops/data-team/#data-analysis-process/) +- [Data for Product Managers](https://about.gitlab.com/handbook/business-ops/data-team/programs/data-for-product-managers/) +- [Data Infrastructure](https://about.gitlab.com/handbook/business-ops/data-team/platform/infrastructure/) ## What is Snowplow Snowplow is an enterprise-grade marketing and product analytics platform which helps track the way users engage with our website and application. -From [Snowplow's documentation](https://github.com/snowplow/snowplow), Snowplow consists of six loosely-coupled sub-systems: +[Snowplow](https://github.com/snowplow/snowplow) consists of the following loosely-coupled sub-systems: -- **Trackers** fire Snowplow events. Currently Snowplow has 12 trackers, covering web, mobile, desktop, server and IoT -- **Collectors** receive Snowplow events from trackers. Currently we have three different event collectors, sinking events either to Amazon S3, Apache Kafka or Amazon Kinesis -- **Enrich** cleans up the raw Snowplow events, enriches them and puts them into storage. Currently we have a Hadoop-based enrichment process, and a Kinesis- or Kafka-based process -- **Storage** is where the Snowplow events live. Currently we store the Snowplow events in a flat file structure on S3, and in the Redshift and PostgreSQL databases -- **Data modeling** is where event-level data is joined with other data sets and aggregated into smaller data sets, and business logic is applied. This produces a clean set of tables which make it easier to perform analysis on the data. We have data models for Redshift and Looker +- **Trackers** fire Snowplow events. Snowplow has 12 trackers, covering web, mobile, desktop, server, and IoT. +- **Collectors** receive Snowplow events from trackers. We have three different event collectors, synchronizing events either to Amazon S3, Apache Kafka, or Amazon Kinesis. +- **Enrich** cleans up the raw Snowplow events, enriches them and puts them into storage. We have an Hadoop-based enrichment process, and a Kinesis-based or Kafka-based process. +- **Storage** is where the Snowplow events live. We store the Snowplow events in a flat file structure on S3, and in the Redshift and PostgreSQL databases. +- **Data modeling** is where event-level data is joined with other data sets and aggregated into smaller data sets, and business logic is applied. This produces a clean set of tables which make it easier to perform analysis on the data. We have data models for Redshift and Looker. - **Analytics** are performed on the Snowplow events or on the aggregate tables.  ->  ## Snowplow schema -We currently have many definitions of Snowplow's schema. We have an active issue to [standardize this schema](https://gitlab.com/gitlab-org/gitlab/-/issues/207930) including the following definitions: +We have many definitions of Snowplow's schema. We have an active issue to [standardize this schema](https://gitlab.com/gitlab-org/gitlab/-/issues/207930) including the following definitions: - Frontend and backend taxonomy as listed below -- [Feature instrumentation taxonomy](https://about.gitlab.com/handbook/product/feature-instrumentation/#taxonomy) +- [Feature instrumentation taxonomy](https://about.gitlab.com/handbook/product/product-processes/#taxonomy) - [Self describing events](https://github.com/snowplow/snowplow/wiki/Custom-events#self-describing-events) - [Iglu schema](https://gitlab.com/gitlab-org/iglu/) - [Snowplow authored events](https://github.com/snowplow/snowplow/wiki/Snowplow-authored-events) @@ -58,8 +49,8 @@ We currently have many definitions of Snowplow's schema. We have an active issue Tracking can be enabled at: -- The instance level, which will enable tracking on both the frontend and backend layers. -- User level, though user tracking can be disabled on a per-user basis. GitLab tracking respects the [Do Not Track](https://www.eff.org/issues/do-not-track) standard, so any user who has enabled the Do Not Track option in their browser will also not be tracked from a user level. +- The instance level, which enables tracking on both the frontend and backend layers. +- User level, though user tracking can be disabled on a per-user basis. GitLab tracking respects the [Do Not Track](https://www.eff.org/issues/do-not-track) standard, so any user who has enabled the Do Not Track option in their browser is not tracked at a user level. We utilize Snowplow for the majority of our tracking strategy and it is enabled on GitLab.com. On a self-managed instance, Snowplow can be enabled by navigating to: @@ -69,14 +60,20 @@ We utilize Snowplow for the majority of our tracking strategy and it is enabled The following configuration is required: | Name | Value | -| ------------- | ------------------------- | +|---------------|---------------------------| | Collector | `snowplow.trx.gitlab.net` | | Site ID | `gitlab` | | Cookie domain | `.gitlab.com` | ## Snowplow request flow -The following example shows a basic request/response flow between a Snowplow JS / Ruby Trackers on GitLab.com, [the GitLab.com Snowplow Collector](https://about.gitlab.com/handbook/engineering/infrastructure/library/snowplow/), GitLab's S3 Bucket, GitLab's Snowflake Data Warehouse, and Sisense.: +The following example shows a basic request/response flow between the following components: + +- Snowplow JS / Ruby Trackers on GitLab.com +- [GitLab.com Snowplow Collector](https://gitlab.com/gitlab-com/gl-infra/readiness/-/blob/master/library/snowplow/index.md) +- GitLab's S3 Bucket +- GitLab's Snowflake Data Warehouse +- Sisense: ```mermaid sequenceDiagram @@ -101,17 +98,17 @@ sequenceDiagram ## Implementing Snowplow JS (Frontend) tracking -GitLab provides `Tracking`, an interface that wraps the [Snowplow JavaScript Tracker](https://github.com/snowplow/snowplow/wiki/javascript-tracker) for tracking custom events. There are a few ways to utilize tracking, but each generally requires at minimum, a `category` and an `action`. Additional data can be provided that adheres to our [Feature instrumentation taxonomy](https://about.gitlab.com/handbook/product/feature-instrumentation/#taxonomy). +GitLab provides `Tracking`, an interface that wraps the [Snowplow JavaScript Tracker](https://github.com/snowplow/snowplow/wiki/javascript-tracker) for tracking custom events. There are a few ways to utilize tracking, but each generally requires at minimum, a `category` and an `action`. Additional data can be provided that adheres to our [Feature instrumentation taxonomy](https://about.gitlab.com/handbook/product/product-processes/#taxonomy). -| field | type | default value | description | -|:-----------|:-------|:---------------------------|:------------| -| `category` | string | document.body.dataset.page | Page or subsection of a page that events are being captured within. | +| field | type | default value | description | +|:-----------|:-------|:---------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `category` | string | document.body.dataset.page | Page or subsection of a page that events are being captured within. | | `action` | string | 'generic' | Action the user is taking. Clicks should be `click` and activations should be `activate`, so for example, focusing a form field would be `activate_form_input`, and clicking a button would be `click_button`. | -| `data` | object | {} | Additional data such as `label`, `property`, `value`, and `context` as described [in our Feature Instrumentation taxonomy](https://about.gitlab.com/handbook/product/feature-instrumentation/#taxonomy). | +| `data` | object | {} | Additional data such as `label`, `property`, `value`, and `context` as described [in our Feature Instrumentation taxonomy](https://about.gitlab.com/handbook/product/product-processes/#taxonomy). | ### Tracking in HAML (or Vue Templates) -When working within HAML (or Vue templates) we can add `data-track-*` attributes to elements of interest. All elements that have a `data-track-event` attribute will automatically have event tracking bound on clicks. +When working within HAML (or Vue templates) we can add `data-track-*` attributes to elements of interest. All elements that have a `data-track-event` attribute automatically have event tracking bound on clicks. Below is an example of `data-track-*` attributes assigned to a button: @@ -127,17 +124,17 @@ Below is an example of `data-track-*` attributes assigned to a button: /> ``` -Event listeners are bound at the document level to handle click events on or within elements with these data attributes. This allows for them to be properly handled on re-rendering and changes to the DOM, but it's important to know that because of the way these events are bound, click events shouldn't be stopped from propagating up the DOM tree. If for any reason click events are being stopped from propagating, you'll need to implement your own listeners and follow the instructions in [Tracking in raw JavaScript](#tracking-in-raw-javascript). +Event listeners are bound at the document level to handle click events on or within elements with these data attributes. This allows them to be properly handled on re-rendering and changes to the DOM. Note that because of the way these events are bound, click events should not be stopped from propagating up the DOM tree. If for any reason click events are being stopped from propagating, you need to implement your own listeners and follow the instructions in [Tracking in raw JavaScript](#tracking-in-raw-javascript). Below is a list of supported `data-track-*` attributes: | attribute | required | description | |:----------------------|:---------|:------------| | `data-track-event` | true | Action the user is taking. Clicks must be prepended with `click` and activations must be prepended with `activate`. For example, focusing a form field would be `activate_form_input` and clicking a button would be `click_button`. | -| `data-track-label` | false | The `label` as described [in our Feature Instrumentation taxonomy](https://about.gitlab.com/handbook/product/feature-instrumentation/#taxonomy). | -| `data-track-property` | false | The `property` as described [in our Feature Instrumentation taxonomy](https://about.gitlab.com/handbook/product/feature-instrumentation/#taxonomy). | -| `data-track-value` | false | The `value` as described [in our Feature Instrumentation taxonomy](https://about.gitlab.com/handbook/product/feature-instrumentation/#taxonomy). If omitted, this will be the element's `value` property or an empty string. For checkboxes, the default value will be the element's checked attribute or `false` when unchecked. | -| `data-track-context` | false | The `context` as described [in our Feature Instrumentation taxonomy](https://about.gitlab.com/handbook/product/feature-instrumentation/#taxonomy). | +| `data-track-label` | false | The `label` as described [in our Feature Instrumentation taxonomy](https://about.gitlab.com/handbook/product/product-processes/#taxonomy). | +| `data-track-property` | false | The `property` as described [in our Feature Instrumentation taxonomy](https://about.gitlab.com/handbook/product/product-processes/#taxonomy). | +| `data-track-value` | false | The `value` as described [in our Feature Instrumentation taxonomy](https://about.gitlab.com/handbook/product/product-processes/#taxonomy). If omitted, this is the element's `value` property or an empty string. For checkboxes, the default value is the element's checked attribute or `false` when unchecked. | +| `data-track-context` | false | The `context` as described [in our Feature Instrumentation taxonomy](https://about.gitlab.com/handbook/product/product-processes/#taxonomy). | ### Tracking within Vue components @@ -148,9 +145,9 @@ import Tracking from '~/tracking'; const trackingMixin = Tracking.mixin({ label: 'right_sidebar' }); ``` -You can provide default options that will be passed along whenever an event is tracked from within your component. For instance, if all events within a component should be tracked with a given `label`, you can provide one at this time. Available defaults are `category`, `label`, `property`, and `value`. If no category is specified, `document.body.dataset.page` is used as the default. +You can provide default options that are passed along whenever an event is tracked from within your component. For instance, if all events within a component should be tracked with a given `label`, you can provide one at this time. Available defaults are `category`, `label`, `property`, and `value`. If no category is specified, `document.body.dataset.page` is used as the default. -You can then use the mixin normally in your component with the `mixin`, Vue declaration. The mixin also provides the ability to specify tracking options in `data` or `computed`. These will override any defaults and allows the values to be dynamic from props, or based on state. +You can then use the mixin normally in your component with the `mixin` Vue declaration. The mixin also provides the ability to specify tracking options in `data` or `computed`. These override any defaults and allow the values to be dynamic from props, or based on state. ```javascript export default { @@ -240,7 +237,6 @@ describe('MyTracking', () => { }); }); }); - ``` In obsolete Karma tests it's used as below: @@ -278,13 +274,13 @@ GitLab provides `Gitlab::Tracking`, an interface that wraps the [Snowplow Ruby T Custom event tracking and instrumentation can be added by directly calling the `GitLab::Tracking.event` class method, which accepts the following arguments: -| argument | type | default value | description | -|:-----------|:-------|:---------------------------|:------------| -| `category` | string | 'application' | Area or aspect of the application. This could be `HealthCheckController` or `Lfs::FileTransformer` for instance. | -| `action` | string | 'generic' | The action being taken, which can be anything from a controller action like `create` to something like an Active Record callback. | -| `data` | object | {} | Additional data such as `label`, `property`, `value`, and `context` as described [in our Feature Instrumentation taxonomy](https://about.gitlab.com/handbook/product/feature-instrumentation/#taxonomy). These will be set as empty strings if you don't provide them. | +| argument | type | default value | description | +|:-----------|:-------|:--------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `category` | string | 'application' | Area or aspect of the application. This could be `HealthCheckController` or `Lfs::FileTransformer` for instance. | +| `action` | string | 'generic' | The action being taken, which can be anything from a controller action like `create` to something like an Active Record callback. | +| `data` | object | {} | Additional data such as `label`, `property`, `value`, and `context` as described [in our Feature Instrumentation taxonomy](https://about.gitlab.com/handbook/product/feature-instrumentation/#taxonomy). These are set as empty strings if you don't provide them. | -Tracking can be viewed as either tracking user behavior, or can be utilized for instrumentation to monitor and visual performance over time in an area or aspect of code. +Tracking can be viewed as either tracking user behavior, or can be utilized for instrumentation to monitor and visualize performance over time in an area or aspect of code. For example: @@ -309,27 +305,27 @@ We use the [AsyncEmitter](https://github.com/snowplow/snowplow/wiki/Ruby-Tracker There are several tools for developing and testing Snowplow Event -| Testing Tool | Frontend Tracking | Backend Tracking | Local Development Environment | Production Environment | -| ------ | ------ | ------ | ------ | ------ | -| Snowplow Analytics Debugger Chrome Extension | ✅ | ❌ | ✅ | ✅ | -| Snowplow Inspector Chrome Extension | ✅ | ❌ | ✅ | ✅ | -| Snowplow Micro | ✅ | ✅ | ✅ | ❌ | -| Snowplow Mini | ✅ | ✅ | ❌ | ✅ | +| Testing Tool | Frontend Tracking | Backend Tracking | Local Development Environment | Production Environment | +|----------------------------------------------|--------------------|---------------------|-------------------------------|------------------------| +| Snowplow Analytics Debugger Chrome Extension | **{check-circle}** | **{dotted-circle}** | **{check-circle}** | **{check-circle}** | +| Snowplow Inspector Chrome Extension | **{check-circle}** | **{dotted-circle}** | **{check-circle}** | **{check-circle}** | +| Snowplow Micro | **{check-circle}** | **{check-circle}** | **{check-circle}** | **{dotted-circle}** | +| Snowplow Mini | **{check-circle}** | **{check-circle}** | **{dotted-circle}** | **{check-circle}** | ### Snowplow Analytics Debugger Chrome Extension Snowplow Analytics Debugger is a browser extension for testing frontend events. This works on production, staging and local development environments. -1. Install [Snowplow Analytics Debugger](https://chrome.google.com/webstore/detail/snowplow-analytics-debugg/jbnlcgeengmijcghameodeaenefieedm) chrome browser extension -1. Open Chrome DevTools to the Snowplow Analytics Debugger tab -1. Learn more at [Igloo Analytics](https://www.iglooanalytics.com/blog/snowplow-analytics-debugger-chrome-extension.html) +1. Install the [Snowplow Analytics Debugger](https://chrome.google.com/webstore/detail/snowplow-analytics-debugg/jbnlcgeengmijcghameodeaenefieedm) Chrome browser extension. +1. Open Chrome DevTools to the Snowplow Analytics Debugger tab. +1. Learn more at [Igloo Analytics](https://www.iglooanalytics.com/blog/snowplow-analytics-debugger-chrome-extension.html). ### Snowplow Inspector Chrome Extension Snowplow Inspector Chrome Extension is a browser extension for testing frontend events. This works on production, staging and local development environments. -1. Install [Snowplow Inspector](https://chrome.google.com/webstore/detail/snowplow-inspector/maplkdomeamdlngconidoefjpogkmljm?hl=en) -1. Open the chrome extension by pressing the Snowplow Inspector icon beside the address bar +1. Install [Snowplow Inspector](https://chrome.google.com/webstore/detail/snowplow-inspector/maplkdomeamdlngconidoefjpogkmljm?hl=en). +1. Open the Chrome extension by pressing the Snowplow Inspector icon beside the address bar. 1. Click around on a webpage with Snowplow and you should see JavaScript events firing in the inspector window. ### Snowplow Micro @@ -342,53 +338,59 @@ Snowplow Micro is a Docker-based solution for testing frontend and backend event - Look at the [Snowplow Micro repository](https://github.com/snowplow-incubator/snowplow-micro) - Watch our [installation guide recording](https://www.youtube.com/watch?v=OX46fo_A0Ag) -1. Install [Snowplow Micro](https://github.com/snowplow-incubator/snowplow-micro) +1. Install [Snowplow Micro](https://github.com/snowplow-incubator/snowplow-micro): -``` bash -docker run --mount type=bind,source=$(pwd)/example,destination=/config -p 9090:9090 snowplow/snowplow-micro:latest --collector-config /config/micro.conf --iglu /config/iglu.json -``` + ```shell + docker run --mount type=bind,source=$(pwd)/example,destination=/config -p 9090:9090 snowplow/snowplow-micro:latest --collector-config /config/micro.conf --iglu /config/iglu.json + ``` -1. Install snowplow micro by cloning the settings in [this project](https://gitlab.com/a_akgun/snowplow-micro). +1. Install snowplow micro by cloning the settings in [this project](https://gitlab.com/a_akgun/snowplow-micro): - ``` bash - git clone git@gitlab.com:a_akgun/snowplow-micro.git - ./snowplow-micro.sh - ``` + ```shell + git clone git@gitlab.com:a_akgun/snowplow-micro.git + ./snowplow-micro.sh + ``` -1. Update port in SQL (needed to set 9090) +1. Update port in SQL to set `9090`: - ``` bash - gdk psql -d gitlabhq_development - update application_settings set snowplow_collector_hostname='localhost:9090', snowplow_enabled=true, snowplow_cookie_domain='.gitlab.com'; - ``` + ```shell + gdk psql -d gitlabhq_development + update application_settings set snowplow_collector_hostname='localhost:9090', snowplow_enabled=true, snowplow_cookie_domain='.gitlab.com'; + ``` 1. Update `app/assets/javascripts/tracking.js` to [remove this line](https://gitlab.com/snippets/1918635): - ``` javascript - forceSecureTracker: true - ``` + ```javascript + forceSecureTracker: true + ``` 1. Update `lib/gitlab/tracking.rb` to [add these lines](https://gitlab.com/snippets/1918635): - ``` ruby - protocol: 'http', - port: 9090, - ``` + ```ruby + protocol: 'http', + port: 9090, + ``` 1. Update `lib/gitlab/tracking.rb` to [change async emitter from https to http](https://gitlab.com/snippets/1918635): - ``` ruby - SnowplowTracker::AsyncEmitter.new(Gitlab::CurrentSettings.snowplow_collector_hostname, protocol: 'http'), - ``` + ```ruby + SnowplowTracker::AsyncEmitter.new(Gitlab::CurrentSettings.snowplow_collector_hostname, protocol: 'http'), + ``` 1. Enable Snowplow in the admin area, Settings::Integrations::Snowplow to point to: - `http://localhost:3000/admin/application_settings/integrations#js-snowplow-settings` -1. `gdk restart` -1. Send a test Snowplow event from the Rails console + `http://localhost:3000/admin/application_settings/integrations#js-snowplow-settings`. + +1. Restart GDK: + + ```shell + `gdk restart` + ``` + +1. Send a test Snowplow event from the Rails console: - ``` ruby - Gitlab::Tracking.self_describing_event('iglu:com.gitlab/pageview_context/jsonschema/1-0-0', { page_type: ‘MY_TYPE' }, context: nil ) - ``` + ```ruby + Gitlab::Tracking.self_describing_event('iglu:com.gitlab/pageview_context/jsonschema/1-0-0', { page_type: ‘MY_TYPE' }, context: nil ) + ``` ### Snowplow Mini @@ -396,4 +398,4 @@ docker run --mount type=bind,source=$(pwd)/example,destination=/config -p 9090:9 Snowplow Mini can be used for testing frontend and backend events on a production, staging and local development environment. -For GitLab.com, we are currently setting up a [QA and Testing environment](https://gitlab.com/gitlab-org/telemetry/-/issues/266) using Snowplow Mini. +For GitLab.com, we're setting up a [QA and Testing environment](https://gitlab.com/gitlab-org/telemetry/-/issues/266) using Snowplow Mini. diff --git a/doc/development/telemetry/usage_ping.md b/doc/development/telemetry/usage_ping.md index 0f438e02772..5e78e2c5f25 100644 --- a/doc/development/telemetry/usage_ping.md +++ b/doc/development/telemetry/usage_ping.md @@ -21,9 +21,9 @@ For more information about Telemetry, see: More useful links: - [Telemetry Direction](https://about.gitlab.com/direction/telemetry/) -- [Data Analysis Process](https://about.gitlab.com/handbook/business-ops/data-team/#-data-analysis-process) -- [Data for Product Managers](https://about.gitlab.com/handbook/business-ops/data-team/data-for-product-managers/) -- [Data Infrastructure](https://about.gitlab.com/handbook/business-ops/data-team/data-infrastructure/) +- [Data Analysis Process](https://about.gitlab.com/handbook/business-ops/data-team/#data-analysis-process/) +- [Data for Product Managers](https://about.gitlab.com/handbook/business-ops/data-team/programs/data-for-product-managers/) +- [Data Infrastructure](https://about.gitlab.com/handbook/business-ops/data-team/platform/infrastructure/) ## What is Usage Ping? @@ -299,7 +299,7 @@ Paste the SQL query into `#database-lab` to see how the query performs at scale. - `#database-lab` is a Slack channel which uses a production-sized environment to test your queries. - GitLab.com’s production database has a 15 second timeout. -- For each query we require an execution time of under 1 second due to cold caches which can 10x this time. +- Any single query must stay below 1 second execution time with cold caches. - Add a specialized index on columns involved to reduce the execution time. In order to have an understanding of the query's execution we add in the MR description the following information: @@ -326,7 +326,17 @@ When adding, changing, or updating metrics, please update the [Usage Statistics Check if new metrics need to be added to the Versions Application. See `usage_data` [schema](https://gitlab.com/gitlab-services/version-gitlab-com/-/blob/master/db/schema.rb#L147) and usage data [parameters accepted](https://gitlab.com/gitlab-services/version-gitlab-com/-/blob/master/app/services/usage_ping.rb). Any metrics added under the `counts` key are saved in the `counts` column. -### 6. Ask for a Telemetry Review +For further details, see the [Process to add additional instrumentation to the Usage Ping](https://about.gitlab.com/handbook/product/product-processes/#process-to-add-additional-instrumentation-to-the-usage-ping). + +### 6. Add the feature label + +Add the `feature` label to the Merge Request for new Usage Ping metrics. These are user-facing changes and are part of expanding the Usage Ping feature. + +### 7. Add a changelog file + +Ensure you comply with the [Changelog entries guide](../changelog.md). + +### 8. Ask for a Telemetry Review On GitLab.com, we have DangerBot setup to monitor Telemetry related files and DangerBot will recommend a Telemetry review. Mention `@gitlab-org/growth/telemetry/engineers` in your MR for a review. @@ -378,275 +388,355 @@ appear to be associated to any of the services running, since they all appear to ## Usage Statistics definitions -| Statistic | Section | Stage | Tier | Description | -|:--------------------------------------------------------|:-----------------------------------|:------------|:---------------|:--------------------------------------------------| -| `uuid` | | | | | -| `hostname` | | | | | -| `version` | | | | | -| `installation_type` | | | | | -| `active_user_count` | | | | | -| `recorded_at` | | | | | -| `edition` | | | | | -| `license_md5` | | | | | -| `license_id` | | | | | -| `historical_max_users` | | | | | -| `Name` | `licensee` | | | | -| `Email` | `licensee` | | | | -| `Company` | `licensee` | | | | -| `license_user_count` | | | | | -| `license_starts_at` | | | | | -| `license_expires_at` | | | | | -| `license_plan` | | | | | -| `license_trial` | | | | | -| `assignee_lists` | `counts` | | | | -| `boards` | `counts` | | | | -| `ci_builds` | `counts` | `verify` | | Unique builds in project | -| `ci_internal_pipelines` | `counts` | `verify` | | Total pipelines in GitLab repositories | -| `ci_external_pipelines` | `counts` | `verify` | | Total pipelines in external repositories | -| `ci_pipeline_config_auto_devops` | `counts` | `verify` | | Total pipelines from an Auto DevOps template | -| `ci_pipeline_config_repository` | `counts` | `verify` | | Total Pipelines from templates in repository | -| `ci_runners` | `counts` | `verify` | | Total configured Runners in project | -| `ci_triggers` | `counts` | `verify` | | Total configured Triggers in project | -| `ci_pipeline_schedules` | `counts` | `verify` | | Pipeline schedules in GitLab | -| `auto_devops_enabled` | `counts` |`configure` | | Projects with Auto DevOps template enabled | -| `auto_devops_disabled` | `counts` |`configure` | | Projects with Auto DevOps template disabled | -| `deploy_keys` | `counts` | | | | -| `deployments` | `counts` |`release` | | Total deployments | -| `dast_jobs` | `counts` | | | | -| `successful_deployments` | `counts` |`release` | | Total successful deployments | -| `failed_deployments` | `counts` |`release` | | Total failed deployments | -| `environments` | `counts` |`release` | | Total available and stopped environments | -| `clusters` | `counts` |`configure` | | Total GitLab Managed clusters both enabled and disabled | -| `clusters_enabled` | `counts` |`configure` | | Total GitLab Managed clusters currently enabled | -| `project_clusters_enabled` | `counts` |`configure` | | Total GitLab Managed clusters attached to projects| -| `group_clusters_enabled` | `counts` |`configure` | | Total GitLab Managed clusters attached to groups | -| `instance_clusters_enabled` | `counts` |`configure` | | Total GitLab Managed clusters attached to the instance | -| `clusters_disabled` | `counts` |`configure` | | Total GitLab Managed disabled clusters | -| `project_clusters_disabled` | `counts` |`configure` | | Total GitLab Managed disabled clusters previously attached to projects | -| `group_clusters_disabled` | `counts` |`configure` | | Total GitLab Managed disabled clusters previously attached to groups | -| `instance_clusters_disabled` | `counts` |`configure` | | Total GitLab Managed disabled clusters previously attached to the instance | -| `clusters_platforms_eks` | `counts` |`configure` | | Total GitLab Managed clusters provisioned with GitLab on AWS EKS | -| `clusters_platforms_gke` | `counts` |`configure` | | Total GitLab Managed clusters provisioned with GitLab on GCE GKE | -| `clusters_platforms_user` | `counts` |`configure` | | Total GitLab Managed clusters that are user provisioned | -| `clusters_applications_helm` | `counts` |`configure` | | Total GitLab Managed clusters with Helm enabled | -| `clusters_applications_ingress` | `counts` |`configure` | | Total GitLab Managed clusters with Ingress enabled | -| `clusters_applications_cert_managers` | `counts` |`configure` | | Total GitLab Managed clusters with Cert Manager enabled | -| `clusters_applications_crossplane` | `counts` |`configure` | | Total GitLab Managed clusters with Crossplane enabled | -| `clusters_applications_prometheus` | `counts` |`configure` | | Total GitLab Managed clusters with Prometheus enabled | -| `clusters_applications_runner` | `counts` |`configure` | | Total GitLab Managed clusters with Runner enabled | -| `clusters_applications_knative` | `counts` |`configure` | | Total GitLab Managed clusters with Knative enabled | -| `clusters_applications_elastic_stack` | `counts` |`configure` | | Total GitLab Managed clusters with Elastic Stack enabled | -| `clusters_management_project` | `counts` |`configure` | | Total GitLab Managed clusters with defined cluster management project | -| `in_review_folder` | `counts` | | | | -| `grafana_integrated_projects` | `counts` | | | | -| `groups` | `counts` | | | | -| `issues` | `counts` | | | | -| `issues_created_from_gitlab_error_tracking_ui` | `counts` | `monitor` | | | -| `issues_with_associated_zoom_link` | `counts` | `monitor` | | | -| `issues_using_zoom_quick_actions` | `counts` | `monitor` | | | -| `issues_with_embedded_grafana_charts_approx` | `counts` | `monitor` | | | -| `issues_with_health_status` | `counts` | | | | -| `keys` | `counts` | | | | -| `label_lists` | `counts` | | | | -| `lfs_objects` | `counts` | | | | -| `milestone_lists` | `counts` | | | | -| `milestones` | `counts` | | | | -| `pages_domains` | `counts` |`release` | | Total GitLab Pages domains | -| `pool_repositories` | `counts` | | | | -| `projects` | `counts` | | | | -| `projects_imported_from_github` | `counts` | | | | -| `projects_with_repositories_enabled` | `counts` | | | | -| `projects_with_error_tracking_enabled` | `counts` | `monitor` | | | -| `protected_branches` | `counts` | | | | -| `releases` | `counts` |`release` | | Unique release tags | -| `remote_mirrors` | `counts` | | | | -| `requirements_created` | `counts` | | | | -| `snippets` | `counts` | | | | -| `suggestions` | `counts` | | | | -| `todos` | `counts` | | | | -| `uploads` | `counts` | | | | -| `web_hooks` | `counts` | | | | -| `projects_alerts_active` | `counts` | | | | -| `projects_asana_active` | `counts` | | | | -| `projects_assembla_active` | `counts` | | | | -| `projects_bamboo_active` | `counts` | | | | -| `projects_bugzilla_active` | `counts` | | | | -| `projects_buildkite_active` | `counts` | | | | -| `projects_campfire_active` | `counts` | | | | -| `projects_custom_issue_tracker_active` | `counts` | | | | -| `projects_discord_active` | `counts` | | | | -| `projects_drone_ci_active` | `counts` | | | | -| `projects_emails_on_push_active` | `counts` | | | | -| `projects_external_wiki_active` | `counts` | | | -| `projects_flowdock_active` | `counts` | | | | -| `projects_github_active` | `counts` | | | | -| `projects_hangouts_chat_active` | `counts` | | | | -| `projects_hipchat_active` | `counts` | | | | -| `projects_irker_active` | `counts` | | | | -| `projects_jenkins_active` | `counts` | | | | -| `projects_jira_active` | `counts` | | | | -| `projects_mattermost_active` | `counts` | | | | -| `projects_mattermost_slash_commands_active` | `counts` | | | | -| `projects_microsoft_teams_active` | `counts` | | | | -| `projects_packagist_active` | `counts` | | | | -| `projects_pipelines_email_active` | `counts` | | | | -| `projects_pivotaltracker_active` | `counts` | | | | -| `projects_prometheus_active` | `counts` | | | | -| `projects_pushover_active` | `counts` | | | | -| `projects_redmine_active` | `counts` | | | | -| `projects_slack_active` | `counts` | | | | -| `projects_slack_slash_commands_active` | `counts` | | | | -| `projects_teamcity_active` | `counts` | | | | -| `projects_unify_circuit_active` | `counts` | | | | -| `projects_webex_teams_active` | `counts` | | | | -| `projects_youtrack_active` | `counts` | | | | -| `projects_slack_notifications_active` | `counts` | | | | -| `projects_slack_slash_active` | `counts` | | | | -| `projects_jira_server_active` | `counts` | | | | -| `projects_jira_cloud_active` | `counts` | | | | -| `projects_jira_dvcs_cloud_active` | `counts` | | | | -| `projects_jira_dvcs_server_active` | `counts` | | | | -| `labels` | `counts` | | | | -| `merge_requests` | `counts` | | | | -| `merge_requests_users` | `counts` | | | | -| `notes` | `counts` | | | | -| `wiki_pages_create` | `counts` | | | | -| `wiki_pages_update` | `counts` | | | | -| `wiki_pages_delete` | `counts` | | | | -| `web_ide_commits` | `counts` | | | | -| `web_ide_views` | `counts` | | | | -| `web_ide_merge_requests` | `counts` | | | | -| `web_ide_previews` | `counts` | | | | -| `snippet_comment` | `counts` | | | | -| `commit_comment` | `counts` | | | | -| `merge_request_comment` | `counts` | | | | -| `snippet_create` | `counts` | | | | -| `snippet_update` | `counts` | | | | -| `navbar_searches` | `counts` | | | | -| `cycle_analytics_views` | `counts` | | | | -| `productivity_analytics_views` | `counts` | | | | -| `source_code_pushes` | `counts` | | | | -| `merge_request_create` | `counts` | | | | -| `design_management_designs_create` | `counts` | | | | -| `design_management_designs_update` | `counts` | | | | -| `design_management_designs_delete` | `counts` | | | | -| `licenses_list_views` | `counts` | | | | -| `user_preferences_group_overview_details` | `counts` | | | | -| `user_preferences_group_overview_security_dashboard` | `counts` | | | | -| `ingress_modsecurity_logging` | `counts` | | | | -| `ingress_modsecurity_blocking` | `counts` | | | | -| `ingress_modsecurity_disabled` | `counts` | | | | -| `ingress_modsecurity_not_installed` | `counts` | | | | -| `dependency_list_usages_total` | `counts` | | | | -| `epics` | `counts` | | | | -| `feature_flags` | `counts` | | | | -| `geo_nodes` | `counts` | `geo` | | Number of sites in a Geo deployment | -| `geo_event_log_max_id` | `counts` | `geo` | | Number of replication events on a Geo primary | -| `incident_issues` | `counts` | `monitor` | | Issues created by the alert bot | -| `alert_bot_incident_issues` | `counts` | `monitor` | | Issues created by the alert bot | -| `incident_labeled_issues` | `counts` | `monitor` | | Issues with the incident label | -| `issues_created_gitlab_alerts` | `counts` | `monitor` | | Issues created from alerts by non-alert bot users | -| `issues_created_manually_from_alerts` | `counts` | `monitor` | | Issues created from alerts by non-alert bot users | -| `issues_created_from_alerts` | `counts` | `monitor` | | Issues created from Prometheus and alert management alerts | -| `ldap_group_links` | `counts` | | | | -| `ldap_keys` | `counts` | | | | -| `ldap_users` | `counts` | | | | -| `pod_logs_usages_total` | `counts` | | | | -| `projects_enforcing_code_owner_approval` | `counts` | | | | -| `projects_mirrored_with_pipelines_enabled` | `counts` |`release` | | Projects with repository mirroring enabled | -| `projects_reporting_ci_cd_back_to_github` | `counts` |`verify` | | Projects with a GitHub service pipeline enabled | -| `projects_with_packages` | `counts` |`package` | | Projects with package registry configured | -| `projects_with_prometheus_alerts` | `counts` |`monitor` | | Projects with Prometheus alerting enabled | -| `projects_with_tracing_enabled` | `counts` |`monitor` | | Projects with tracing enabled | -| `projects_with_alerts_service_enabled` | `counts` |`monitor` | | Projects with alerting service enabled | -| `template_repositories` | `counts` | | | | -| `container_scanning_jobs` | `counts` | | | | -| `dependency_scanning_jobs` | `counts` | | | | -| `license_management_jobs` | `counts` | | | | -| `sast_jobs` | `counts` | | | | -| `status_page_projects` | `counts` | `monitor` | | Projects with status page enabled | -| `status_page_issues` | `counts` | `monitor` | | Issues published to a Status Page | -| `status_page_incident_publishes` | `counts` | `monitor` | | Cumulative count of usages of publish operation | -| `status_page_incident_unpublishes` | `counts` | `monitor` | | Cumulative count of usages of unpublish operation | -| `epics_deepest_relationship_level` | `counts` | | | | -| `operations_dashboard_default_dashboard` | `counts` | `monitor` | | Active users with enabled operations dashboard | -| `operations_dashboard_users_with_projects_added` | `counts` | `monitor` | | Active users with projects on operations dashboard| -| `container_registry_enabled` | | | | | -| `dependency_proxy_enabled` | | | | | -| `gitlab_shared_runners_enabled` | | | | | -| `gravatar_enabled` | | | | | -| `ldap_enabled` | | | | | -| `mattermost_enabled` | | | | | -| `omniauth_enabled` | | | | | -| `prometheus_metrics_enabled` | | | | | -| `reply_by_email_enabled` | | | | | -| `average` | `avg_cycle_analytics - code` | | | | -| `sd` | `avg_cycle_analytics - code` | | | | -| `missing` | `avg_cycle_analytics - code` | | | | -| `average` | `avg_cycle_analytics - test` | | | | -| `sd` | `avg_cycle_analytics - test` | | | | -| `missing` | `avg_cycle_analytics - test` | | | | -| `average` | `avg_cycle_analytics - review` | | | | -| `sd` | `avg_cycle_analytics - review` | | | | -| `missing` | `avg_cycle_analytics - review` | | | | -| `average` | `avg_cycle_analytics - staging` | | | | -| `sd` | `avg_cycle_analytics - staging` | | | | -| `missing` | `avg_cycle_analytics - staging` | | | | -| `average` | `avg_cycle_analytics - production` | | | | -| `sd` | `avg_cycle_analytics - production` | | | | -| `missing` | `avg_cycle_analytics - production` | | | | -| `total` | `avg_cycle_analytics` | | | | -| `clusters_applications_cert_managers` | `usage_activity_by_stage` | `configure` | | Unique clusters with certificate managers enabled | -| `clusters_applications_helm` | `usage_activity_by_stage` | `configure` | | Unique clusters with Helm enabled | -| `clusters_applications_ingress` | `usage_activity_by_stage` | `configure` | | Unique clusters with Ingress enabled | -| `clusters_applications_knative` | `usage_activity_by_stage` | `configure` | | Unique clusters with Knative enabled | -| `clusters_management_project` | `usage_activity_by_stage` | `configure` | | Unique clusters with project management enabled | -| `clusters_disabled` | `usage_activity_by_stage` | `configure` | | Total non-"GitLab Managed clusters" | -| `clusters_enabled` | `usage_activity_by_stage` | `configure` | | Total GitLab Managed clusters | -| `clusters_platforms_gke` | `usage_activity_by_stage` | `configure` | | Unique clusters with Google Cloud installed | -| `clusters_platforms_eks` | `usage_activity_by_stage` | `configure` | | Unique clusters with AWS installed | -| `clusters_platforms_user` | `usage_activity_by_stage` | `configure` | | Unique clusters that are user provided | -| `instance_clusters_disabled` | `usage_activity_by_stage` | `configure` | | Unique clusters disabled on instance | -| `instance_clusters_enabled` | `usage_activity_by_stage` | `configure` | | Unique clusters enabled on instance | -| `group_clusters_disabled` | `usage_activity_by_stage` | `configure` | | Unique clusters disabled on group | -| `group_clusters_enabled` | `usage_activity_by_stage` | `configure` | | Unique clusters enabled on group | -| `project_clusters_disabled` | `usage_activity_by_stage` | `configure` | | Unique clusters disabled on project | -| `project_clusters_enabled` | `usage_activity_by_stage` | `configure` | | Unique clusters enabled on project | -| `projects_slack_notifications_active` | `usage_activity_by_stage` | `configure` | | Unique projects with Slack service enabled | -| `projects_slack_slash_active` | `usage_activity_by_stage` | `configure` | | Unique projects with Slack '/' commands enabled | -| `projects_with_prometheus_alerts: 0` | `usage_activity_by_stage` | `monitor` | | Projects with Prometheus enabled and no alerts | -| `deploy_keys` | `usage_activity_by_stage` | `create` | | | -| `keys` | `usage_activity_by_stage` | `create` | | | -| `projects_jira_dvcs_server_active` | `usage_activity_by_stage` | `plan` | | | -| `service_desk_enabled_projects` | `usage_activity_by_stage` | `plan` | | | -| `service_desk_issues` | `usage_activity_by_stage` | `plan` | | | -| `todos: 0` | `usage_activity_by_stage` | `plan` | | | -| `deployments` | `usage_activity_by_stage` | `release` | | Total deployments | -| `failed_deployments` | `usage_activity_by_stage` | `release` | | Total failed deployments | -| `projects_mirrored_with_pipelines_enabled` | `usage_activity_by_stage` | `release` | | Projects with repository mirroring enabled | -| `releases` | `usage_activity_by_stage` | `release` | | Unique release tags in project | -| `successful_deployments: 0` | `usage_activity_by_stage` | `release` | | Total successful deployments | -| `user_preferences_group_overview_security_dashboard: 0` | `usage_activity_by_stage` | `secure` | | | -| `ci_builds` | `usage_activity_by_stage` | `verify` | | Unique builds in project | -| `ci_external_pipelines` | `usage_activity_by_stage` | `verify` | | Total pipelines in external repositories | -| `ci_internal_pipelines` | `usage_activity_by_stage` | `verify` | | Total pipelines in GitLab repositories | -| `ci_pipeline_config_auto_devops` | `usage_activity_by_stage` | `verify` | | Total pipelines from an Auto DevOps template | -| `ci_pipeline_config_repository` | `usage_activity_by_stage` | `verify` | | Pipelines from templates in repository | -| `ci_pipeline_schedules` | `usage_activity_by_stage` | `verify` | | Pipeline schedules in GitLab | -| `ci_pipelines` | `usage_activity_by_stage` | `verify` | | Total pipelines | -| `ci_triggers` | `usage_activity_by_stage` | `verify` | | Triggers enabled | -| `clusters_applications_runner` | `usage_activity_by_stage` | `verify` | | Unique clusters with Runner enabled | -| `projects_reporting_ci_cd_back_to_github: 0` | `usage_activity_by_stage` | `verify` | | Unique projects with a GitHub pipeline enabled | -| `nodes` | `topology` | `enablement`| | The list of server nodes on which GitLab components are running | -| `duration_s` | `topology` | `enablement`| | Time it took to collect topology data | -| `node_memory_total_bytes` | `topology > nodes` | `enablement`| | The total available memory of this node | -| `node_cpus` | `topology > nodes` | `enablement`| | The number of CPU cores of this node | -| `node_services` | `topology > nodes` | `enablement`| | The list of GitLab services running on this node | -| `name` | `topology > nodes > node_services` | `enablement`| | The name of the GitLab service running on this node | -| `process_count` | `topology > nodes > node_services` | `enablement`| | The number of processes running for this service | -| `process_memory_rss` | `topology > nodes > node_services` | `enablement`| | The average Resident Set Size of a service process | -| `process_memory_uss` | `topology > nodes > node_services` | `enablement`| | The average Unique Set Size of a service process | -| `process_memory_pss` | `topology > nodes > node_services` | `enablement`| | The average Proportional Set Size of a service process | +| Statistic | Section | Stage | Tier | Edition | Description | +| --------------------------------------------------------- | ------------------------------------ | ------------- | ---------------- | ------- | -------------------------------------------------------------------------- | +| `uuid` | | | | | | +| `hostname` | | | | | | +| `version` | | | | | | +| `installation_type` | | | | | | +| `active_user_count` | | | | | | +| `recorded_at` | | | | | | +| `recording_ce_finished_at` | | | | CE+EE | When the core features were computed | +| `recording_ee_finished_at` | | | | EE | When the EE-specific features were computed | +| `edition` | | | | | | +| `license_md5` | | | | | | +| `license_id` | | | | | | +| `historical_max_users` | | | | | | +| `Name` | `licensee` | | | | | +| `Email` | `licensee` | | | | | +| `Company` | `licensee` | | | | | +| `license_user_count` | | | | | | +| `license_starts_at` | | | | | | +| `license_expires_at` | | | | | | +| `license_plan` | | | | | | +| `license_trial` | | | | | | +| `assignee_lists` | `counts` | | | | | +| `boards` | `counts` | | | | | +| `ci_builds` | `counts` | `verify` | | | Unique builds in project | +| `ci_internal_pipelines` | `counts` | `verify` | | | Total pipelines in GitLab repositories | +| `ci_external_pipelines` | `counts` | `verify` | | | Total pipelines in external repositories | +| `ci_pipeline_config_auto_devops` | `counts` | `verify` | | | Total pipelines from an Auto DevOps template | +| `ci_pipeline_config_repository` | `counts` | `verify` | | | Total Pipelines from templates in repository | +| `ci_runners` | `counts` | `verify` | | | Total configured Runners in project | +| `ci_triggers` | `counts` | `verify` | | | Total configured Triggers in project | +| `ci_pipeline_schedules` | `counts` | `verify` | | | Pipeline schedules in GitLab | +| `auto_devops_enabled` | `counts` | `configure` | | | Projects with Auto DevOps template enabled | +| `auto_devops_disabled` | `counts` | `configure` | | | Projects with Auto DevOps template disabled | +| `deploy_keys` | `counts` | | | | | +| `deployments` | `counts` | `release` | | | Total deployments | +| `deployments` | `counts_monthly` | `release` | | | Total deployments last 28 days | +| `dast_jobs` | `counts` | | | | | +| `successful_deployments` | `counts` | `release` | | | Total successful deployments | +| `successful_deployments` | `counts_monthly` | `release` | | | Total successful deployments last 28 days | +| `failed_deployments` | `counts` | `release` | | | Total failed deployments | +| `failed_deployments` | `counts_monthly` | `release` | | | Total failed deployments last 28 days | +| `environments` | `counts` | `release` | | | Total available and stopped environments | +| `clusters` | `counts` | `configure` | | | Total GitLab Managed clusters both enabled and disabled | +| `clusters_enabled` | `counts` | `configure` | | | Total GitLab Managed clusters currently enabled | +| `project_clusters_enabled` | `counts` | `configure` | | | Total GitLab Managed clusters attached to projects | +| `group_clusters_enabled` | `counts` | `configure` | | | Total GitLab Managed clusters attached to groups | +| `instance_clusters_enabled` | `counts` | `configure` | | | Total GitLab Managed clusters attached to the instance | +| `clusters_disabled` | `counts` | `configure` | | | Total GitLab Managed disabled clusters | +| `project_clusters_disabled` | `counts` | `configure` | | | Total GitLab Managed disabled clusters previously attached to projects | +| `group_clusters_disabled` | `counts` | `configure` | | | Total GitLab Managed disabled clusters previously attached to groups | +| `instance_clusters_disabled` | `counts` | `configure` | | | Total GitLab Managed disabled clusters previously attached to the instance | +| `clusters_platforms_eks` | `counts` | `configure` | | | Total GitLab Managed clusters provisioned with GitLab on AWS EKS | +| `clusters_platforms_gke` | `counts` | `configure` | | | Total GitLab Managed clusters provisioned with GitLab on GCE GKE | +| `clusters_platforms_user` | `counts` | `configure` | | | Total GitLab Managed clusters that are user provisioned | +| `clusters_applications_helm` | `counts` | `configure` | | | Total GitLab Managed clusters with Helm enabled | +| `clusters_applications_ingress` | `counts` | `configure` | | | Total GitLab Managed clusters with Ingress enabled | +| `clusters_applications_cert_managers` | `counts` | `configure` | | | Total GitLab Managed clusters with Cert Manager enabled | +| `clusters_applications_crossplane` | `counts` | `configure` | | | Total GitLab Managed clusters with Crossplane enabled | +| `clusters_applications_prometheus` | `counts` | `configure` | | | Total GitLab Managed clusters with Prometheus enabled | +| `clusters_applications_runner` | `counts` | `configure` | | | Total GitLab Managed clusters with Runner enabled | +| `clusters_applications_knative` | `counts` | `configure` | | | Total GitLab Managed clusters with Knative enabled | +| `clusters_applications_elastic_stack` | `counts` | `configure` | | | Total GitLab Managed clusters with Elastic Stack enabled | +| `clusters_applications_cilium` | `counts` | `configure` | | | Total GitLab Managed clusters with Cilium enabled | +| `clusters_management_project` | `counts` | `configure` | | | Total GitLab Managed clusters with defined cluster management project | +| `in_review_folder` | `counts` | | | | | +| `grafana_integrated_projects` | `counts` | | | | | +| `groups` | `counts` | | | | | +| `issues` | `counts` | | | | | +| `issues_created_from_gitlab_error_tracking_ui` | `counts` | `monitor` | | | | +| `issues_with_associated_zoom_link` | `counts` | `monitor` | | | | +| `issues_using_zoom_quick_actions` | `counts` | `monitor` | | | | +| `issues_with_embedded_grafana_charts_approx` | `counts` | `monitor` | | | | +| `issues_with_health_status` | `counts` | | | | | +| `keys` | `counts` | | | | | +| `label_lists` | `counts` | | | | | +| `lfs_objects` | `counts` | | | | | +| `milestone_lists` | `counts` | | | | | +| `milestones` | `counts` | | | | | +| `pages_domains` | `counts` | `release` | | | Total GitLab Pages domains | +| `pool_repositories` | `counts` | | | | | +| `projects` | `counts` | | | | | +| `projects_imported_from_github` | `counts` | | | | | +| `projects_with_repositories_enabled` | `counts` | | | | | +| `projects_with_error_tracking_enabled` | `counts` | `monitor` | | | | +| `protected_branches` | `counts` | | | | | +| `releases` | `counts` | `release` | | | Unique release tags | +| `remote_mirrors` | `counts` | | | | | +| `requirements_created` | `counts` | | | | | +| `snippets` | `counts` | 'create' | | CE+EE | | +| `snippets` | `counts_monthly` | 'create' | | CE+EE | | +| `personal_snippets` | `counts` | 'create' | | CE+EE | | +| `personal_snippets` | `counts_monthly` | 'create' | | CE+EE | | +| `project_snippets` | `counts` | 'create' | | CE+EE | | +| `project_snippets` | `counts_monthly` | 'create' | | CE+EE | | +| `suggestions` | `counts` | | | | | +| `todos` | `counts` | | | | | +| `uploads` | `counts` | | | | | +| `web_hooks` | `counts` | | | | | +| `projects_alerts_active` | `counts` | | | | | +| `projects_asana_active` | `counts` | | | | | +| `projects_assembla_active` | `counts` | | | | | +| `projects_bamboo_active` | `counts` | | | | | +| `projects_bugzilla_active` | `counts` | | | | | +| `projects_buildkite_active` | `counts` | | | | | +| `projects_campfire_active` | `counts` | | | | | +| `projects_custom_issue_tracker_active` | `counts` | | | | | +| `projects_discord_active` | `counts` | | | | | +| `projects_drone_ci_active` | `counts` | | | | | +| `projects_emails_on_push_active` | `counts` | | | | | +| `projects_external_wiki_active` | `counts` | | | | | +| `projects_flowdock_active` | `counts` | | | | | +| `projects_github_active` | `counts` | | | | | +| `projects_hangouts_chat_active` | `counts` | | | | | +| `projects_hipchat_active` | `counts` | | | | | +| `projects_irker_active` | `counts` | | | | | +| `projects_jenkins_active` | `counts` | | | | | +| `projects_jira_active` | `counts` | | | | | +| `projects_mattermost_active` | `counts` | | | | | +| `projects_mattermost_slash_commands_active` | `counts` | | | | | +| `projects_microsoft_teams_active` | `counts` | | | | | +| `projects_packagist_active` | `counts` | | | | | +| `projects_pipelines_email_active` | `counts` | | | | | +| `projects_pivotaltracker_active` | `counts` | | | | | +| `projects_prometheus_active` | `counts` | | | | | +| `projects_pushover_active` | `counts` | | | | | +| `projects_redmine_active` | `counts` | | | | | +| `projects_slack_active` | `counts` | | | | | +| `projects_slack_slash_commands_active` | `counts` | | | | | +| `projects_teamcity_active` | `counts` | | | | | +| `projects_unify_circuit_active` | `counts` | | | | | +| `projects_webex_teams_active` | `counts` | | | | | +| `projects_youtrack_active` | `counts` | | | | | +| `projects_jira_server_active` | `counts` | | | | | +| `projects_jira_cloud_active` | `counts` | | | | | +| `projects_jira_dvcs_cloud_active` | `counts` | | | | | +| `projects_jira_dvcs_server_active` | `counts` | | | | | +| `projects_jira_issuelist_active` | `counts` | `create` | | EE | Total Jira Issue feature enabled | +| `labels` | `counts` | | | | | +| `merge_requests` | `counts` | | | | | +| `merge_requests_users` | `counts` | | | | | +| `notes` | `counts` | | | | | +| `wiki_pages_create` | `counts` | | | | | +| `wiki_pages_update` | `counts` | | | | | +| `wiki_pages_delete` | `counts` | | | | | +| `web_ide_commits` | `counts` | | | | | +| `web_ide_views` | `counts` | | | | | +| `web_ide_merge_requests` | `counts` | | | | | +| `web_ide_previews` | `counts` | | | | | +| `snippet_comment` | `counts` | | | | | +| `commit_comment` | `counts` | | | | | +| `merge_request_comment` | `counts` | | | | | +| `snippet_create` | `counts` | | | | | +| `snippet_update` | `counts` | | | | | +| `navbar_searches` | `counts` | | | | | +| `cycle_analytics_views` | `counts` | | | | | +| `productivity_analytics_views` | `counts` | | | | | +| `source_code_pushes` | `counts` | | | | | +| `merge_request_create` | `counts` | | | | | +| `design_management_designs_create` | `counts` | | | | | +| `design_management_designs_update` | `counts` | | | | | +| `design_management_designs_delete` | `counts` | | | | | +| `licenses_list_views` | `counts` | | | | | +| `user_preferences_group_overview_details` | `counts` | | | | | +| `user_preferences_group_overview_security_dashboard` | `counts` | | | | | +| `ingress_modsecurity_logging` | `counts` | | | | | +| `ingress_modsecurity_blocking` | `counts` | | | | | +| `ingress_modsecurity_disabled` | `counts` | | | | | +| `ingress_modsecurity_not_installed` | `counts` | | | | | +| `dependency_list_usages_total` | `counts` | | | | | +| `epics` | `counts` | | | | | +| `feature_flags` | `counts` | | | | | +| `geo_nodes` | `counts` | `geo` | | | Number of sites in a Geo deployment | +| `geo_event_log_max_id` | `counts` | `geo` | | | Number of replication events on a Geo primary | +| `incident_issues` | `counts` | `monitor` | | | Issues created by the alert bot | +| `alert_bot_incident_issues` | `counts` | `monitor` | | | Issues created by the alert bot | +| `incident_labeled_issues` | `counts` | `monitor` | | | Issues with the incident label | +| `issues_created_gitlab_alerts` | `counts` | `monitor` | | | Issues created from alerts by non-alert bot users | +| `issues_created_manually_from_alerts` | `counts` | `monitor` | | | Issues created from alerts by non-alert bot users | +| `issues_created_from_alerts` | `counts` | `monitor` | | | Issues created from Prometheus and alert management alerts | +| `ldap_group_links` | `counts` | | | | | +| `ldap_keys` | `counts` | | | | | +| `ldap_users` | `counts` | | | | | +| `pod_logs_usages_total` | `counts` | | | | | +| `projects_enforcing_code_owner_approval` | `counts` | | | | | +| `projects_mirrored_with_pipelines_enabled` | `counts` | `release` | | | Projects with repository mirroring enabled | +| `projects_reporting_ci_cd_back_to_github` | `counts` | `verify` | | | Projects with a GitHub service pipeline enabled | +| `projects_with_packages` | `counts` | `package` | | | Projects with package registry configured | +| `projects_with_prometheus_alerts` | `counts` | `monitor` | | | Projects with Prometheus alerting enabled | +| `projects_with_tracing_enabled` | `counts` | `monitor` | | | Projects with tracing enabled | +| `projects_with_alerts_service_enabled` | `counts` | `monitor` | | | Projects with alerting service enabled | +| `template_repositories` | `counts` | | | | | +| `container_scanning_jobs` | `counts` | | | | | +| `dependency_scanning_jobs` | `counts` | | | | | +| `license_management_jobs` | `counts` | | | | | +| `sast_jobs` | `counts` | | | | | +| `status_page_projects` | `counts` | `monitor` | | | Projects with status page enabled | +| `status_page_issues` | `counts` | `monitor` | | | Issues published to a Status Page | +| `status_page_incident_publishes` | `counts` | `monitor` | | | Cumulative count of usages of publish operation | +| `status_page_incident_unpublishes` | `counts` | `monitor` | | | Cumulative count of usages of unpublish operation | +| `epics_deepest_relationship_level` | `counts` | | | | | +| `operations_dashboard_default_dashboard` | `counts` | `monitor` | | | Active users with enabled operations dashboard | +| `operations_dashboard_users_with_projects_added` | `counts` | `monitor` | | | Active users with projects on operations dashboard | +| `container_registry_enabled` | | | | | | +| `dependency_proxy_enabled` | | | | | | +| `gitlab_shared_runners_enabled` | | | | | | +| `gravatar_enabled` | | | | | | +| `ldap_enabled` | | | | | | +| `mattermost_enabled` | | | | | | +| `omniauth_enabled` | | | | | | +| `prometheus_enabled` | | | | | Whether the bundled Prometheus is enabled | +| `prometheus_metrics_enabled` | | | | | | +| `reply_by_email_enabled` | | | | | | +| `average` | `avg_cycle_analytics - code` | | | | | +| `sd` | `avg_cycle_analytics - code` | | | | | +| `missing` | `avg_cycle_analytics - code` | | | | | +| `average` | `avg_cycle_analytics - test` | | | | | +| `sd` | `avg_cycle_analytics - test` | | | | | +| `missing` | `avg_cycle_analytics - test` | | | | | +| `average` | `avg_cycle_analytics - review` | | | | | +| `sd` | `avg_cycle_analytics - review` | | | | | +| `missing` | `avg_cycle_analytics - review` | | | | | +| `average` | `avg_cycle_analytics - staging` | | | | | +| `sd` | `avg_cycle_analytics - staging` | | | | | +| `missing` | `avg_cycle_analytics - staging` | | | | | +| `average` | `avg_cycle_analytics - production` | | | | | +| `sd` | `avg_cycle_analytics - production` | | | | | +| `missing` | `avg_cycle_analytics - production` | | | | | +| `total` | `avg_cycle_analytics` | | | | | +| `g_analytics_contribution` | `analytics_unique_visits` | `manage` | | | Visits to /groups/:group/-/contribution_analytics | +| `g_analytics_insights` | `analytics_unique_visits` | `manage` | | | Visits to /groups/:group/-/insights | +| `g_analytics_issues` | `analytics_unique_visits` | `manage` | | | Visits to /groups/:group/-/issues_analytics | +| `g_analytics_productivity` | `analytics_unique_visits` | `manage` | | | Visits to /groups/:group/-/analytics/productivity_analytics | +| `g_analytics_valuestream` | `analytics_unique_visits` | `manage` | | | Visits to /groups/:group/-/analytics/value_stream_analytics | +| `p_analytics_pipelines` | `analytics_unique_visits` | `manage` | | | Visits to /:group/:project/pipelines/charts | +| `p_analytics_code_reviews` | `analytics_unique_visits` | `manage` | | | Visits to /:group/:project/-/analytics/code_reviews | +| `p_analytics_valuestream` | `analytics_unique_visits` | `manage` | | | Visits to /:group/:project/-/value_stream_analytics | +| `p_analytics_insights` | `analytics_unique_visits` | `manage` | | | Visits to /:group/:project/insights | +| `p_analytics_issues` | `analytics_unique_visits` | `manage` | | | Visits to /:group/:project/-/analytics/issues_analytics | +| `p_analytics_repo` | `analytics_unique_visits` | `manage` | | | Visits to /:group/:project/-/graphs/master/charts | +| `u_analytics_todos` | `analytics_unique_visits` | `manage` | | | Visits to /dashboard/todos | +| `i_analytics_cohorts` | `analytics_unique_visits` | `manage` | | | Visits to /-/instance_statistics/cohorts | +| `i_analytics_dev_ops_score` | `analytics_unique_visits` | `manage` | | | Visits to /-/instance_statistics/dev_ops_score | +| `analytics_unique_visits_for_any_target` | `analytics_unique_visits` | `manage` | | | Visits to any of the pages listed above | +| `clusters_applications_cert_managers` | `usage_activity_by_stage` | `configure` | | CE+EE | Unique clusters with certificate managers enabled | +| `clusters_applications_helm` | `usage_activity_by_stage` | `configure` | | CE+EE | Unique clusters with Helm enabled | +| `clusters_applications_ingress` | `usage_activity_by_stage` | `configure` | | CE+EE | Unique clusters with Ingress enabled | +| `clusters_applications_knative` | `usage_activity_by_stage` | `configure` | | CE+EE | Unique clusters with Knative enabled | +| `clusters_management_project` | `usage_activity_by_stage` | `configure` | | CE+EE | Unique clusters with project management enabled | +| `clusters_disabled` | `usage_activity_by_stage` | `configure` | | CE+EE | Total non-"GitLab Managed clusters" | +| `clusters_enabled` | `usage_activity_by_stage` | `configure` | | CE+EE | Total GitLab Managed clusters | +| `clusters_platforms_gke` | `usage_activity_by_stage` | `configure` | | CE+EE | Unique clusters with Google Cloud installed | +| `clusters_platforms_eks` | `usage_activity_by_stage` | `configure` | | CE+EE | Unique clusters with AWS installed | +| `clusters_platforms_user` | `usage_activity_by_stage` | `configure` | | CE+EE | Unique clusters that are user provided | +| `instance_clusters_disabled` | `usage_activity_by_stage` | `configure` | | CE+EE | Unique clusters disabled on instance | +| `instance_clusters_enabled` | `usage_activity_by_stage` | `configure` | | CE+EE | Unique clusters enabled on instance | +| `group_clusters_disabled` | `usage_activity_by_stage` | `configure` | | CE+EE | Unique clusters disabled on group | +| `group_clusters_enabled` | `usage_activity_by_stage` | `configure` | | CE+EE | Unique clusters enabled on group | +| `project_clusters_disabled` | `usage_activity_by_stage` | `configure` | | CE+EE | Unique clusters disabled on project | +| `project_clusters_enabled` | `usage_activity_by_stage` | `configure` | | CE+EE | Unique clusters enabled on project | +| `projects_slack_notifications_active` | `usage_activity_by_stage` | `configure` | | EE | Unique projects with Slack service enabled | +| `projects_slack_slash_active` | `usage_activity_by_stage` | `configure` | | EE | Unique projects with Slack '/' commands enabled | +| `projects_with_prometheus_alerts` | `usage_activity_by_stage` | `configure` | | EE | Projects with Prometheus enabled and no alerts | +| `deploy_keys` | `usage_activity_by_stage` | `create` | | CE+EE | | +| `keys` | `usage_activity_by_stage` | `create` | | CE+EE | | +| `merge_requests` | `usage_activity_by_stage` | `create` | | CE+EE | | +| `projects_with_disable_overriding_approvers_per_merge_request` | `usage_activity_by_stage` | `create` | | CE+EE | | +| `projects_without_disable_overriding_approvers_per_merge_request` | `usage_activity_by_stage` | `create` | | CE+EE | | +| `remote_mirrors` | `usage_activity_by_stage` | `create` | | CE+EE | | +| `snippets` | `usage_activity_by_stage` | `create` | | CE+EE | | +| `merge_requests_users` | `usage_activity_by_stage_monthly` | `create` | | CE+EE | Unique count of users who used a merge request | +| `action_monthly_active_users_project_repo` | `usage_activity_by_stage_monthly` | `create` | | CE+EE | Unique count of users who pushed to a project repo | +| `action_monthly_active_users_design_management` | `usage_activity_by_stage_monthly` | `create` | | CE+EE | Unique count of users who interacted with the design system management | +| `action_monthly_active_users_wiki_repo` | `usage_activity_by_stage_monthly` | `create` | | CE+EE | Unique count of users who created or updated a wiki repo | +| `projects_enforcing_code_owner_approval` | `usage_activity_by_stage` | `create` | | EE | | +| `merge_requests_with_optional_codeowners` | `usage_activity_by_stage` | `create` | | EE | | +| `merge_requests_with_required_codeowners` | `usage_activity_by_stage` | `create` | | EE | | +| `projects_imported_from_github` | `usage_activity_by_stage` | `create` | | EE | | +| `projects_with_repositories_enabled` | `usage_activity_by_stage` | `create` | | EE | | +| `protected_branches` | `usage_activity_by_stage` | `create` | | EE | | +| `suggestions` | `usage_activity_by_stage` | `create` | | EE | | +| `approval_project_rules` | `usage_activity_by_stage` | `create` | | EE | Number of project approval rules | +| `approval_project_rules_with_target_branch` | `usage_activity_by_stage` | `create` | | EE | Number of project approval rules with not default target branch | +| `merge_requests_with_added_rules` | `usage_activity_by_stage` | `create` | | EE | Merge Requests with added rules | +| `clusters` | `usage_activity_by_stage` | `monitor` | | CE+EE | | +| `clusters_applications_prometheus` | `usage_activity_by_stage` | `monitor` | | CE+EE | | +| `operations_dashboard_default_dashboard` | `usage_activity_by_stage` | `monitor` | | CE+EE | | +| `operations_dashboard_users_with_projects_added` | `usage_activity_by_stage` | `monitor` | | EE | | +| `projects_prometheus_active` | `usage_activity_by_stage` | `monitor` | | EE | | +| `projects_with_error_tracking_enabled` | `usage_activity_by_stage` | `monitor` | | EE | | +| `projects_with_tracing_enabled` | `usage_activity_by_stage` | `monitor` | | EE | | +| `events` | `usage_activity_by_stage` | `manage` | | CE+EE | | +| `groups` | `usage_activity_by_stage` | `manage` | | CE+EE | | +| `users_created_at` | `usage_activity_by_stage` | `manage` | | CE+EE | | +| `omniauth_providers` | `usage_activity_by_stage` | `manage` | | CE+EE | | +| `ldap_keys` | `usage_activity_by_stage` | `manage` | | EE | | +| `ldap_users` | `usage_activity_by_stage` | `manage` | | EE | | +| `value_stream_management_customized_group_stages` | `usage_activity_by_stage` | `manage` | | EE | | +| `projects_with_compliance_framework` | `usage_activity_by_stage` | `manage` | | EE | | +| `ldap_servers` | `usage_activity_by_stage` | `manage` | | EE | | +| `ldap_group_sync_enabled` | `usage_activity_by_stage` | `manage` | | EE | | +| `ldap_admin_sync_enabled` | `usage_activity_by_stage` | `manage` | | EE | | +| `group_saml_enabled` | `usage_activity_by_stage` | `manage` | | EE | | +| `issues` | `usage_activity_by_stage` | `plan` | | CE+EE | | +| `notes` | `usage_activity_by_stage` | `plan` | | CE+EE | | +| `projects` | `usage_activity_by_stage` | `plan` | | CE+EE | | +| `todos` | `usage_activity_by_stage` | `plan` | | CE+EE | | +| `assignee_lists` | `usage_activity_by_stage` | `plan` | | EE | | +| `epics` | `usage_activity_by_stage` | `plan` | | EE | | +| `label_lists` | `usage_activity_by_stage` | `plan` | | EE | | +| `milestone_lists` | `usage_activity_by_stage` | `plan` | | EE | | +| `projects_jira_active` | `usage_activity_by_stage` | `plan` | | EE | | +| `projects_jira_dvcs_server_active` | `usage_activity_by_stage` | `plan` | | EE | | +| `projects_jira_dvcs_server_active` | `usage_activity_by_stage` | `plan` | | EE | | +| `service_desk_enabled_projects` | `usage_activity_by_stage` | `plan` | | CE+EE | | +| `service_desk_issues` | `usage_activity_by_stage` | `plan` | | CE+EE | | +| `deployments` | `usage_activity_by_stage` | `release` | | CE+EE | Total deployments | +| `failed_deployments` | `usage_activity_by_stage` | `release` | | CE+EE | Total failed deployments | +| `projects_mirrored_with_pipelines_enabled` | `usage_activity_by_stage` | `release` | | EE | Projects with repository mirroring enabled | +| `releases` | `usage_activity_by_stage` | `release` | | CE+EE | Unique release tags in project | +| `successful_deployments` | `usage_activity_by_stage` | `release` | | CE+EE | Total successful deployments | +| `user_preferences_group_overview_security_dashboard` | `usage_activity_by_stage` | `secure` | | | | +| `ci_builds` | `usage_activity_by_stage` | `verify` | | CE+EE | Unique builds in project | +| `ci_external_pipelines` | `usage_activity_by_stage` | `verify` | | CE+EE | Total pipelines in external repositories | +| `ci_internal_pipelines` | `usage_activity_by_stage` | `verify` | | CE+EE | Total pipelines in GitLab repositories | +| `ci_pipeline_config_auto_devops` | `usage_activity_by_stage` | `verify` | | CE+EE | Total pipelines from an Auto DevOps template | +| `ci_pipeline_config_repository` | `usage_activity_by_stage` | `verify` | | CE+EE | Pipelines from templates in repository | +| `ci_pipeline_schedules` | `usage_activity_by_stage` | `verify` | | CE+EE | Pipeline schedules in GitLab | +| `ci_pipelines` | `usage_activity_by_stage` | `verify` | | CE+EE | Total pipelines | +| `ci_triggers` | `usage_activity_by_stage` | `verify` | | CE+EE | Triggers enabled | +| `clusters_applications_runner` | `usage_activity_by_stage` | `verify` | | CE+EE | Unique clusters with Runner enabled | +| `projects_reporting_ci_cd_back_to_github` | `usage_activity_by_stage` | `verify` | | EE | Unique projects with a GitHub pipeline enabled | +| `merge_requests_users` | `usage_activity_by_stage_monthly` | `create` | | | Unique count of users who used a merge request | +| `duration_s` | `topology` | `enablement` | | | Time it took to collect topology data | +| `application_requests_per_hour` | `topology` | `enablement` | | | Number of requests to the web application per hour | +| `failures` | `topology` | `enablement` | | | Contains information about failed queries | +| `nodes` | `topology` | `enablement` | | | The list of server nodes on which GitLab components are running | +| `node_memory_total_bytes` | `topology > nodes` | `enablement` | | | The total available memory of this node | +| `node_cpus` | `topology > nodes` | `enablement` | | | The number of CPU cores of this node | +| `node_uname_info` | `topology > nodes` | `enablement` | | | The basic hardware architecture and OS release information on this node | +| `node_services` | `topology > nodes` | `enablement` | | | The list of GitLab services running on this node | +| `name` | `topology > nodes > node_services` | `enablement` | | | The name of the GitLab service running on this node | +| `process_count` | `topology > nodes > node_services` | `enablement` | | | The number of processes running for this service | +| `process_memory_rss` | `topology > nodes > node_services` | `enablement` | | | The average Resident Set Size of a service process | +| `process_memory_uss` | `topology > nodes > node_services` | `enablement` | | | The average Unique Set Size of a service process | +| `process_memory_pss` | `topology > nodes > node_services` | `enablement` | | | The average Proportional Set Size of a service process | +| `server` | `topology > nodes > node_services` | `enablement` | | | The type of web server used (Unicorn or Puma) | +| `network_policy_forwards` | `counts` | `defend` | | EE | Cumulative count of forwarded packets by Container Network | +| `network_policy_drops` | `counts` | `defend` | | EE | Cumulative count of dropped packets by Container Network | ## Example Usage Ping payload @@ -690,6 +780,7 @@ The following is example content of the Usage Ping payload. "ldap_enabled": false, "mattermost_enabled": false, "omniauth_enabled": true, + "prometheus_enabled": false, "prometheus_metrics_enabled": false, "reply_by_email_enabled": "incoming+%{key}@incoming.gitlab.com", "signup_enabled": true, @@ -765,6 +856,10 @@ The following is example content of the Usage Ping payload. }, "total": 999 }, + "analytics_unique_visits": { + "g_analytics_contribution": 999, + ... + }, "usage_activity_by_stage": { "configure": { "project_clusters_enabled": 999, @@ -840,17 +935,26 @@ The following is example content of the Usage Ping payload. } }, "topology": { + "duration_s": 0.013836685999194742, + "application_requests_per_hour": 4224, + "failures": [], "nodes": [ { "node_memory_total_bytes": 33269903360, "node_cpus": 16, + "node_uname_info": { + "machine": "x86_64", + "sysname": "Linux", + "release": "4.19.76-linuxkit" + }, "node_services": [ { "name": "web", "process_count": 16, "process_memory_pss": 233349888, "process_memory_rss": 788220927, - "process_memory_uss": 195295487 + "process_memory_uss": 195295487, + "server": "puma" }, { "name": "sidekiq", @@ -864,8 +968,7 @@ The following is example content of the Usage Ping payload. ... }, ... - ], - "duration_s": 0.013836685999194742 + ] } } ``` diff --git a/doc/development/testing_guide/best_practices.md b/doc/development/testing_guide/best_practices.md index 7bb8473117f..4e46e691405 100644 --- a/doc/development/testing_guide/best_practices.md +++ b/doc/development/testing_guide/best_practices.md @@ -110,7 +110,8 @@ Use the coverage reports to ensure your tests cover 100% of your code. ### System / Feature tests -NOTE: **Note:** Before writing a new system test, [please consider **not** +NOTE: **Note:** +Before writing a new system test, [please consider **not** writing one](testing_levels.md#consider-not-writing-a-system-test)! - Feature specs should be named `ROLE_ACTION_spec.rb`, such as @@ -741,7 +742,7 @@ GitLab uses [factory_bot](https://github.com/thoughtbot/factory_bot) as a test f - There should be only one top-level factory definition per file. - FactoryBot methods are mixed in to all RSpec groups. This means you can (and should) call `create(...)` instead of `FactoryBot.create(...)`. -- Make use of [traits](https://www.rubydoc.info/gems/factory_bot/file/GETTING_STARTED.md#Traits) to clean up definitions and usages. +- Make use of [traits](https://www.rubydoc.info/gems/factory_bot/file/GETTING_STARTED.md#traits) to clean up definitions and usages. - When defining a factory, don't define attributes that are not required for the resulting record to pass validation. - When instantiating from a factory, don't supply attributes that aren't @@ -813,6 +814,40 @@ file which is used by the `spec/fast_spec_helper.rb` file. See [Fast unit tests](#fast-unit-tests) for more details about the `spec/fast_spec_helper.rb` file. +### Test environment logging + +Services for the test environment are automatically configured and started when +tests are run, including Gitaly, Workhorse, Elasticsearch, and Capybara. When run in CI, or +if the service needs to be installed, the test environment will log information +about set-up time, producing log messages like the following: + +```plaintext +==> Setting up Gitaly... + Gitaly set up in 31.459649 seconds... + +==> Setting up GitLab Workhorse... + GitLab Workhorse set up in 29.695619 seconds... +fatal: update refs/heads/diff-files-symlink-to-image: invalid <newvalue>: 8cfca84 +From https://gitlab.com/gitlab-org/gitlab-test + * [new branch] diff-files-image-to-symlink -> origin/diff-files-image-to-symlink + * [new branch] diff-files-symlink-to-image -> origin/diff-files-symlink-to-image + * [new branch] diff-files-symlink-to-text -> origin/diff-files-symlink-to-text + * [new branch] diff-files-text-to-symlink -> origin/diff-files-text-to-symlink + b80faa8..40232f7 snippet/multiple-files -> origin/snippet/multiple-files + * [new branch] testing/branch-with-#-hash -> origin/testing/branch-with-#-hash + +==> Setting up GitLab Elasticsearch Indexer... + GitLab Elasticsearch Indexer set up in 26.514623 seconds... +``` + +This information is omitted when running locally and when no action needs +to be performed. If you would always like to see these messages, set the +following environment variable: + +```shell +GITLAB_TESTING_LOG_LEVEL=debug +``` + --- [Return to Testing documentation](index.md) diff --git a/doc/development/testing_guide/end_to_end/beginners_guide.md b/doc/development/testing_guide/end_to_end/beginners_guide.md index 73960a2f74d..15a9b4406ab 100644 --- a/doc/development/testing_guide/end_to_end/beginners_guide.md +++ b/doc/development/testing_guide/end_to_end/beginners_guide.md @@ -62,7 +62,7 @@ end-to-end flows, and is easiest to understand. The GitLab QA end-to-end tests are organized by the different [stages in the DevOps lifecycle](https://gitlab.com/gitlab-org/gitlab-foss/tree/master/qa/qa/specs/features/browser_ui). Determine where the test should be placed by -[stage](https://about.gitlab.com/handbook/product/categories/#devops-stages), +[stage](https://about.gitlab.com/handbook/product/product-categories/#devops-stages), determine which feature the test will belong to, and then place it in a subdirectory under the stage. @@ -80,13 +80,21 @@ file `basic_login_spec.rb`. ### The outer `context` block -Specs have an outer `context` indicating the DevOps stage. +See the [`RSpec.describe` outer block](#the-outer-rspecdescribe-block) + +CAUTION: **Deprecation notice:** +The outer `context` [was deprecated](https://gitlab.com/gitlab-org/quality/team-tasks/-/issues/550) in `13.2` +in adherance to RSpec 4.0 specifications. Use `RSpec.describe` instead. + +### The outer `RSpec.describe` block + +Specs have an outer `RSpec.describe` indicating the DevOps stage. ```ruby # frozen_string_literal: true module QA - context 'Manage' do + RSpec.describe 'Manage' do end end @@ -94,13 +102,13 @@ end ### The `describe` block -Inside of our outer `context`, describe the feature to test. In this case, `Login`. +Inside of our outer `RSpec.describe`, describe the feature to test. In this case, `Login`. ```ruby # frozen_string_literal: true module QA - context 'Manage' do + RSpec.describe 'Manage' do describe 'Login' do end @@ -115,7 +123,7 @@ writing end-to-end tests is to write test case descriptions as `it` blocks: ```ruby module QA - context 'Manage' do + RSpec.describe 'Manage' do describe 'Login' do it 'can login' do @@ -139,7 +147,7 @@ Begin by logging in. # frozen_string_literal: true module QA - context 'Manage' do + RSpec.describe 'Manage' do describe 'Login' do it 'can login' do Flow::Login.sign_in @@ -162,7 +170,7 @@ should answer the question "What do we test?" # frozen_string_literal: true module QA - context 'Manage' do + RSpec.describe 'Manage' do describe 'Login' do it 'can login' do Flow::Login.sign_in @@ -210,7 +218,7 @@ a call to `sign_in`. # frozen_string_literal: true module QA - context 'Manage' do + RSpec.describe 'Manage' do describe 'Login' do before do Flow::Login.sign_in @@ -247,7 +255,7 @@ stage, so [create a file](#identify-the-devops-stage) in # frozen_string_literal: true module QA - context 'Plan' do + RSpec.describe 'Plan' do describe 'Issues' do let(:issue) do Resource::Issue.fabricate_via_api! do |issue| diff --git a/doc/development/testing_guide/end_to_end/environment_selection.md b/doc/development/testing_guide/end_to_end/environment_selection.md new file mode 100644 index 00000000000..9eb7db72a51 --- /dev/null +++ b/doc/development/testing_guide/end_to_end/environment_selection.md @@ -0,0 +1,54 @@ +# Environment selection + +Some tests are designed to be run against specific environments. We can specify +what environments to run tests against using the `only` metadata. + +## Available switches + +| Switch | Function | Type | +| -------| ------- | ----- | +| `tld` | Set the top-level domain matcher | `String` | +| `subdomain` | Set the subdomain matcher | `Array` or `String` | +| `domain` | Set the domain matcher | `String` | +| `production` | Match against production | `Static` | + +CAUTION: **Caution:** +You cannot specify `:production` and `{ <switch>: 'value' }` simultaneously. +These options are mutually exclusive. If you want to specify production, you +can control the `tld` and `domain` independently. + +## Examples + +| Environment | Key | Matches (regex) | +| ---------------- | --- | --------------- | +| `any` | `` | `.+.com` | +| `gitlab.com` | `only: :production` | `gitlab.com` | +| `staging.gitlab.com` | `only: { subdomain: :staging }` | `(staging).+.com` | +| `gitlab.com and staging.gitlab.com` | `only: { subdomain: /(staging.)?/, domain: 'gitlab' }` | `(staging.)?gitlab.com` | +| `dev.gitlab.org` | `only: { tld: '.org', domain: 'gitlab', subdomain: 'dev' }` | `(dev).gitlab.org` | +| `staging.gitlab.com & domain.gitlab.com` | `only: { subdomain: %i[staging domain] }` | `(staging|domain).+.com` | + +```ruby +RSpec.describe 'Area' do + it 'runs in any environment' do; end + + it 'runs only in production', only: :production do; end + + it 'runs only in staging', only: { subdomain: :staging } do; end + + it 'runs in dev', only: { tld: '.org', domain: 'gitlab', subdomain: 'dev' } do; end + + it 'runs in prod and staging', only: { subdomain: /(staging.)?/, domain: 'gitlab' } {} +end +``` + +NOTE: **Note:** +If the test has a `before` or `after`, you must add the `only` metadata +to the outer `RSpec.describe`. + +## Quarantining a test for a specific environment + +Similarly to specifying that a test should only run against a specific environment, it's also possible to quarantine a +test only when it runs against a specific environment. The syntax is exactly the same, except that the `only: { ... }` +hash is nested in the [`quarantine: { ... }`](https://about.gitlab.com/handbook/engineering/quality/guidelines/debugging-qa-test-failures/#quarantining-tests) hash. +For instance, `quarantine: { only: { subdomain: :staging } }` will only quarantine the test when run against staging. diff --git a/doc/development/testing_guide/end_to_end/feature_flags.md b/doc/development/testing_guide/end_to_end/feature_flags.md index ada20cc9dad..87a9738b313 100644 --- a/doc/development/testing_guide/end_to_end/feature_flags.md +++ b/doc/development/testing_guide/end_to_end/feature_flags.md @@ -7,7 +7,7 @@ Note that administrator authorization is required to change feature flags. `QA:: Please be sure to include the tag `:requires_admin` so that the test can be skipped in environments where admin access is not available. ```ruby -context "with feature flag enabled", :requires_admin do +RSpec.describe "with feature flag enabled", :requires_admin do before do Runtime::Feature.enable('feature_flag_name') end diff --git a/doc/development/testing_guide/end_to_end/rspec_metadata_tests.md b/doc/development/testing_guide/end_to_end/rspec_metadata_tests.md index b1a8a14163c..4059c1960e2 100644 --- a/doc/development/testing_guide/end_to_end/rspec_metadata_tests.md +++ b/doc/development/testing_guide/end_to_end/rspec_metadata_tests.md @@ -10,10 +10,11 @@ This is a partial list of the [RSpec metadata](https://relishapp.com/rspec/rspec | `:elasticsearch` | The test requires an Elasticsearch service. It is used by the [instance-level scenario](https://gitlab.com/gitlab-org/gitlab-qa#definitions) [`Test::Integration::Elasticsearch`](https://gitlab.com/gitlab-org/gitlab/-/blob/72b62b51bdf513e2936301cb6c7c91ec27c35b4d/qa/qa/ee/scenario/test/integration/elasticsearch.rb) to include only tests that require Elasticsearch. | | `:kubernetes` | The test includes a GitLab instance that is configured to be run behind an SSH tunnel, allowing a TLS-accessible GitLab. This test will also include provisioning of at least one Kubernetes cluster to test against. *This tag is often be paired with `:orchestrated`.* | | `:orchestrated` | The GitLab instance under test may be [configured by `gitlab-qa`](https://gitlab.com/gitlab-org/gitlab-qa/-/blob/master/docs/what_tests_can_be_run.md#orchestrated-tests) to be different to the default GitLab configuration, or `gitlab-qa` may launch additional services in separate Docker containers, or both. Tests tagged with `:orchestrated` are excluded when testing environments where we can't dynamically modify GitLab's configuration (for example, Staging). | -| `:quarantine` | The test has been [quarantined](https://about.gitlab.com/handbook/engineering/quality/guidelines/debugging-qa-test-failures/#quarantining-tests), will run in a separate job that only includes quarantined tests, and is allowed to fail. The test will be skipped in its regular job so that if it fails it will not hold up the pipeline. | +| `:quarantine` | The test has been [quarantined](https://about.gitlab.com/handbook/engineering/quality/guidelines/debugging-qa-test-failures/#quarantining-tests), will run in a separate job that only includes quarantined tests, and is allowed to fail. The test will be skipped in its regular job so that if it fails it will not hold up the pipeline. Note that you can also [quarantine a test only when it runs against specific environment](environment_selection.md#quarantining-a-test-for-a-specific-environment). | | `:reliable` | The test has been [promoted to a reliable test](https://about.gitlab.com/handbook/engineering/quality/guidelines/reliable-tests/#promoting-an-existing-test-to-reliable) meaning it passes consistently in all pipelines, including merge requests. | | `:requires_admin` | The test requires an admin account. Tests with the tag are excluded when run against Canary and Production environments. | | `:runner` | The test depends on and will set up a GitLab Runner instance, typically to run a pipeline. | | `:gitaly_ha` | The test will run against a GitLab instance where repositories are stored on redundant Gitaly nodes behind a Praefect node. All nodes are [separate containers](../../../administration/gitaly/praefect.md#requirements-for-configuring-a-gitaly-cluster). Tests that use this tag have a longer setup time since there are three additional containers that need to be started. | | `:skip_live_env` | The test will be excluded when run against live deployed environments such as Staging, Canary, and Production. | | `:jira` | The test requires a Jira Server. [GitLab-QA](https://gitlab.com/gitlab-org/gitlab-qa) will provision the Jira Server in a Docker container when the `Test::Integration::Jira` test scenario is run. +| `:only` | The test is only to be run against specific environments. See [Environment selection](environment_selection.md) for more information. | diff --git a/doc/development/testing_guide/frontend_testing.md b/doc/development/testing_guide/frontend_testing.md index 37e1066e7aa..ef9fd748dbb 100644 --- a/doc/development/testing_guide/frontend_testing.md +++ b/doc/development/testing_guide/frontend_testing.md @@ -545,6 +545,99 @@ In order to ensure that a clean wrapper object and DOM are being used in each te See also the [Vue Test Utils documentation on `destroy`](https://vue-test-utils.vuejs.org/api/wrapper/#destroy). +### Jest best practices + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/34209) in GitLab 13.2. + +#### Prefer `toBe` over `toEqual` when comparing primitive values + +Jest has [`toBe`](https://jestjs.io/docs/en/expect#tobevalue) and +[`toEqual`](https://jestjs.io/docs/en/expect#toequalvalue) matchers. +As [`toBe`](https://jestjs.io/docs/en/expect#tobevalue) uses +[`Object.is`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object/is) +to compare values, it's faster (by default) than using `toEqual`. +While the latter will eventually fallback to leverage [`Object.is`](https://github.com/facebook/jest/blob/master/packages/expect/src/jasmineUtils.ts#L91), +for primitive values, it should only be used when complex objects need a comparison. + +Examples: + +```javascript +const foo = 1; + +// good +expect(foo).toBe(1); + +// bad +expect(foo).toEqual(1); +``` + +#### Prefer more befitting matchers + +Jest provides useful matchers like `toHaveLength` or `toBeUndefined` to make your tests more +readable and to produce more understandable error messages. Check their docs for the +[full list of matchers](https://jestjs.io/docs/en/expect#methods). + +Examples: + +```javascript +const arr = [1, 2]; + +// prints: +// Expected length: 1 +// Received length: 2 +expect(arr).toHaveLength(1); + +// prints: +// Expected: 1 +// Received: 2 +expect(arr.length).toBe(1); + +// prints: +// expect(received).toBe(expected) // Object.is equality +// Expected: undefined +// Received: "bar" +const foo = 'bar'; +expect(foo).toBe(undefined); + +// prints: +// expect(received).toBeUndefined() +// Received: "bar" +const foo = 'bar'; +expect(foo).toBeUndefined(); +``` + +#### Avoid using `toBeTruthy` or `toBeFalsy` + +Jest also provides following matchers: `toBeTruthy` and `toBeFalsy`. We should not use them because +they make tests weaker and produce false-positive results. + +For example, `expect(someBoolean).toBeFalsy()` passes when `someBoolean === null`, and when +`someBoolean === false`. + +#### Tricky `toBeDefined` matcher + +Jest has the tricky `toBeDefined` matcher that can produce false positive test. Because it +[validates](https://github.com/facebook/jest/blob/master/packages/expect/src/matchers.ts#L204) +the given value for `undefined` only. + +```javascript +// good +expect(wrapper.find('foo').exists()).toBe(true); + +// bad +// if finder returns null, the test will pass +expect(wrapper.find('foo')).toBeDefined(); +``` + +#### Avoid using `setImmediate` + +Try to avoid using `setImmediate`. `setImmediate` is an ad-hoc solution to run your callback after +the I/O completes. And it's not part of the Web API, hence, we target NodeJS environments in our +unit tests. + +Instead of `setImmediate`, use `jest.runAllTimers` or `jest.runOnlyPendingTimers` to run pending timers. +The latter is useful when you have `setInterval` in the code. **Remember:** our Jest configuration uses fake timers. + ## Factories TBU @@ -856,7 +949,8 @@ Some regressions only affect a specific browser version. We can install and test [BrowserStack](https://www.browserstack.com/) allows you to test more than 1200 mobile devices and browsers. You can use it directly through the [live app](https://www.browserstack.com/live) or you can install the [chrome extension](https://chrome.google.com/webstore/detail/browserstack/nkihdmlheodkdfojglpcjjmioefjahjb) for easy access. -You can find the credentials on 1Password, under `frontendteam@gitlab.com`. +Sign in to BrowserStack with the credentials saved in the **Engineering** vault of GitLab's +[shared 1Password account](https://about.gitlab.com/handbook/security/#1password-guide). ### Firefox diff --git a/doc/development/testing_guide/review_apps.md b/doc/development/testing_guide/review_apps.md index 3d7aea89e73..54f8ca0d98b 100644 --- a/doc/development/testing_guide/review_apps.md +++ b/doc/development/testing_guide/review_apps.md @@ -9,7 +9,7 @@ pipeline](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/6665). ```mermaid graph TD - A["build-qa-image, gitlab:assets:compile pull-cache<br/>(canonical default refs only)"]; + A["build-qa-image, compile-production-assets<br/>(canonical default refs only)"]; B[review-build-cng]; C[review-deploy]; D[CNG-mirror]; @@ -30,7 +30,7 @@ subgraph "2. gitlab `review-prepare` stage" end subgraph "3. gitlab `review` stage" - C["review-deploy<br><br>Helm deploys the Review App using the Cloud<br/>Native images built by the CNG-mirror pipeline.<br><br>Cloud Native images are deployed to the `review-apps-ce` or `review-apps-ee`<br>Kubernetes (GKE) cluster, in the GCP `gitlab-review-apps` project."] + C["review-deploy<br><br>Helm deploys the Review App using the Cloud<br/>Native images built by the CNG-mirror pipeline.<br><br>Cloud Native images are deployed to the `review-apps`<br>Kubernetes (GKE) cluster, in the GCP `gitlab-review-apps` project."] end subgraph "4. gitlab `qa` stage" @@ -44,25 +44,27 @@ subgraph "CNG-mirror pipeline" ### Detailed explanation -1. On every [pipeline](https://gitlab.com/gitlab-org/gitlab/pipelines/125315730) during the `test` stage, the - [`gitlab:assets:compile`](https://gitlab.com/gitlab-org/gitlab/-/jobs/467724487) job is automatically started. - - Once it's done, it starts the [`review-build-cng`](https://gitlab.com/gitlab-org/gitlab/-/jobs/467724808) - manual job since the [`CNG-mirror`](https://gitlab.com/gitlab-org/build/CNG-mirror) pipeline triggered in the +1. On every [pipeline](https://gitlab.com/gitlab-org/gitlab/pipelines/125315730) during the `prepare` stage, the + [`compile-production-assets`](https://gitlab.com/gitlab-org/gitlab/-/jobs/641770154) job is automatically started. + - Once it's done, the [`review-build-cng`](https://gitlab.com/gitlab-org/gitlab/-/jobs/467724808) + job starts since the [`CNG-mirror`](https://gitlab.com/gitlab-org/build/CNG-mirror) pipeline triggered in the following step depends on it. -1. The [`review-build-cng`](https://gitlab.com/gitlab-org/gitlab/-/jobs/467724808) job [triggers a pipeline](https://gitlab.com/gitlab-org/build/CNG-mirror/pipelines/44364657) +1. Once `compile-production-assets` is done, the [`review-build-cng`](https://gitlab.com/gitlab-org/gitlab/-/jobs/467724808) + job [triggers a pipeline](https://gitlab.com/gitlab-org/build/CNG-mirror/pipelines/44364657) in the [`CNG-mirror`](https://gitlab.com/gitlab-org/build/CNG-mirror) project. + - The `review-build-cng` job automatically starts only if your MR includes + [CI or frontend changes](../pipelines.md#changes-patterns). In other cases, the job is manual. - The [`CNG-mirror`](https://gitlab.com/gitlab-org/build/CNG-mirror/pipelines/44364657) pipeline creates the Docker images of each component (e.g. `gitlab-rails-ee`, `gitlab-shell`, `gitaly` etc.) based on the commit from the [GitLab pipeline](https://gitlab.com/gitlab-org/gitlab/pipelines/125315730) and stores them in its [registry](https://gitlab.com/gitlab-org/build/CNG-mirror/container_registry). - We use the [`CNG-mirror`](https://gitlab.com/gitlab-org/build/CNG-mirror) project so that the `CNG`, (Cloud - Native GitLab), project's registry is not overloaded with a - lot of transient Docker images. + Native GitLab), project's registry is not overloaded with a lot of transient Docker images. - Note that the official CNG images are built by the `cloud-native-image` job, which runs only for tags, and triggers itself a [`CNG`](https://gitlab.com/gitlab-org/build/CNG) pipeline. -1. Once the `test` stage is done, the [`review-deploy`](https://gitlab.com/gitlab-org/gitlab/-/jobs/467724810) job +1. Once `review-build-cng` is done, the [`review-deploy`](https://gitlab.com/gitlab-org/gitlab/-/jobs/467724810) job deploys the Review App using [the official GitLab Helm chart](https://gitlab.com/gitlab-org/charts/gitlab/) to - the [`review-apps-ce`](https://console.cloud.google.com/kubernetes/clusters/details/us-central1-a/review-apps-ce?project=gitlab-review-apps) / [`review-apps-ee`](https://console.cloud.google.com/kubernetes/clusters/details/us-central1-b/review-apps-ee?project=gitlab-review-apps) + the [`review-apps`](https://console.cloud.google.com/kubernetes/clusters/details/us-central1-b/review-apps?project=gitlab-review-apps) Kubernetes cluster on GCP. - The actual scripts used to deploy the Review App can be found at [`scripts/review_apps/review-apps.sh`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/scripts/review_apps/review-apps.sh). @@ -94,10 +96,9 @@ subgraph "CNG-mirror pipeline" - The manual `review-stop` can be used to stop a Review App manually, and is also started by GitLab once a merge request's branch is deleted after being merged. -- The Kubernetes cluster is connected to the `gitlab-{ce,ee}` projects using +- The Kubernetes cluster is connected to the `gitlab` projects using [GitLab's Kubernetes integration](../../user/project/clusters/index.md). This basically - allows to have a link to the Review App directly from the merge request - widget. + allows to have a link to the Review App directly from the merge request widget. ### Auto-stopping of Review Apps @@ -136,11 +137,10 @@ browser performance testing using a ### Node pools -The `review-apps-ee` and `review-apps-ce` clusters are currently set up with +The `review-apps` cluster is currently set up with the following node pools: -- `review-apps-ee` of pre-emptible `e2-highcpu-16` (16 vCPU, 16 GB memory) nodes with autoscaling -- `review-apps-ce` of pre-emptible `n1-standard-8` (8 vCPU, 16 GB memory) nodes with autoscaling +- `e2-highcpu-16` (16 vCPU, 16 GB memory) pre-emptible nodes with autoscaling ### Helm @@ -189,9 +189,7 @@ secure note named `gitlab-{ce,ee} Review App's root password`. 1. Click on the `KUBECTL` dropdown, then `Exec` -> `task-runner`. 1. Replace `-c task-runner -- ls` with `-it -- gitlab-rails console` from the default command or - - Run `kubectl exec --namespace review-apps-ce review-qa-raise-e-12chm0-task-runner-d5455cc8-2lsvz -it -- gitlab-rails console` and - - Replace `review-apps-ce` with `review-apps-ee` if the Review App - is running EE, and + - Run `kubectl exec --namespace review-apps review-qa-raise-e-12chm0-task-runner-d5455cc8-2lsvz -it -- gitlab-rails console` and - Replace `review-qa-raise-e-12chm0-task-runner-d5455cc8-2lsvz` with your Pod's name. diff --git a/doc/development/testing_guide/testing_levels.md b/doc/development/testing_guide/testing_levels.md index 2210ec94696..88d7cf9ca08 100644 --- a/doc/development/testing_guide/testing_levels.md +++ b/doc/development/testing_guide/testing_levels.md @@ -482,7 +482,7 @@ Every new feature should come with a [test plan](https://gitlab.com/gitlab-org/g | Tests path | Testing engine | Notes | | ---------- | -------------- | ----- | -| `qa/qa/specs/features/` | [Capybara](https://github.com/teamcapybara/capybara) + [RSpec](https://github.com/rspec/rspec-rails#feature-specs) + Custom QA framework | Tests should be placed under their corresponding [Product category](https://about.gitlab.com/handbook/product/categories/) | +| `qa/qa/specs/features/` | [Capybara](https://github.com/teamcapybara/capybara) + [RSpec](https://github.com/rspec/rspec-rails#feature-specs) + Custom QA framework | Tests should be placed under their corresponding [Product category](https://about.gitlab.com/handbook/product/product-categories/) | > See [end-to-end tests](end_to_end/index.md) for more information. diff --git a/doc/development/testing_guide/testing_migrations_guide.md b/doc/development/testing_guide/testing_migrations_guide.md index a03b940fe40..8ee758177c3 100644 --- a/doc/development/testing_guide/testing_migrations_guide.md +++ b/doc/development/testing_guide/testing_migrations_guide.md @@ -112,7 +112,7 @@ migration. You can find the complete spec in require 'spec_helper' require Rails.root.join('db', 'post_migrate', '20170526185842_migrate_pipeline_stages.rb') -describe MigratePipelineStages do +RSpec.describe MigratePipelineStages do # Create test data - pipeline and CI/CD jobs. let(:jobs) { table(:ci_builds) } let(:stages) { table(:ci_stages) } diff --git a/doc/development/what_requires_downtime.md b/doc/development/what_requires_downtime.md index 407899b23d5..d0b8aa18f5c 100644 --- a/doc/development/what_requires_downtime.md +++ b/doc/development/what_requires_downtime.md @@ -130,7 +130,8 @@ class CleanupUsersUpdatedAtRename < ActiveRecord::Migration[4.2] end ``` -NOTE: **Note:** If you're renaming a [large table](https://gitlab.com/gitlab-org/gitlab/-/blob/master/rubocop/rubocop-migrations.yml#L3), please carefully consider the state when the first migration has run but the second cleanup migration hasn't been run yet. +NOTE: **Note:** +If you're renaming a [large table](https://gitlab.com/gitlab-org/gitlab/-/blob/master/rubocop/rubocop-migrations.yml#L3), please carefully consider the state when the first migration has run but the second cleanup migration hasn't been run yet. With [Canary](https://about.gitlab.com/handbook/engineering/infrastructure/library/canary/) it is possible that the system runs in this state for a significant amount of time. ## Changing Column Constraints @@ -202,6 +203,21 @@ end And that's it, we're done! +### Casting data to a new type + +Some type changes require casting data to a new type. For example when changing from `text` to `jsonb`. +In this case, use the `type_cast_function` option. +Make sure there is no bad data and the cast will always succeed. You can also provide a custom function that handles +casting errors. + +Example migration: + +```ruby + def up + change_column_type_concurrently :users, :settings, :jsonb, type_cast_function: 'jsonb' + end +``` + ## Changing The Schema For Large Tables While `change_column_type_concurrently` and `rename_column_concurrently` can be @@ -317,30 +333,11 @@ migrations](background_migrations.md#cleaning-up). ## Adding Indexes -Adding indexes is an expensive process that blocks INSERT and UPDATE queries for -the duration. You can work around this by using the `CONCURRENTLY` option: - -```sql -CREATE INDEX CONCURRENTLY index_name ON projects (column_name); -``` - -Migrations can take advantage of this by using the method -`add_concurrent_index`. For example: - -```ruby -class MyMigration < ActiveRecord::Migration[4.2] - def up - add_concurrent_index :projects, :column_name - end - - def down - remove_index(:projects, :column_name) if index_exists?(:projects, :column_name) - end -end -``` +Adding indexes does not require downtime when `add_concurrent_index` +is used. -Note that `add_concurrent_index` can not be reversed automatically, thus you -need to manually define `up` and `down`. +See also [Migration Style Guide](migration_style_guide.md#adding-indexes) +for more information. ## Dropping Indexes diff --git a/doc/gitlab-basics/start-using-git.md b/doc/gitlab-basics/start-using-git.md index 439032f39b3..d767bee4cf1 100644 --- a/doc/gitlab-basics/start-using-git.md +++ b/doc/gitlab-basics/start-using-git.md @@ -1,7 +1,7 @@ --- type: howto, tutorial description: "Introduction to using Git through the command line." -last_updated: 2020-04-22 +last_updated: 2020-06-30 --- # Start using Git on the command line @@ -104,68 +104,134 @@ repository. You can read more on how Git manages configurations in the [Git Config](https://git-scm.com/book/en/v2/Customizing-Git-Git-Configuration) documentation. -## Basic Git commands +## Git authentication methods -Start using Git via the command line with the most basic commands as described below. +To connect your computer with GitLab, you need to add your credentials to identify yourself. +You have two options: -### Initialize a local directory for Git version control +- Authenticate on a project-by-project basis through HTTPS, and enter your credentials every time + you perform an operation between your computer and GitLab. +- Authenticate through SSH once and GitLab won't ask your credentials every time you pull, push, + and clone. -If you have an existing local directory that you want to *initialize* for version -control, use the `init` command to instruct Git to begin tracking the directory: +To start the authentication process, we'll [clone](#clone-a-repository) an existing repository +to our computer: -```shell -git init -``` +- If you want to use **SSH** to authenticate, follow the instructions on the [SSH documentation](../ssh/README.md) + to set it up before cloning. +- If you want to use **HTTPS**, GitLab will request your user name and password: + - If you have 2FA enabled for your account, you'll have to use a [Personal Access Token](../user/profile/personal_access_tokens.md) + with **read_repository** or **write_repository** permissions instead of your account's password. + Create one before cloning. + - If you don't have 2FA enabled, use your account's password. -This creates a `.git` directory that contains the Git configuration files. +NOTE: **Note:** +Authenticating via SSH is GitLab's recommended method. You can read more about credential storage +in the [Git Credentials documentation](https://git-scm.com/book/en/v2/Git-Tools-Credential-Storage). -Once the directory has been initialized, you can [add a remote repository](#add-a-remote-repository) -and [send changes to GitLab.com](#send-changes-to-gitlabcom). You will also need to -[create a new project in GitLab](../gitlab-basics/create-project.md#push-to-create-a-new-project) -for your Git repository. +## Git terminology -### Clone a repository +If you're familiar with the Git terminology, you may want to jump directly +into the [basic commands](#basic-git-commands). + +### Namespace + +A **namespace** is either a **user name** or a **group name**. + +For example, suppose Jo is a GitLab.com user and they chose their user name as +`jo`. You can see Jo's profile at `https://gitlab.com/jo`. `jo` is a namespace. + +Jo also created a group in GitLab, and chose the path `test-group` for their +group. The group can be accessed under `https://gitlab.com/test-group`. `test-group` is a namespace. + +### Repository + +Your files in GitLab live in a **repository**, similar to how you have them in a folder or +directory in your computer. **Remote** repository refers to the files in +GitLab and the copy in your computer is called **local** copy. +A **project** in GitLab is what holds a repository, which holds your files. +Often, the word "repository" is shortened to "repo". + +### Fork + +When you want to copy someone else's repository, you [**fork**](../user/project/repository/forking_workflow.md#creating-a-fork) +the project. By forking it, you'll create a copy of the project into your own +namespace to have read and write permissions to modify the project files +and settings. + +For example, if you fork this project, <https://gitlab.com/gitlab-tests/sample-project/> into your namespace, you'll create your own copy of the repository in your namespace (`https://gitlab.com/your-namespace/sample-project/`). From there, you can clone it into your computer, +work on its files, and (optionally) submit proposed changes back to the +original repository if you'd like. + +### Download vs clone -To start working locally on an existing remote repository, clone it with the command -`git clone <repository path>`. By cloning a repository, you'll download a copy of its -files to your local computer, automatically preserving the Git connection with the -remote repository. +To create a copy of a remote repository files on your computer, you can either +**download** or **clone** it. If you download it, you cannot sync it with the +remote repository on GitLab. -You can either clone it via [HTTPS](#clone-via-https) or [SSH](#clone-via-ssh). If you chose to -clone it via HTTPS, you'll have to enter your credentials every time you pull and push. -You can read more about credential storage in the -[Git Credentials documentation](https://git-scm.com/book/en/v2/Git-Tools-Credential-Storage). -With [SSH](../ssh/README.md), you enter your credentials only once. +On the other hand, by cloning a repository, you'll download a copy of its +files to your local computer, but preserve the Git connection with the remote +repository, so that you can work on the its files on your computer and then +upload the changes to GitLab. + +### Pull and push + +After you saved a local copy of a repository and modified its files on your computer, you can upload the +changes to GitLab. This is referred to as **pushing** to GitLab, as this is achieved by the command +[`git push`](#send-changes-to-gitlabcom). + +When the remote repository changes, your local copy will be behind it. You can update it with the new +changes in the remote repo. +This is referred to as **pulling** from GitLab, as this is achieved by the command +[`git pull`](#download-the-latest-changes-in-the-project). + +## Basic Git commands + +For the purposes of this guide, we will use this example project on GitLab.com: +[https://gitlab.com/gitlab-tests/sample-project/](https://gitlab.com/gitlab-tests/sample-project/). + +To use it, log into GitLab.com and fork the example project into your +namespace to have your own copy to playing with. Your sample +project will be available under `https://gitlab.com/<your-namespace>/sample-project/`. + +You can also choose any other project to follow this guide. Then, replace the +example URLs with your own project's. + +If you want to start by copying an existing GitLab repository onto your +computer, see how to [clone a repository](#clone-a-repository). On the other +hand, if you want to start by uploading an existing folder from your computer +to GitLab, see how to [convert a local folder into a Git repository](#convert-a-local-directory-into-a-repository). + +### Clone a repository + +To start working locally on an existing remote repository, clone it with the +command `git clone <repository path>`. You can either clone it via [HTTPS](#clone-via-https) or [SSH](#clone-via-ssh), according to your preferred [authentication method](#git-authentication-methods). You can find both paths (HTTPS and SSH) by navigating to your project's landing page and clicking **Clone**. GitLab will prompt you with both paths, from which you can copy and paste in your command line. -As an example, consider this repository path: +For example, considering our [sample project](https://gitlab.com/gitlab-tests/sample-project/): -- HTTPS: `https://gitlab.com/gitlab-org/gitlab.git` -- SSH: `git@gitlab.com:gitlab-org/gitlab.git` +- To clone through HTTPS, use `https://gitlab.com/gitlab-tests/sample-project.git`. +- To clone through SSH, use `git@gitlab.com:gitlab-tests/sample-project.git`. -To get started, open a terminal window in the directory you wish to clone the +To get started, open a terminal window in the directory you wish to add the repository files into, and run one of the `git clone` commands as described below. Both commands will download a copy of the files in a folder named after the project's -name. You can then navigate to the new directory and start working on it locally. +name and preserve the connection with the remote repository. +You can then navigate to the new directory with `cd sample-project` and start working on it +locally. #### Clone via HTTPS -To clone `https://gitlab.com/gitlab-org/gitlab.git` via HTTPS: +To clone `https://gitlab.com/gitlab-tests/sample-project/` via HTTPS: ```shell -git clone https://gitlab.com/gitlab-org/gitlab.git +git clone https://gitlab.com/gitlab-tests/sample-project.git ``` -You'll have to add your password every time you clone through HTTPS. If you have 2FA enabled -for your account, you'll have to use a [Personal Access Token](../user/profile/personal_access_tokens.md) -with **read_repository** or **write_repository** permissions instead of your account's password. - -If you don't have 2FA enabled, use your account's password. - TIP: **Troubleshooting:** On Windows, if you entered incorrect passwords multiple times and GitLab is responding `Access denied`, you may have to add your namespace (user name or group name) to clone through HTTPS: @@ -179,16 +245,44 @@ To clone `git@gitlab.com:gitlab-org/gitlab.git` via SSH: git clone git@gitlab.com:gitlab-org/gitlab.git ``` -### Switch to the master branch +### Convert a local directory into a repository -You are always in a branch when working with Git. The main branch is the master -branch, but you can use the same command to switch to a different branch by -changing `master` to the branch name. +When you have your files in a local folder and want to convert it into +a repository, you'll need to _initialize_ the folder through the `git init` +command. This will instruct Git to begin to track that directory as a +repository. To do so, open the terminal on the directory you'd like to convert +and run: ```shell -git checkout master +git init ``` +This command creates a `.git` folder in your directory that contains Git +records and configuration files. We advise against editing these files +directly. + +Then, on the next step, add the [path to your remote repository](#add-a-remote-repository) +so that Git can upload your files into the correct project. + +#### Add a remote repository + +By "adding a remote repository" to your local directory you'll tell Git that +the path to that specific project in GitLab corresponds to that specific +folder you have in your computer. This way, your local folder will be +identified by Git as the local content for that specific remote project. + +To add a remote repository to your local copy: + +1. In GitLab, [create a new project](../gitlab-basics/create-project.md#push-to-create-a-new-project) to hold your files. +1. Visit this project's homepage, scroll down to **Push an existing folder**, and copy the command that starts with `git remote add`. +1. On your computer, open the terminal in the directory you've initialized, paste the command you copied, and press <kbd>enter</kbd>: + + ```shell + git remote add origin <git@gitlab.com:username/projectpath.git + ``` + +After you've done that, you can [stage your files](#add-and-commit-local-changes) and [upload them to GitLab](#send-changes-to-gitlabcom). + ### Download the latest changes in the project To work on an up-to-date copy of the project (it is important to do this every time @@ -219,16 +313,16 @@ git remote -v The `-v` flag stands for verbose. -### Add a remote repository +## Branching -To add a link to a remote repository: +If you want to add code to a project but you're not sure if it will work properly, or you're +collaborating on the project with others, and don't want your work to get mixed up, it's a good idea +to work on a different **branch**. -```shell -git remote add <source-name> <repository-path> -``` - -You'll use this source name every time you [push changes to GitLab.com](#send-changes-to-gitlabcom), -so use something easy to remember and type. +When you create a branch in a Git repository, you make a copy of its files at the time of branching. You're free +to do whatever you want with the code in your branch without impacting the main branch or other branches. And when +you're ready to bring your changes to the main codebase, you can merge your branch into the main one +used in your project (such as `master`). ### Create a branch @@ -240,6 +334,16 @@ use a hyphen or underscore): git checkout -b <name-of-branch> ``` +### Switch to the master branch + +You are always in a branch when working with Git. The main branch is the master +branch, but you can use the same command to switch to a different branch by +changing `master` to the branch name. + +```shell +git checkout master +``` + ### Work on an existing branch To switch to an existing branch, so you can work on it: @@ -279,7 +383,7 @@ git add <file-name OR folder-name> git commit -m "COMMENT TO DESCRIBE THE INTENTION OF THE COMMIT" ``` -### Add all changes to commit +#### Add all changes to commit To add and commit (save) all local changes quickly: @@ -293,10 +397,6 @@ The `.` character means _all file changes in the current directory and all subdi ### Send changes to GitLab.com -NOTE: **Note:** -To create a merge request from a fork to an upstream repository, see the -[forking workflow](../user/project/repository/forking_workflow.md) - To push all local commits (saved changes) to the remote repository: ```shell @@ -309,6 +409,10 @@ For example, to push your local commits to the _`master`_ branch of the _`origin git push origin master ``` +NOTE: **Note:** +To create a merge request from a fork to an upstream repository, see the +[forking workflow](../user/project/repository/forking_workflow.md). + ### Delete all changes in the branch To delete all local changes in the branch that have not been added to the staging @@ -353,7 +457,7 @@ git checkout <name-of-branch> git merge master ``` -### Synchronize changes in a forked repository with the upstream +## Synchronize changes in a forked repository with the upstream [Forking a repository](../user/project/repository/forking_workflow.md) lets you create a copy of a repository in your namespace. Changes made to your copy of the repository diff --git a/doc/install/aws/index.md b/doc/install/aws/index.md index 813e343f2cc..35c046423b0 100644 --- a/doc/install/aws/index.md +++ b/doc/install/aws/index.md @@ -7,8 +7,8 @@ type: howto This page offers a walkthrough of a common configuration for GitLab on AWS. You should customize it to accommodate your needs. -NOTE: **Note** -For organizations with 300 users or less, the recommended AWS installation method is to launch an EC2 single box [Omnibus Installation](https://about.gitlab.com/install/) and implement a snapshot strategy for backing up the data. +NOTE: **Note:** +For organizations with 1,000 users or less, the recommended AWS installation method is to launch an EC2 single box [Omnibus Installation](https://about.gitlab.com/install/) and implement a snapshot strategy for backing up the data. See the [1,000 user reference architecture](../../administration/reference_architectures/1k_users.md) for more. ## Introduction @@ -30,7 +30,8 @@ In addition to having a basic familiarity with [AWS](https://docs.aws.amazon.com - A domain name for the GitLab instance - An SSL/TLS certificate to secure your domain. If you do not already own one, you can provision a free public SSL/TLS certificate through [AWS Certificate Manager](https://aws.amazon.com/certificate-manager/)(ACM) for use with the [Elastic Load Balancer](#load-balancer) we'll create. -NOTE: **Note:** It can take a few hours to validate a certificate provisioned through ACM. To avoid delays later, request your certificate as soon as possible. +NOTE: **Note:** +It can take a few hours to validate a certificate provisioned through ACM. To avoid delays later, request your certificate as soon as possible. ## Architecture @@ -230,7 +231,10 @@ On the EC2 dashboard, look for Load Balancer in the left navigation bar: 1. Click the **Create Load Balancer** button. 1. Choose the **Classic Load Balancer**. 1. Give it a name (we'll use `gitlab-loadbalancer`) and for the **Create LB Inside** option, select `gitlab-vpc` from the dropdown menu. - 1. In the **Listeners** section, set HTTP port 80, HTTPS port 443, and TCP port 22 for both load balancer and instance protocols and ports. + 1. In the **Listeners** section, set the following listeners: + - HTTP port 80 for both load balancer and instance protocol and ports + - TCP port 22 for both load balancer and instance protocols and ports + - HTTPS port 443 for load balancer protocol and ports, forwarding to HTTP port 80 on the instance (we will configure GitLab to listen on port 80 [later in the guide](#add-support-for-proxied-ssl)) 1. In the **Select Subnets** section, select both public subnets from the list so that the load balancer can route traffic to both availability zones. 1. We'll add a security group for our load balancer to act as a firewall to control what traffic is allowed through. Click **Assign Security Groups** and select **Create a new security group**, give it a name (we'll use `gitlab-loadbalancer-sec-group`) and description, and allow both HTTP and HTTPS traffic @@ -244,8 +248,7 @@ On the EC2 dashboard, look for Load Balancer in the left navigation bar: 1. For **Ping Path**, enter `/users/sign_in`. (We use `/users/sign_in` as it's a public endpoint that does not require authorization.) 1. Keep the default **Advanced Details** or adjust them according to your needs. -1. Click **Add EC2 Instances** but, as we don't have any instances to add yet, come back -to your load balancer after creating your GitLab instances and add them. +1. Click **Add EC2 Instances** - don't add anything as we will create an Auto Scaling Group later to manage instances for us. 1. Click **Add Tags** and add any tags you need. 1. Click **Review and Create**, review all your settings, and click **Create** if you're happy. @@ -302,7 +305,8 @@ We need a security group for our database that will allow inbound traffic from t ### Create the database -DANGER: **Danger:** Avoid using burstable instances (t class instances) for the database as this could lead to performance issues due to CPU credits running out during sustained periods of high load. +DANGER: **Danger:** +Avoid using burstable instances (t class instances) for the database as this could lead to performance issues due to CPU credits running out during sustained periods of high load. Now, it's time to create the database: @@ -360,7 +364,7 @@ persistence and is used to store session data, temporary cache information, and 1. Navigate back to the ElastiCache dashboard. 1. Select **Redis** on the left menu and click **Create** to create a new - Redis cluster. Do not enable **Cluster Mode** as it is [not supported](../../administration/high_availability/redis.md#provide-your-own-redis-instance-core-only). Even without cluster mode on, you still get the + Redis cluster. Do not enable **Cluster Mode** as it is [not supported](../../administration/redis/replication_and_failover_external.md#requirements). Even without cluster mode on, you still get the chance to deploy Redis in multiple availability zones. 1. In the settings section: 1. Give the cluster a name (`gitlab-redis`) and a description. @@ -384,7 +388,8 @@ persistence and is used to store session data, temporary cache information, and Since our GitLab instances will be in private subnets, we need a way to connect to these instances via SSH to make configuration changes, perform upgrades, etc. One way of doing this is via a [bastion host](https://en.wikipedia.org/wiki/Bastion_host), sometimes also referred to as a jump box. -TIP: **Tip:** If you do not want to maintain bastion hosts, you can set up [AWS Systems Manager Session Manager](https://docs.aws.amazon.com/systems-manager/latest/userguide/session-manager.html) for access to instances. This is beyond the scope of this document. +TIP: **Tip:** +If you do not want to maintain bastion hosts, you can set up [AWS Systems Manager Session Manager](https://docs.aws.amazon.com/systems-manager/latest/userguide/session-manager.html) for access to instances. This is beyond the scope of this document. ### Create Bastion Host A @@ -540,7 +545,9 @@ gitlab=# \q #### Set up Gitaly -CAUTION: **Caution:** In this architecture, having a single Gitaly server creates a single point of failure. This limitation will be removed once [Gitaly Cluster](https://gitlab.com/groups/gitlab-org/-/epics/1489) is released. +CAUTION: **Caution:** +In this architecture, having a single Gitaly server creates a single point of failure. Use +[Gitaly Cluster](../../administration/gitaly/praefect.md) to remove this limitation. Gitaly is a service that provides high-level RPC access to Git repositories. It should be enabled and configured on a separate EC2 instance in one of the @@ -566,7 +573,8 @@ Let's create an EC2 instance where we'll install Gitaly: 1. Click **Review and launch** followed by **Launch** if you're happy with your settings. 1. Finally, acknowledge that you have access to the selected private key file or create a new one. Click **Launch Instances**. -NOTE: **Optional:** Instead of storing configuration _and_ repository data on the root volume, you can also choose to add an additional EBS volume for repository storage. Follow the same guidance as above. See the [Amazon EBS pricing](https://aws.amazon.com/ebs/pricing/). We do not recommend using EFS as it may negatively impact GitLab’s performance. You can review the [relevant documentation](../../administration/high_availability/nfs.md#avoid-using-awss-elastic-file-system-efs) for more details. +NOTE: **Note:** +Instead of storing configuration _and_ repository data on the root volume, you can also choose to add an additional EBS volume for repository storage. Follow the same guidance as above. See the [Amazon EBS pricing](https://aws.amazon.com/ebs/pricing/). We do not recommend using EFS as it may negatively impact GitLab’s performance. You can review the [relevant documentation](../../administration/high_availability/nfs.md#avoid-using-awss-elastic-file-system-efs) for more details. Now that we have our EC2 instance ready, follow the [documentation to install GitLab and set up Gitaly on its own server](../../administration/gitaly/index.md#run-gitaly-on-its-own-server). Perform the client setup steps from that document on the [GitLab instance we created](#install-gitlab) above. @@ -638,7 +646,7 @@ That concludes the configuration changes for our GitLab instance. Next, we'll cr On the EC2 dashboard: -1. Select the `GitLab` instance we [created earlier](#install-gitLab). +1. Select the `GitLab` instance we [created earlier](#install-gitlab). 1. Click on **Actions**, scroll down to **Image** and click **Create Image**. 1. Give your image a name and description (we'll use `GitLab-Source` for both). 1. Leave everything else as default and click **Create Image** @@ -737,7 +745,7 @@ To back up GitLab: sudo gitlab-backup create ``` -NOTE: **Note** +NOTE: **Note:** For GitLab 12.1 and earlier, use `gitlab-rake gitlab:backup:create`. ### Restoring GitLab from a backup @@ -758,7 +766,7 @@ released, you can update your GitLab instance: sudo gitlab-backup create ``` -NOTE: **Note** +NOTE: **Note:** For GitLab 12.1 and earlier, use `gitlab-rake gitlab:backup:create`. 1. Update the repositories and install GitLab: @@ -794,14 +802,14 @@ to request additional material: Activate all GitLab Enterprise Edition functionality with a license. - [Pricing](https://about.gitlab.com/pricing/): Pricing for the different tiers. -<!-- ## Troubleshooting +## Troubleshooting + +### Instances are failing health checks + +If your instances are failing the load balancer's health checks, verify that they are returning a status `200` from the health check endpoint we configured earlier. Any other status, including redirects (e.g. status `302`) will cause the health check to fail. + +You may have to set a password on the `root` user to prevent automatic redirects on the sign-in endpoint before health checks will pass. -Include any troubleshooting steps that you can foresee. If you know beforehand what issues -one might have when setting this up, or when something is changed, or on upgrading, it's -important to describe those, too. Think of things that may go wrong and include them here. -This is important to minimize requests for support and to avoid doc comments with -questions that you know someone might ask. +### "The change you requested was rejected (422)" -Each scenario can be a third-level heading, e.g. `### Getting error message X`. -If you have none to add when creating a doc, leave this section in place -but commented out to help encourage others to add to it in the future. --> +If you see this page when trying to set a password via the web interface, make sure `external_url` in `gitlab.rb` matches the domain you are making a request from, and run `sudo gitlab-ctl reconfigure` after making any changes to it. diff --git a/doc/install/installation.md b/doc/install/installation.md index 0997062006d..8b285e0c9f1 100644 --- a/doc/install/installation.md +++ b/doc/install/installation.md @@ -310,8 +310,7 @@ sudo adduser --disabled-login --gecos 'GitLab' git ## 6. Database NOTE: **Note:** -Starting from GitLab 12.1, only PostgreSQL is supported. Because we need to make -use of extensions and concurrent index removal, you need at least PostgreSQL 9.2. +Starting from GitLab 12.1, only PostgreSQL is supported. Since GitLab 13.0, we require PostgreSQL 11+. 1. Install the database packages: @@ -426,11 +425,20 @@ cd /home/git ### Clone the Source +Clone Community Edition: + ```shell # Clone GitLab repository sudo -u git -H git clone https://gitlab.com/gitlab-org/gitlab-foss.git -b X-Y-stable gitlab ``` +Clone Enterprise Edition: + +```shell +# Clone GitLab repository +sudo -u git -H git clone https://gitlab.com/gitlab-org/gitlab-ee.git -b X-Y-stable gitlab +``` + Make sure to replace `X-Y-stable` with the stable branch that matches the version you want to install. For example, if you want to install 11.8 you would use the branch name `11-8-stable`. @@ -601,7 +609,7 @@ You can specify a different Git repository by providing it as an extra parameter sudo -u git -H bundle exec rake "gitlab:workhorse:install[/home/git/gitlab-workhorse,https://example.com/gitlab-workhorse.git]" RAILS_ENV=production ``` -### Install GitLab-Elasticsearch-indexer +### Install GitLab-Elasticsearch-indexer on Enterprise Edition GitLab-Elasticsearch-Indexer uses [GNU Make](https://www.gnu.org/software/make/). The following command-line will install GitLab-Elasticsearch-Indexer in `/home/git/gitlab-elasticsearch-indexer` @@ -620,6 +628,9 @@ sudo -u git -H bundle exec rake "gitlab:indexer:install[/home/git/gitlab-elastic The source code will first be fetched to the path specified by the first parameter. Then a binary will be built under its `bin` directory. You will then need to update `gitlab.yml`'s `production -> elasticsearch -> indexer_path` setting to point to that binary. +NOTE: **Note:** +Elasticsearch is a feature of GitLab Enterprise Edition and isn't included in GitLab Community Edition. + ### Install GitLab Pages GitLab Pages uses [GNU Make](https://www.gnu.org/software/make/). This step is optional and only needed if you wish to host static sites from within GitLab. The following commands will install GitLab Pages in `/home/git/gitlab-pages`. For additional setup steps, consult the [administration guide](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/administration/pages/source.md) for your version of GitLab as the GitLab Pages daemon can be run several different ways. diff --git a/doc/install/openshift_and_gitlab/index.md b/doc/install/openshift_and_gitlab/index.md index 7ae23d6831e..519a0a5b7aa 100644 --- a/doc/install/openshift_and_gitlab/index.md +++ b/doc/install/openshift_and_gitlab/index.md @@ -25,7 +25,8 @@ For a video demonstration on installing GitLab on OpenShift, check the article [ ## Prerequisites -CAUTION: **Caution:** This information is no longer up to date, as the current versions +CAUTION: **Caution:** +This information is no longer up to date, as the current versions have changed and products have been renamed. OpenShift 3 is not yet deployed on RedHat's offered [Online platform](https://www.openshift.com/), diff --git a/doc/install/requirements.md b/doc/install/requirements.md index 0673f3e7ea3..5bc587627f5 100644 --- a/doc/install/requirements.md +++ b/doc/install/requirements.md @@ -54,8 +54,10 @@ The minimum required Go version is 1.13. ### Git versions -GitLab 11.11 and higher only supports Git 2.24.x and newer, and -[dropped support for older versions](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/54255). +From GitLab 13.1: + +- Git 2.25.x and later is required. +- Git 2.27.x and later [is recommended](https://gitlab.com/gitlab-org/gitaly/-/issues/2829). ### Node.js versions @@ -88,7 +90,8 @@ Apart from a local hard drive you can also mount a volume that supports the netw If you have enough RAM and a recent CPU the speed of GitLab is mainly limited by hard drive seek times. Having a fast drive (7200 RPM and up) or a solid state drive (SSD) will improve the responsiveness of GitLab. -NOTE: **Note:** Since file system performance may affect GitLab's overall performance, [we don't recommend using AWS EFS for storage](../administration/high_availability/nfs.md#avoid-using-awss-elastic-file-system-efs). +NOTE: **Note:** +Since file system performance may affect GitLab's overall performance, [we don't recommend using AWS EFS for storage](../administration/high_availability/nfs.md#avoid-using-awss-elastic-file-system-efs). ### CPU @@ -143,7 +146,8 @@ GitLab database. This extension [can be enabled](https://www.postgresql.org/docs On some systems you may need to install an additional package (for example, `postgresql-contrib`) for this extension to become available. -NOTE: **Note:** Support for [PostgreSQL 9.6 and 10 has been removed in GitLab 13.0](https://about.gitlab.com/releases/2020/05/22/gitlab-13-0-released/#postgresql-11-is-now-the-minimum-required-version-to-install-gitlab) so that GitLab can benefit from PostgreSQL 11 improvements, such as partitioning. For the schedule of transitioning to PostgreSQL 12, see [the related epic](https://gitlab.com/groups/gitlab-org/-/epics/2184). +NOTE: **Note:** +Support for [PostgreSQL 9.6 and 10 has been removed in GitLab 13.0](https://about.gitlab.com/releases/2020/05/22/gitlab-13-0-released/#postgresql-11-is-now-the-minimum-required-version-to-install-gitlab) so that GitLab can benefit from PostgreSQL 11 improvements, such as partitioning. For the schedule of transitioning to PostgreSQL 12, see [the related epic](https://gitlab.com/groups/gitlab-org/-/epics/2184). #### Additional requirements for GitLab Geo @@ -258,7 +262,8 @@ For reference, GitLab.com's [auto-scaling shared runner](../user/gitlab_com/inde ## Supported web browsers -CAUTION: **Caution:** With GitLab 13.0 (May 2020) we have removed official support for Internet Explorer 11. +CAUTION: **Caution:** +With GitLab 13.0 (May 2020) we have removed official support for Internet Explorer 11. With the release of GitLab 13.4 (September 2020) we will remove all code that supports Internet Explorer 11. You can provide feedback [on this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/197987) or via your usual support channels. @@ -275,7 +280,8 @@ For the listed web browsers, GitLab supports: - The current and previous major versions of browsers except Internet Explorer. - The current minor version of a supported major version. -NOTE: **Note:** We don't support running GitLab with JavaScript disabled in the browser and have no plans of supporting that +NOTE: **Note:** +We don't support running GitLab with JavaScript disabled in the browser and have no plans of supporting that in the future because we have features such as Issue Boards which require JavaScript extensively. <!-- ## Troubleshooting diff --git a/doc/integration/README.md b/doc/integration/README.md index f06cb5e4bb6..fdaff74ca58 100644 --- a/doc/integration/README.md +++ b/doc/integration/README.md @@ -56,6 +56,8 @@ GitLab can be integrated with the following enhancements: - Configure [PlantUML](../administration/integration/plantuml.md) to use diagrams in AsciiDoc documents. - Attach merge requests to [Trello](trello_power_up.md) cards. - Enable integrated code intelligence powered by [Sourcegraph](sourcegraph.md). +- Add [Elasticsearch](elasticsearch.md) for [Advanced Global Search](../user/search/advanced_global_search.md), + [Advanced System Search](../user/search/advanced_search_syntax.md), and faster searching. ## Integrations diff --git a/doc/integration/elasticsearch.md b/doc/integration/elasticsearch.md index 07ef5dbb99d..8ca45547f11 100644 --- a/doc/integration/elasticsearch.md +++ b/doc/integration/elasticsearch.md @@ -125,10 +125,7 @@ for each Elasticsearch node, per the [official guidelines](https://www.elastic.c Keep in mind, this is the **minimum requirements** as per Elasticsearch. For production instances, they recommend considerably more resources. -Storage requirements also vary based on the installation side, but as a rule of -thumb, you should allocate the total size of your production database, **plus** -two-thirds of the total size of your Git repositories. Efforts to reduce this -total are being tracked in [epic &153](https://gitlab.com/groups/gitlab-org/-/epics/153). +The necessary storage space largely depends on the size of the repositories you want to store in GitLab but as a rule of thumb you should have at least 50% of free space as all your repositories combined take up. ## Enabling Elasticsearch @@ -153,8 +150,8 @@ The following Elasticsearch settings are available: | `AWS Access Key` | The AWS access key. | | `AWS Secret Access Key` | The AWS secret access key. | | `Maximum field length` | See [the explanation in instance limits.](../administration/instance_limits.md#maximum-field-length). | -| `Maximum bulk request size (MiB)` | The Maximum Bulk Request size is used by GitLab's Golang-based indexer processes and indicates how much data it ought to collect (and store in memory) in a given indexing process before submitting the payload to Elasticsearch’s Bulk API. This setting works in tandem with the Bulk request concurrency setting (see below) and needs to accomodate the resource constraints of both the Elasticsearch host(s) and the host(s) running GitLab's Golang-based indexer either from the `gitlab-rake` command or the Sidekiq tasks. | -| `Bulk request concurrency` | The Bulk request concurrency indicates how many of GitLab's Golang-based indexer processes (or threads) can run in parallel to collect data to subsequently submit to Elasticsearch’s Bulk API. This increases indexing performance, but fills the Elasticsearch bulk requests queue faster. This setting works in tandem with the Maximum bulk request size setting (see above) and needs to accomodate the resource constraints of both the Elasticsearch host(s) and the host(s) running GitLab's Golang-based indexer either from the `gitlab-rake` command or the Sidekiq tasks. | +| `Maximum bulk request size (MiB)` | The Maximum Bulk Request size is used by GitLab's Golang-based indexer processes and indicates how much data it ought to collect (and store in memory) in a given indexing process before submitting the payload to Elasticsearch’s Bulk API. This setting should be used with the Bulk request concurrency setting (see below) and needs to accommodate the resource constraints of both the Elasticsearch host(s) and the host(s) running GitLab's Golang-based indexer either from the `gitlab-rake` command or the Sidekiq tasks. | +| `Bulk request concurrency` | The Bulk request concurrency indicates how many of GitLab's Golang-based indexer processes (or threads) can run in parallel to collect data to subsequently submit to Elasticsearch’s Bulk API. This increases indexing performance, but fills the Elasticsearch bulk requests queue faster. This setting should be used together with the Maximum bulk request size setting (see above) and needs to accommodate the resource constraints of both the Elasticsearch host(s) and the host(s) running GitLab's Golang-based indexer either from the `gitlab-rake` command or the Sidekiq tasks. | ### Limiting namespaces and projects @@ -170,10 +167,10 @@ You can filter the selection dropdown by writing part of the namespace or projec  -NOTE: **Note**: +NOTE: **Note:** If no namespaces or projects are selected, no Elasticsearch indexing will take place. -CAUTION: **Warning**: +CAUTION: **Warning:** If you have already indexed your instance, you will have to regenerate the index in order to delete all existing data for filtering to work correctly. To do this run the Rake tasks `gitlab:elastic:recreate_index` and `gitlab:elastic:clear_index_status`. Afterwards, removing a namespace or a project from the list will delete the data @@ -187,7 +184,7 @@ To disable the Elasticsearch integration: 1. Expand the **Elasticsearch** section and uncheck **Elasticsearch indexing** and **Search with Elasticsearch enabled**. 1. Click **Save changes** for the changes to take effect. -1. (Optional) Delete the existing index by running one of these commands: +1. (Optional) Delete the existing index: ```shell # Omnibus installations @@ -209,7 +206,7 @@ To backfill existing data, you can use one of the methods below to index it in b To index via the Admin Area: 1. [Configure your Elasticsearch host and port](#enabling-elasticsearch). -1. Create empty indexes using one of the following commands: +1. Create empty indexes: ```shell # Omnibus installations @@ -222,7 +219,7 @@ To index via the Admin Area: 1. [Enable **Elasticsearch indexing**](#enabling-elasticsearch). 1. Click **Index all projects** in **Admin Area > Settings > Integrations > Elasticsearch**. 1. Click **Check progress** in the confirmation message to see the status of the background jobs. -1. Personal snippets need to be indexed manually by running one of these commands: +1. Personal snippets need to be indexed manually: ```shell # Omnibus installations @@ -240,13 +237,13 @@ Indexing can be performed using Rake tasks. #### Indexing small instances -CAUTION: **Warning**: +CAUTION: **Warning:** This will delete your existing indexes. If the database size is less than 500 MiB, and the size of all hosted repos is less than 5 GiB: -1. [Enable **Elasticsearch indexing** and configure your host and port](#enabling-elasticsearch). -1. Index your data using one of the following commands: +1. [Configure your Elasticsearch host and port](#enabling-elasticsearch). +1. Index your data: ```shell # Omnibus installations @@ -260,13 +257,13 @@ If the database size is less than 500 MiB, and the size of all hosted repos is l #### Indexing large instances -CAUTION: **Warning**: +CAUTION: **Warning:** Performing asynchronous indexing will generate a lot of Sidekiq jobs. Make sure to prepare for this task by having a [Scalable and Highly Available Setup](README.md) or creating [extra Sidekiq processes](../administration/operations/extra_sidekiq_processes.md) 1. [Configure your Elasticsearch host and port](#enabling-elasticsearch). -1. Create empty indexes using one of the following commands: +1. Create empty indexes: ```shell # Omnibus installations @@ -276,6 +273,16 @@ or creating [extra Sidekiq processes](../administration/operations/extra_sidekiq bundle exec rake gitlab:elastic:create_empty_index RAILS_ENV=production ``` +1. If this is a re-index of your GitLab instance, clear the index status: + + ```shell + # Omnibus installations + sudo gitlab-rake gitlab:elastic:clear_index_status + + # Installations from source + bundle exec rake gitlab:elastic:clear_index_status RAILS_ENV=production + ``` + 1. [Enable **Elasticsearch indexing**](#enabling-elasticsearch). 1. Indexing large Git repositories can take a while. To speed up the process, you can temporarily disable auto-refreshing and replicating. In our experience, you can expect a 20% @@ -350,8 +357,7 @@ or creating [extra Sidekiq processes](../administration/operations/extra_sidekiq indexer to "forget" all progress, so it will retry the indexing process from the start. -1. Personal snippets are not associated with a project and need to be indexed separately - by running one of these commands: +1. Personal snippets are not associated with a project and need to be indexed separately: ```shell # Omnibus installations @@ -415,7 +421,7 @@ The following are some available Rake tasks: | Task | Description | |:--------------------------------------------------------------------------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| [`sudo gitlab-rake gitlab:elastic:index`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Wrapper task for `gitlab:elastic:create_empty_index`, `gitlab:elastic:clear_index_status`, `gitlab:elastic:index_projects`, and `gitlab:elastic:index_snippets`. | +| [`sudo gitlab-rake gitlab:elastic:index`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Enables Elasticsearch Indexing and run `gitlab:elastic:create_empty_index`, `gitlab:elastic:clear_index_status`, `gitlab:elastic:index_projects`, and `gitlab:elastic:index_snippets`. | | [`sudo gitlab-rake gitlab:elastic:index_projects`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Iterates over all projects and queues Sidekiq jobs to index them in the background. | | [`sudo gitlab-rake gitlab:elastic:index_projects_status`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Determines the overall status of the indexing. It is done by counting the total number of indexed projects, dividing by a count of the total number of projects, then multiplying by 100. | | [`sudo gitlab-rake gitlab:elastic:clear_index_status`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Deletes all instances of IndexStatus for all projects. | @@ -424,6 +430,7 @@ The following are some available Rake tasks: | [`sudo gitlab-rake gitlab:elastic:recreate_index[<TARGET_NAME>]`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Wrapper task for `gitlab:elastic:delete_index[<TARGET_NAME>]` and `gitlab:elastic:create_empty_index[<TARGET_NAME>]`. | | [`sudo gitlab-rake gitlab:elastic:index_snippets`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Performs an Elasticsearch import that indexes the snippets data. | | [`sudo gitlab-rake gitlab:elastic:projects_not_indexed`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Displays which projects are not indexed. | +| [`sudo gitlab-rake gitlab:elastic:reindex_cluster`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Schedules a zero-downtime cluster reindexing task. This feature should be used with an index that was created after GitLab 13.0. | NOTE: **Note:** The `TARGET_NAME` parameter is optional and will use the default index/alias name from the current `RAILS_ENV` if not set. @@ -466,6 +473,25 @@ When performing a search, the GitLab index will use the following scopes: ## Tuning +### Guidance on choosing optimal cluster configuration + +For basic guidance on choosing a cluster configuration you may refer to [Elastic Cloud Calculator](https://cloud.elastic.co/pricing). You can find more information below. + +- Generally, you will want to use at least a 2-node cluster configuration with one replica, which will allow you to have resilience. If your storage usage is growing quickly, you may want to plan horizontal scaling (adding more nodes) beforehand. +- It's not recommended to use HDD storage with the search cluster, because it will take a hit on performance. It's better to use SSD storage (NVMe or SATA SSD drives for example). +- You can use the [GitLab Performance Tool](https://gitlab.com/gitlab-org/quality/performance) to benchmark search performance with different search cluster sizes and configurations. +- `Heap size` should be set to no more than 50% of your physical RAM. Additionally, it shouldn't be set to more than the threshold for zero-based compressed oops. The exact threshold varies, but 26 GB is safe on most systems, but can also be as large as 30 GB on some systems. See [Setting the heap size](https://www.elastic.co/guide/en/elasticsearch/reference/current/heap-size.html#heap-size) for more details. +- Number of CPUs (CPU cores) per node usually corresponds to the `Number of Elasticsearch shards` setting described below. +- A good guideline is to ensure you keep the number of shards per node below 20 per GB heap it has configured. A node with a 30GB heap should therefore have a maximum of 600 shards, but the further below this limit you can keep it the better. This will generally help the cluster stay in good health. +- Small shards result in small segments, which increases overhead. Aim to keep the average shard size between at least a few GB and a few tens of GB. Another consideration is the number of documents, you should aim for this simple formula for the number of shards: `number of expected documents / 5M +1`. +- `refresh_interval` is a per index setting. You may want to adjust that from default `1s` to a bigger value if you don't need data in realtime. This will change how soon you will see fresh results. If that's important for you, you should leave it as close as possible to the default value. +- You might want to raise [`indices.memory.index_buffer_size`](https://www.elastic.co/guide/en/elasticsearch/reference/current/indexing-buffer.html) to 30% or 40% if you have a lot of heavy indexing operations. + +### Elasticsearch integration settings guidance + +- The `Number of Elasticsearch shards` setting usually corresponds with the number of CPUs available in your cluster. For example, if you have a 3-node cluster with 4 cores each, this means you will benefit from having at least 3*4=12 shards in the cluster. Please note, it's only possible to change the shards number by using [Split index API](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-split-index.html) or by reindexing to a different index with a changed number of shards. +- The `Number of Elasticsearch replicas` setting should most of the time be equal to `1` (each shard will have 1 replica). Using `0` is not recommended, because losing one node will corrupt the index. + ### Deleted documents Whenever a change or deletion is made to an indexed GitLab object (a merge request description is changed, a file is deleted from the master branch in a repository, a project is deleted, etc), a document in the index is deleted. However, since these are "soft" deletes, the overall number of "deleted documents", and therefore wasted space, increases. Elasticsearch does intelligent merging of segments in order to remove these deleted documents. However, depending on the amount and type of activity in your GitLab installation, it's possible to see as much as 50% wasted space in the index. @@ -516,7 +542,7 @@ Here are some common pitfalls and how to overcome them: If you see `"Kaminari::PaginatableArray"` you are using Elasticsearch. - NOTE: **Note**: + NOTE: **Note:** The above instructions are used to verify that GitLab is using Elasticsearch only when indexing all namespaces. This is not to be used for scenarios that only index a [subset of namespaces](#limiting-namespaces-and-projects). - **I updated GitLab and now I can't find anything** @@ -539,7 +565,7 @@ Here are some common pitfalls and how to overcome them: pp s.search_objects.to_a ``` - NOTE: **Note**: + NOTE: **Note:** The above instructions are not to be used for scenarios that only index a [subset of namespaces](#limiting-namespaces-and-projects). See [Elasticsearch Index Scopes](#elasticsearch-index-scopes) for more information on searching for specific types of data. @@ -556,12 +582,6 @@ Here are some common pitfalls and how to overcome them: You can run `sudo gitlab-rake gitlab:elastic:projects_not_indexed` to display projects that aren't indexed. -- **No new data is added to the Elasticsearch index when I push code** - - When performing the initial indexing of blobs, we lock all projects until the project finishes indexing. It could - happen that an error during the process causes one or multiple projects to remain locked. In order to unlock them, - run the `gitlab:elastic:clear_locked_projects` Rake task. - - **"Can't specify parent if no parent field has been configured"** If you enabled Elasticsearch before GitLab 8.12 and have not rebuilt indexes you will get @@ -609,7 +629,8 @@ Here are some common pitfalls and how to overcome them: **For a single node Elasticsearch cluster the functional cluster health status will be yellow** (will never be green) because the primary shard is allocated but replicas can not be as there is no other node to which Elasticsearch can assign a replica. This also applies if you are using the [Amazon Elasticsearch](https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/aes-handling-errors.html#aes-handling-errors-yellow-cluster-status) service. - CAUTION: **Warning**: Setting the number of replicas to `0` is not something that we recommend (this is not allowed in the GitLab Elasticsearch Integration menu). If you are planning to add more Elasticsearch nodes (for a total of more than 1 Elasticsearch) the number of replicas will need to be set to an integer value larger than `0`. Failure to do so will result in lack of redundancy (losing one node will corrupt the index). + CAUTION: **Warning:** + Setting the number of replicas to `0` is not something that we recommend (this is not allowed in the GitLab Elasticsearch Integration menu). If you are planning to add more Elasticsearch nodes (for a total of more than 1 Elasticsearch) the number of replicas will need to be set to an integer value larger than `0`. Failure to do so will result in lack of redundancy (losing one node will corrupt the index). If you have a **hard requirement to have a green status for your single node Elasticsearch cluster**, please make sure you understand the risks outlined in the previous paragraph and then simply run the following query to set the number of replicas to `0`(the cluster will no longer try to create any shard replicas): diff --git a/doc/integration/github.md b/doc/integration/github.md index 68971c510d5..ccce4672cbf 100644 --- a/doc/integration/github.md +++ b/doc/integration/github.md @@ -12,7 +12,7 @@ When you create an OAuth 2 app in GitHub, you'll need the following information: - The authorization callback URL; in this case, `https://gitlab.example.com/users/auth`. Include the port number if your GitLab instance uses a non-default port. NOTE: **Note:** -To prevent an [OAuth2 covert redirect](http://tetraph.com/covert_redirect/) vulnerability, append `/users/auth` to the end of the GitHub authorization callback URL. +To prevent an [OAuth2 covert redirect](https://oauth.net/advisories/2014-1-covert-redirect/) vulnerability, append `/users/auth` to the end of the GitHub authorization callback URL. See [Initial OmniAuth Configuration](omniauth.md#initial-omniauth-configuration) for initial settings. diff --git a/doc/integration/jira_development_panel.md b/doc/integration/jira_development_panel.md index f032345b9e0..85404ff3164 100644 --- a/doc/integration/jira_development_panel.md +++ b/doc/integration/jira_development_panel.md @@ -16,7 +16,7 @@ A top-level GitLab group is one that does not have any parent group itself. All as well as projects of the top-level group's subgroups nesting down, are connected. Alternatively, you can specify a GitLab personal namespace in the Jira configuration, which will then connect the projects in that personal namespace to Jira. -NOTE: **Note**: +NOTE: **Note:** Note this is different from the [existing Jira](../user/project/integrations/jira.md) project integration, where the mapping is one GitLab project to the entire Jira instance. @@ -55,7 +55,7 @@ There are no special requirements if you are using GitLab.com. replacing `<your-gitlab-instance-domain>` appropriately. So for example, if you are using GitLab.com, this would be `https://gitlab.com/login/oauth/callback`. - NOTE: **Note**: + NOTE: **Note:** If using a GitLab version earlier than 11.3, the `Redirect URI` must be `https://<your-gitlab-instance-domain>/-/jira/login/oauth/callback`. If you want Jira to have access to all projects, GitLab recommends an administrator creates the @@ -90,7 +90,7 @@ There are no special requirements if you are using GitLab.com. replacing `<your-gitlab-instance-domain>` appropriately. So for example, if you are using GitLab.com, this would be `https://gitlab.com/`. - NOTE: **Note**: + NOTE: **Note:** If using a GitLab version earlier than 11.3 the `Host URL` value should be `https://<your-gitlab-instance-domain>/-/jira` For the `Client ID` field, use the `Application ID` value from the previous section. diff --git a/doc/integration/openid_connect_provider.md b/doc/integration/openid_connect_provider.md index 3da17347e91..b66262772da 100644 --- a/doc/integration/openid_connect_provider.md +++ b/doc/integration/openid_connect_provider.md @@ -37,11 +37,11 @@ Currently the following user information is shared with clients: | `auth_time` | `integer` | The timestamp for the user's last authentication | `name` | `string` | The user's full name | `nickname` | `string` | The user's GitLab username -| `email` | `string` | The user's public email address -| `email_verified` | `boolean` | Whether the user's public email address was verified +| `email` | `string` | The user's email address<br>This is the user's *primary* email address if the application has access to the `email` claim and the user's *public* email address otherwise +| `email_verified` | `boolean` | Whether the user's email address was verified | `website` | `string` | URL for the user's website | `profile` | `string` | URL for the user's GitLab profile | `picture` | `string` | URL for the user's GitLab avatar | `groups` | `array` | Names of the groups the user is a member of -Only the `sub` and `sub_legacy` claims are included in the ID token, all other claims are available from the `/oauth/userinfo` endpoint used by OIDC clients. +The claims `sub`, `sub_legacy`, `email` and `email_verified` are included in the ID token, all other claims are available from the `/oauth/userinfo` endpoint used by OIDC clients. diff --git a/doc/integration/recaptcha.md b/doc/integration/recaptcha.md index bf2e6568ea6..48c83f1393b 100644 --- a/doc/integration/recaptcha.md +++ b/doc/integration/recaptcha.md @@ -15,6 +15,12 @@ To use reCAPTCHA, first you must create a site and private key. 1. Fill all reCAPTCHA fields with keys from previous steps. 1. Check the `Enable reCAPTCHA` checkbox. 1. Save the configuration. +1. Change the first line of the `#execute` method in `app/services/spam/spam_verdict_service.rb` + to `return CONDITONAL_ALLOW` so that the spam check short-circuits and triggers the response to + return `recaptcha_html`. + +NOTE: **Note:** +Make sure you are viewing an issuable in a project that is public, and if you're working with an issue, the issue is public. ## Enabling reCAPTCHA for user logins via passwords diff --git a/doc/integration/saml.md b/doc/integration/saml.md index 97384d8f6cf..9dce7269c86 100644 --- a/doc/integration/saml.md +++ b/doc/integration/saml.md @@ -1,11 +1,29 @@ +--- +type: reference +stage: Manage +group: Access +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # SAML OmniAuth Provider **(CORE ONLY)** -Note that: +This page describes instance-wide SAML for self-managed GitLab instances. For SAML on GitLab.com, see [SAML SSO for GitLab.com groups](../user/group/saml_sso/index.md). + +You should also reference the [OmniAuth documentation](omniauth.md) for general settings that apply to all OmniAuth providers. -- SAML OmniAuth Provider is for SAML on self-managed GitLab instances. For SAML on - GitLab.com, see [SAML SSO for GitLab.com Groups](../user/group/saml_sso/index.md). -- Starting from GitLab 11.4, OmniAuth is enabled by default. If you're using an - earlier version, you'll need to explicitly enable it. +## Common SAML Terms + +| Term | Description | +|------|-------------| +| Identity Provider (IdP) | The service which manages your user identities such as ADFS, Okta, Onelogin, or Ping Identity. | +| Service Provider (SP) | SAML considers GitLab to be a service provider. | +| Assertion | A piece of information about a user's identity, such as their name or role. Also know as claims or attributes. | +| SSO | Single Sign-On. | +| Assertion consumer service URL | The callback on GitLab where users will be redirected after successfully authenticating with the identity provider. | +| Issuer | How GitLab identifies itself to the identity provider. Also known as a "Relying party trust identifier". | +| Certificate fingerprint | Used to confirm that communications over SAML are secure by checking that the server is signing communications with the correct certificate. Also known as a certificate thumbprint. | + +## General Setup GitLab can be configured to act as a SAML 2.0 Service Provider (SP). This allows GitLab to consume assertions from a SAML 2.0 Identity Provider (IdP) such as @@ -143,17 +161,10 @@ On the sign in page there should now be a SAML button below the regular sign in Click the icon to begin the authentication process. If everything goes well the user will be returned to GitLab and will be signed in. -## Marking Users as External based on SAML Groups - ->**Note:** -This setting is only available on GitLab 8.7 and above. +## SAML Groups -SAML login includes support for automatically identifying whether a user should -be considered an [external](../user/permissions.md) user based on the user's group -membership in the SAML identity provider. This feature **does not** allow you to -automatically add users to GitLab [Groups](../user/group/index.md), it simply -allows you to mark users as External if they are members of certain groups in the -Identity Provider. +You can require users to be members of a certain group, or assign users `external`, `admin` or `auditor` roles based on group membership. This feature **does not** allow you to +automatically add users to GitLab [Groups](../user/group/index.md). ### Requirements @@ -164,74 +175,73 @@ with the regular SAML response. Here is an example: ```xml <saml:AttributeStatement> <saml:Attribute Name="Groups"> - <saml:AttributeValue xsi:type="xs:string">SecurityGroup</saml:AttributeValue> <saml:AttributeValue xsi:type="xs:string">Developers</saml:AttributeValue> - <saml:AttributeValue xsi:type="xs:string">Designers</saml:AttributeValue> + <saml:AttributeValue xsi:type="xs:string">Freelancers</saml:AttributeValue> + <saml:AttributeValue xsi:type="xs:string">Admins</saml:AttributeValue> + <saml:AttributeValue xsi:type="xs:string">Auditors</saml:AttributeValue> </saml:Attribute> </saml:AttributeStatement> ``` The name of the attribute can be anything you like, but it must contain the groups to which a user belongs. In order to tell GitLab where to find these groups, you need -to add a `groups_attribute:` element to your SAML settings. You will also need to -tell GitLab which groups are external via the `external_groups:` element: +to add a `groups_attribute:` element to your SAML settings. + +### Required groups **(STARTER ONLY)** + +Your IdP passes Group Information to the SP (GitLab) in the SAML Response. You need to configure GitLab to identify: + +- Where to look for the groups in the SAML response via the `groups_attribute` setting +- Which group membership is requisite to sign in via the `required_groups` setting + +When `required_groups` is not set or it is empty, anyone with proper authentication +will be able to use the service. + +Example: ```yaml { name: 'saml', label: 'Our SAML Provider', groups_attribute: 'Groups', - external_groups: ['Freelancers', 'Interns'], + required_groups: ['Developers', 'Freelancers', 'Admins', 'Auditors'], args: { assertion_consumer_service_url: 'https://gitlab.example.com/users/auth/saml/callback', idp_cert_fingerprint: '43:51:43:a1:b5:fc:8b:b7:0a:3a:a9:b1:0f:66:73:a8', idp_sso_target_url: 'https://login.example.com/idp', issuer: 'https://gitlab.example.com', - name_identifier_format: 'urn:oasis:names:tc:SAML:2.0:nameid-format:persistent' + name_identifier_format: 'urn:oasis:names:tc:SAML:2.0:nameid-format:transient' } } ``` -## Required groups **(STARTER ONLY)** - ->**Note:** -This setting is only available on GitLab 10.2 EE and above. - -This setting works like `External Groups` setting. Just like there, your IdP has to -pass Group Information to GitLab, you have to tell GitLab where to look or the -groups SAML response, and which group membership should be requisite for logging in. -When `required_groups` is not set or it is empty, anyone with proper authentication -will be able to use the service. +### External Groups **(STARTER ONLY)** -Example: +SAML login supports automatic identification on whether a user should be considered an [external](../user/permissions.md) user. This is based on the user's group membership in the SAML identity provider. ```yaml { name: 'saml', label: 'Our SAML Provider', groups_attribute: 'Groups', - required_groups: ['Developers', 'Managers', 'Admins'], + external_groups: ['Freelancers'], args: { assertion_consumer_service_url: 'https://gitlab.example.com/users/auth/saml/callback', idp_cert_fingerprint: '43:51:43:a1:b5:fc:8b:b7:0a:3a:a9:b1:0f:66:73:a8', idp_sso_target_url: 'https://login.example.com/idp', issuer: 'https://gitlab.example.com', - name_identifier_format: 'urn:oasis:names:tc:SAML:2.0:nameid-format:transient' + name_identifier_format: 'urn:oasis:names:tc:SAML:2.0:nameid-format:persistent' } } ``` -## Admin Groups **(STARTER ONLY)** - ->**Note:** -This setting is only available on GitLab 8.8 EE and above. +### Admin Groups **(STARTER ONLY)** -This setting works very similarly to the `External Groups` setting. The requirements -are the same, your IdP needs to pass Group information to GitLab, you need to tell -GitLab where to look for the groups in the SAML response, and which group should be -considered `admin groups`. +The requirements are the same as the previous settings, your IdP needs to pass Group information to GitLab, you need to tell +GitLab where to look for the groups in the SAML response, and which group(s) should be +considered admin users. ```yaml { name: 'saml', label: 'Our SAML Provider', groups_attribute: 'Groups', - admin_groups: ['Managers', 'Admins'], + admin_groups: ['Admins'], args: { assertion_consumer_service_url: 'https://gitlab.example.com/users/auth/saml/callback', idp_cert_fingerprint: '43:51:43:a1:b5:fc:8b:b7:0a:3a:a9:b1:0f:66:73:a8', @@ -241,18 +251,20 @@ considered `admin groups`. } } ``` -## Auditor Groups **(STARTER ONLY)** +### Auditor Groups **(STARTER ONLY)** >**Note:** This setting is only available on GitLab 11.4 EE and above. -This setting also follows the requirements documented for the `External Groups` setting. GitLab uses the Group information provided by your IdP to determine if a user should be assigned the `auditor` role. +The requirements are the same as the previous settings, your IdP needs to pass Group information to GitLab, you need to tell +GitLab where to look for the groups in the SAML response, and which group(s) should be +considered auditor users. ```yaml { name: 'saml', label: 'Our SAML Provider', groups_attribute: 'Groups', - auditor_groups: ['Auditors', 'Security'], + auditor_groups: ['Auditors'], args: { assertion_consumer_service_url: 'https://gitlab.example.com/users/auth/saml/callback', idp_cert_fingerprint: '43:51:43:a1:b5:fc:8b:b7:0a:3a:a9:b1:0f:66:73:a8', @@ -265,7 +277,18 @@ This setting also follows the requirements documented for the `External Groups` ## Bypass two factor authentication If you want some SAML authentication methods to count as 2FA on a per session basis, you can register them in the -`upstream_two_factor_authn_contexts` list: +`upstream_two_factor_authn_contexts` list. + +In addition to the changes in GitLab, make sure that your IdP is returning the +`AuthnContext`. For example: + +```xml +<saml:AuthnStatement> + <saml:AuthnContext> + <saml:AuthnContextClassRef>urn:oasis:names:tc:SAML:2.0:ac:classes:MediumStrongCertificateProtectedTransport</saml:AuthnContextClassRef> + </saml:AuthnContext> +</saml:AuthnStatement> +``` **For Omnibus installations:** @@ -324,18 +347,7 @@ If you want some SAML authentication methods to count as 2FA on a per session ba } ``` -1. Save the file and [restart GitLab](../administration/restart_gitlab.md#installations-from-source) for the changes ot take effect - -In addition to the changes in GitLab, make sure that your Idp is returning the -`AuthnContext`. For example: - -```xml -<saml:AuthnStatement> - <saml:AuthnContext> - <saml:AuthnContextClassRef>urn:oasis:names:tc:SAML:2.0:ac:classes:MediumStrongCertificateProtectedTransport</saml:AuthnContextClassRef> - </saml:AuthnContext> -</saml:AuthnStatement> -``` +1. Save the file and [restart GitLab](../administration/restart_gitlab.md#installations-from-source) for the changes to take effect ## Customization @@ -368,7 +380,6 @@ You may also bypass the auto signin feature by browsing to ### `attribute_statements` >**Note:** -This setting is only available on GitLab 8.6 and above. This setting should only be used to map attributes that are part of the OmniAuth `info` hash schema. @@ -396,6 +407,21 @@ args: { } ``` +#### Setting a username + +By default, the email in the SAML response will be used to automatically generate the user's GitLab username. If you'd like to set another attribute as the username, assign it to the `nickname` OmniAuth `info` hash attribute. For example, if you wanted to set the `username` attribute in your SAML Response to the username in GitLab, use the following setting: + +```yaml +args: { + assertion_consumer_service_url: 'https://gitlab.example.com/users/auth/saml/callback', + idp_cert_fingerprint: '43:51:43:a1:b5:fc:8b:b7:0a:3a:a9:b1:0f:66:73:a8', + idp_sso_target_url: 'https://login.example.com/idp', + issuer: 'https://gitlab.example.com', + name_identifier_format: 'urn:oasis:names:tc:SAML:2.0:nameid-format:persistent', + attribute_statements: { nickname: ['username'] } +} +``` + ### `allowed_clock_drift` The clock of the Identity Provider may drift slightly ahead of your system clocks. @@ -557,10 +583,12 @@ Avoid user control of the following attributes: These attributes define the SAML user. If users can change these attributes, they can impersonate others. -Refer to the documentation for your [SAML Identity Provider](../user/group/saml_sso/index.md#providers) for information on how to fix these attributes. +Refer to the documentation for your SAML Identity Provider for information on how to fix these attributes. ## Troubleshooting +You can find the base64-encoded SAML Response in the [`production_json.log`](../administration/logs.md#production_jsonlog). + ### GitLab+SAML Testing Environments If you need to troubleshoot, [a complete GitLab+SAML testing environment using Docker compose](https://gitlab.com/gitlab-com/support/toolbox/replication/tree/master/compose_files) is available. @@ -580,13 +608,14 @@ Make sure the IdP provides a claim containing the user's email address, using cl If after signing in into your SAML server you are redirected back to the sign in page and no error is displayed, check your `production.log` file. It will most likely contain the message `Can't verify CSRF token authenticity`. This means that there is an error during -the SAML request, but this error never reaches GitLab due to the CSRF check. +the SAML request, but in GitLab 11.7 and earlier this error never reaches GitLab due to +the CSRF check. To bypass this you can add `skip_before_action :verify_authenticity_token` to the `omniauth_callbacks_controller.rb` file immediately after the `class` line and -comment out the `protect_from_forgery` line using a `#` then restart Unicorn. This -will allow the error to hit GitLab, where it can then be seen in the usual logs, -or as a flash message on the login screen. +comment out the `protect_from_forgery` line using a `#`. Restart Unicorn for this +change to take effect. This will allow the error to hit GitLab, where it can then +be seen in the usual logs, or as a flash message on the login screen. That file is located in `/opt/gitlab/embedded/service/gitlab-rails/app/controllers` for Omnibus installations and by default in `/home/git/gitlab/app/controllers` for @@ -613,6 +642,8 @@ is not providing this information, all SAML requests will fail. Make sure this information is provided. +Another issue that can result in this error is when the correct information is being sent by the IdP, but the attributes don't match the names in the OmniAuth `info` hash. In this case, you'll need to set `attribute_statements` in the SAML configuration to [map the attribute names in your SAML Response to the corresponding OmniAuth `info` hash names](#attribute_statements). + ### Key validation error, Digest mismatch or Fingerprint mismatch These errors all come from a similar place, the SAML certificate. SAML requests diff --git a/doc/migrate_ci_to_ce/README.md b/doc/migrate_ci_to_ce/README.md index 10f22af30f7..6710356d760 100644 --- a/doc/migrate_ci_to_ce/README.md +++ b/doc/migrate_ci_to_ce/README.md @@ -77,7 +77,7 @@ sudo -u git -H bundle exec rake gitlab:backup:create RAILS_ENV=production SKIP=r If this fails you need to fix it before upgrading to 8.0. Also see <https://about.gitlab.com/get-help/> -NOTE: **Note** +NOTE: **Note:** For GitLab 12.1 and earlier, use `gitlab-rake gitlab:backup:create`. ### 2. Check source and target database types diff --git a/doc/operations/README.md b/doc/operations/README.md deleted file mode 100644 index 319e5f48d38..00000000000 --- a/doc/operations/README.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../administration/operations/index.md' ---- - -This document was moved to [another location](../administration/operations/index.md). diff --git a/doc/operations/feature_flags.md b/doc/operations/feature_flags.md new file mode 100644 index 00000000000..7614d70e132 --- /dev/null +++ b/doc/operations/feature_flags.md @@ -0,0 +1,328 @@ +--- +stage: Release +group: Progressive Delivery +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Feature Flags **(PREMIUM)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/7433) in GitLab 11.4. + +With Feature Flags, you can deploy your application's new features to production in smaller batches. +You can toggle a feature on and off to subsets of users, helping you achieve Continuous Delivery. +Feature flags help reduce risk, allowing you to do controlled testing, and separate feature +delivery from customer launch. + +<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> +For an example of feature flags in action, see [GitLab for Deploys, Feature Flags, and Error Tracking](https://www.youtube.com/embed/5tw2p6lwXxo). + +## How it works + +GitLab uses [Unleash](https://github.com/Unleash/unleash), a feature +toggle service. + +By enabling or disabling a flag in GitLab, your application +can determine which features to enable or disable. + +You can create feature flags in GitLab and use the API from your application +to get the list of feature flags and their statuses. The application must be configured to communicate +with GitLab, so it's up to developers to use a compatible client library and +[integrate the feature flags in your app](#integrate-feature-flags-with-your-application). + +## Create a feature flag + +To create and enable a feature flag: + +1. Navigate to your project's **Operations > Feature Flags**. +1. Click the **New feature flag** button. +1. Enter a name that starts with a letter and contains only lowercase letters, digits, underscores (`_`), + or dashes (`-`), and does not end with a dash (`-`) or underscore (`_`). +1. Enter a description (optional, 255 characters max). +1. Enter details about how the flag should be applied: + - In GitLab 13.0 and earlier, add **Environment specs**. For each environment, + include the **Status** (default enabled) and [**Rollout strategy**](#rollout-strategy-legacy) + (defaults to **All users**). + - In GitLab 13.1 and later, add Feature Flag [**Strategies**](#feature-flag-strategies). + For each strategy, include the **Type** (defaults to [**All users**](#all-users)) + and **Environments** (defaults to all environments). +1. Click **Create feature flag**. + +You can change these settings by clicking the **{pencil}** (edit) button +next to any feature flag in the list. + +## Rollout strategy (legacy) + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/8240) in GitLab 12.2. + +In GitLab 13.0 and earlier, the **Rollout strategy** setting affects which users will experience +the feature as enabled. Choose the percentage of users that the feature will be enabled +for. The rollout strategy will have no effect if the environment spec is disabled. + +It can be set to: + +- All users +- [Percent of users](#percent-of-users) + - Optionally, you can click the **Include additional user IDs** checkbox and add a list + of specific users IDs to enable the feature for. +- [User IDs](#user-ids) + +## Feature flag strategies + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/35555) in GitLab 13.0. +> - It's deployed behind a feature flag, disabled by default. +> - It's enabled on GitLab.com. +> - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enable-or-disable-feature-flag-strategies). **(CORE ONLY)** + +You can apply a feature flag strategy across multiple environments, without defining +the strategy multiple times. + +GitLab Feature Flags use [Unleash](https://unleash.github.io) as the feature flag +engine. In Unleash, there are [strategies](https://unleash.github.io/docs/activation_strategy) +for granular feature flag controls. GitLab Feature Flags can have multiple strategies, +and the supported strategies are: + +- [All users](#all-users) +- [Percent of Users](#percent-of-users) +- [User IDs](#user-ids) +- [User List](#user-list) + +Strategies can be added to feature flags when [creating a feature flag](#create-a-feature-flag), +or by editing an existing feature flag after creation by navigating to **Operations > Feature Flags** +and clicking **{pencil}** (edit). + +### All users + +Enables the feature for all users. It uses the [`default`](https://unleash.github.io/docs/activation_strategy#default) +Unleash activation strategy. + +### Percent of Users + +Enables the feature for a percentage of authenticated users. It uses the +[`gradualRolloutUserId`](https://unleash.github.io/docs/activation_strategy#gradualrolloutuserid) +Unleash activation strategy. + +For example, set a value of 15% to enable the feature for 15% of authenticated users. + +The rollout percentage can be from 0% to 100%. + +NOTE: **Note:** +Stickiness (consistent application behavior for the same user) is guaranteed for logged-in users, but not anonymous users. + +CAUTION: **Caution:** +If this strategy is selected, then the Unleash client **must** be given a user +ID for the feature to be enabled. See the [Ruby example](#ruby-application-example) below. + +### User IDs + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/8240) in GitLab 12.2. [Updated](https://gitlab.com/gitlab-org/gitlab/-/issues/34363) to be defined per environment in GitLab 12.6. + +Enables the feature for a list of target users. It is implemented +using the Unleash [`userWithId`](https://unleash.github.io/docs/activation_strategy#userwithid) +activation strategy. + +Enter user IDs as a comma-separated list of values. For example, +`user@example.com, user2@example.com`, or `username1,username2,username3`, and so on. + +NOTE: **Note:** +User IDs are identifiers for your application users. They do not need to be GitLab users. + +CAUTION: **Caution:** +The Unleash client **must** be given a user ID for the feature to be enabled for +target users. See the [Ruby example](#ruby-application-example) below. + +### User List + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/35930) in GitLab 13.1. + +Enables the feature for lists of users created with the [Feature Flag User List API](../api/feature_flag_user_lists.md). +Similar to [User IDs](#user-ids), it uses the Unleash [`userWithId`](https://unleash.github.io/docs/activation_strategy#userwithid) +activation strategy. + +### Enable or disable feature flag strategies + +This feature is under development, but is ready for testing. It's +deployed behind a feature flag that is **disabled by default**. +[GitLab administrators with access to the GitLab Rails console](../administration/feature_flags.md) +can enable it for your instance. + +To enable it: + +```ruby +Feature.enable(:feature_flags_new_version) +``` + +To disable it: + +```ruby +Feature.disable(:feature_flags_new_version) +``` + +## Disable a feature flag for a specific environment + +In [GitLab 13.0 and earlier](https://gitlab.com/gitlab-org/gitlab/-/issues/8621), +to disable a feature flag for a specific environment: + +1. Navigate to your project's **Operations > Feature Flags**. +1. For the feature flag you want to disable, click the Pencil icon. +1. To disable the flag: + + - In GitLab 13.0 and earlier: Slide the Status toggle for the environment. Or, to delete the + environment spec, on the right, click the **Remove (X)** icon. + - In GitLab 13.1 and later: For each strategy it applies to, under **Environments**, delete the environment. + +1. Click **Save changes**. + +## Disable a feature flag for all environments + +To disable a feature flag for all environments: + +1. Navigate to your project's **Operations > Feature Flags**. +1. For the feature flag you want to disable, slide the Status toggle to **Disabled**. + +The feature flag is displayed on the **Disabled** tab. + +## Integrate feature flags with your application + +To use feature flags with your application, get access credentials from GitLab. +Then prepare your application with a client library. + +### Get access credentials + +To get the access credentials that your application needs to communicate with GitLab: + +1. Navigate to your project's **Operations > Feature Flags**. +1. Click the **Configure** button to view the following: + - **API URL**: URL where the client (application) connects to get a list of feature flags. + - **Instance ID**: Unique token that authorizes the retrieval of the feature flags. + - **Application name**: The name of the running environment. For instance, + if the application runs for a production server, the application name would be + `production` or similar. This value is used for the environment spec evaluation. + +NOTE: **Note:** +The meaning of these fields might change over time. For example, we are not sure +if **Instance ID** will be single token or multiple tokens, assigned to the +**Environment**. Also, **Application name** could describe the version of +application instead of the running environment. + +### Choose a client library + +GitLab implements a single backend that is compatible with Unleash clients. + +With the Unleash client, developers can define, in the application code, the default values for flags. +Each feature flag evaluation can express the desired outcome if the flag isn't present in the +provided configuration file. + +Unleash currently [offers many SDKs for various languages and frameworks](https://github.com/Unleash/unleash#client-implementations). + +### Feature flags API information + +For API content, see: + +- [Feature Flags API](../api/feature_flags.md) +- [Feature Flag Specs API](../api/feature_flag_specs.md) (Deprecated and [scheduled for removal in GitLab 14.0](https://gitlab.com/gitlab-org/gitlab/-/issues/213369).) +- [Feature Flag User Lists API](../api/feature_flag_user_lists.md) +- [Legacy Feature Flags API](../api/feature_flags_legacy.md) + +### Golang application example + +Here's an example of how to integrate feature flags in a Golang application: + +```golang +package main + +import ( + "io" + "log" + "net/http" + + "github.com/Unleash/unleash-client-go" +) + +type metricsInterface struct { +} + +func init() { + unleash.Initialize( + unleash.WithUrl("https://gitlab.com/api/v4/feature_flags/unleash/42"), + unleash.WithInstanceId("29QmjsW6KngPR5JNPMWx"), + unleash.WithAppName("production"), + unleash.WithListener(&metricsInterface{}), + ) +} + +func helloServer(w http.ResponseWriter, req *http.Request) { + if unleash.IsEnabled("my_feature_name") { + io.WriteString(w, "Feature enabled\n") + } else { + io.WriteString(w, "hello, world!\n") + } +} + +func main() { + http.HandleFunc("/", helloServer) + log.Fatal(http.ListenAndServe(":8080", nil)) +} +``` + +### Ruby application example + +Here's an example of how to integrate feature flags in a Ruby application. + +The Unleash client is given a user ID for use with a **Percent rollout (logged in users)** rollout strategy or a list of **Target Users**. + +```ruby +#!/usr/bin/env ruby + +require 'unleash' +require 'unleash/context' + +unleash = Unleash::Client.new({ + url: 'http://gitlab.com/api/v4/feature_flags/unleash/42', + app_name: 'production', + instance_id: '29QmjsW6KngPR5JNPMWx' +}) + +unleash_context = Unleash::Context.new +# Replace "123" with the id of an authenticated user. +# Note that the context's user id must be a string: +# https://unleash.github.io/docs/unleash_context +unleash_context.user_id = "123" + +if unleash.is_enabled?("my_feature_name", unleash_context) + puts "Feature enabled" +else + puts "hello, world!" +end +``` + +## Feature Flag Related Issues + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/36617) in GitLab 13.2. +> - It's deployed behind a feature flag, enabled by default. +> - It's enabled on GitLab.com. +> - It can't be enabled or disabled per-project +> - It's recommended for production use. +> - For GitLab self-managed instances, GitLab administrators can opt to disable it. + +You can link related issues to a feature flag. In the **Linked issues** section, click the `+` button and input the issue reference number or the full URL of the issue. + +This feature is similar to the [related issues](../user/project/issues/related_issues.md) feature. + +### Enable or disable Feature Flag Related Issues **(CORE ONLY)** + +Feature Flag Related Issues is under development but ready for production use. +It is deployed behind a feature flag that is **enabled by default**. +[GitLab administrators with access to the GitLab Rails console](../administration/feature_flags.md) +can opt to disable it for your instance. + +To disable it: + +```ruby +Feature.disable(:feature_flags_issue_links) +``` + +To enable it: + +```ruby +Feature.enable(:feature_flags_issue_links) +``` diff --git a/doc/operations/index.md b/doc/operations/index.md new file mode 100644 index 00000000000..314a1b231ba --- /dev/null +++ b/doc/operations/index.md @@ -0,0 +1,20 @@ +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Project operations + +GitLab provides a variety of tools to help operate and maintain +your applications: + +- Collect [Prometheus metrics](../user/project/integrations/prometheus_library/index.md). +- Deploy to different [environments](../ci/environments/index.md). +- Manage your [Alerts](../user/project/operations/alert_management.md) and [Incidents](../user/incident_management/index.md). +- Connect your project to a [Kubernetes cluster](../user/project/clusters/index.md). +- Manage your infrastructure with [Infrastructure as Code](../user/infrastructure/index.md) approaches. +- Discover and view errors generated by your applications with [Error Tracking](../user/project/operations/error_tracking.md). +- Create, toggle, and remove [Feature Flags](feature_flags.md). **(PREMIUM)** +- [Trace](tracing.md) the performance and health of a deployed application. **(ULTIMATE)** +- Change the [settings of the Monitoring Dashboard](../user/project/operations/dashboard_settings.md). diff --git a/doc/operations/metrics/alerts.md b/doc/operations/metrics/alerts.md new file mode 100644 index 00000000000..43debbd1b78 --- /dev/null +++ b/doc/operations/metrics/alerts.md @@ -0,0 +1,110 @@ +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Set up alerts for Prometheus metrics + +After [configuring metrics for your CI/CD environment](index.md), you can set up +alerting for Prometheus metrics depending on the location of your instances, and +[trigger actions from alerts](#trigger-actions-from-alerts-ultimate) to notify +your team when environment performance falls outside of the boundaries you set. + +## Managed Prometheus instances + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/6590) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.2 for [custom metrics](index.md#adding-custom-metrics), and 11.3 for [library metrics](../../user/project/integrations/prometheus_library/metrics.md). + +For managed Prometheus instances using auto configuration, you can +[configure alerts for metrics](index.md#adding-custom-metrics) directly in the +[metrics dashboard](index.md). To set an alert: + +1. In your project, navigate to **{cloud-gear}** **Operations > Metrics**, +1. Identify the metric you want to create the alert for, and click the + **ellipsis** **{ellipsis_v}** icon in the top right corner of the metric. +1. Choose **Alerts**. +1. Set threshold and operator. +1. Click **Add** to save and activate the alert. + + + +To remove the alert, click back on the alert icon for the desired metric, and click **Delete**. + +## External Prometheus instances + +>- [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9258) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.8. +>- [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/42640) to [GitLab Core](https://about.gitlab.com/pricing/) in 12.10. + +For manually configured Prometheus servers, GitLab provides a notify endpoint for +use with Prometheus webhooks. If you have manual configuration enabled, an +**Alerts** section is added to **{settings}** **Settings > Integrations > Prometheus**. +This section contains the **URL** and **Authorization Key** you will need. The +**Reset Key** button will invalidate the key and generate a new one. + + + +To send GitLab alert notifications, copy the **URL** and **Authorization Key** into the +[`webhook_configs`](https://prometheus.io/docs/alerting/latest/configuration/#webhook_config) +section of your Prometheus Alertmanager configuration: + +```yaml +receivers: + name: gitlab + webhook_configs: + - http_config: + bearer_token: 9e1cbfcd546896a9ea8be557caf13a76 + send_resolved: true + url: http://192.168.178.31:3001/root/manual_prometheus/prometheus/alerts/notify.json + ... +``` + +For GitLab to associate your alerts with an [environment](../../ci/environments/index.md), +you must configure a `gitlab_environment_name` label on the alerts you set up in +Prometheus. The value of this should match the name of your environment in GitLab. + +NOTE: **Note:** +In GitLab versions 13.1 and greater, you can configure your manually configured +Prometheus server to use the +[Generic alerts integration](../../user/project/integrations/generic_alerts.md). + +## Trigger actions from alerts **(ULTIMATE)** + +>- [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/4925) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.11. +>- [From GitLab Ultimate 12.5](https://gitlab.com/gitlab-org/gitlab/-/issues/13401), when GitLab receives a recovery alert, it will automatically close the associated issue. + +Alerts can be used to trigger actions, like opening an issue automatically +(disabled by default since `13.1`). To configure the actions: + +1. Navigate to your project's **{settings}** **Settings > Operations > Incidents**. +1. Enable the option to create issues. +1. Choose the [issue template](../../user/project/description_templates.md) to create the issue from. +1. Optionally, select whether to send an email notification to the developers of the project. +1. Click **Save changes**. + +After enabling, GitLab automatically opens an issue when an alert is triggered containing +values extracted from [alert's payload](https://prometheus.io/docs/alerting/latest/configuration/#webhook_config): + +- Issue author: `GitLab Alert Bot` +- Issue title: Extract from `annotations/title`, `annotations/summary` or `labels/alertname` +- Alert `Summary`: A list of properties + - `starts_at`: Alert start time via `startsAt` + - `full_query`: Alert query extracted from `generatorURL` + - Optional list of attached annotations extracted from `annotations/*` +- Alert [GFM](../../user/markdown.md): GitLab Flavored Markdown from `annotations/gitlab_incident_markdown` + +When GitLab receives a **Recovery Alert**, it closes the associated issue. +This action is recorded as a system message on the issue indicating that it +was closed automatically by the GitLab Alert bot. + +To further customize the issue, you can add labels, mentions, or any other supported +[quick action](../../user/project/quick_actions.md) in the selected issue template, +which applies to all incidents. To limit quick actions or other information to +only specific types of alerts, use the `annotations/gitlab_incident_markdown` field. + +Since [version 12.2](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/63373), +GitLab tags each incident issue with the `incident` label automatically. If the label +does not yet exist, it is also created automatically. + +If the metric exceeds the threshold of the alert for over 5 minutes, GitLab sends +an email to all [Maintainers and Owners](../../user/permissions.md#project-members-permissions) +of the project. diff --git a/doc/operations/metrics/dashboards/index.md b/doc/operations/metrics/dashboards/index.md new file mode 100644 index 00000000000..a46abdee2dc --- /dev/null +++ b/doc/operations/metrics/dashboards/index.md @@ -0,0 +1,249 @@ +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Using the Metrics Dashboard + +## Manage the metrics dashboard settings + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/223204) in GitLab 13.2. + +To manage the settings for your metrics dashboard: + +1. Sign in as a user with project Maintainer or Admin + [permissions](../../../user/permissions.md#project-members-permissions). +1. Navigate to your dashboard at **{cloud-gear}** **Operations > Metrics**. +1. In the top-right corner of your dashboard, click **{settings}** **Metrics Settings**: + +  + +## Chart Context Menu + +From each of the panels in the dashboard, you can access the context menu by clicking the **{ellipsis_v}** **More actions** dropdown box above the upper right corner of the panel to take actions related to the chart's data. + + + +The options are: + +- [Expand panel](#expand-panel) +- [View logs](#view-logs-ultimate) +- [Download CSV](#downloading-data-as-csv) +- [Copy link to chart](../embed.md#embedding-gitlab-managed-kubernetes-metrics) +- [Alerts](../alerts.md) + +### View and edit the source file of a custom dashboard + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/34779) in GitLab 12.5. + +When viewing a custom dashboard of a project, you can view the original +`.yml` file by clicking on the **Edit dashboard** button. + +### Expand panel + +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3100) in GitLab 13.0. + +To view a larger version of a visualization, expand the panel by clicking the +**{ellipsis_v}** **More actions** icon and selecting **Expand panel**. + +To return to the metrics dashboard, click the **Back** button in your +browser, or pressing the <kbd>Esc</kbd> key. + +### View Logs **(ULTIMATE)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/122013) in GitLab 12.8. + +If you have [Logs](../../../user/project/clusters/kubernetes_pod_logs.md) enabled, +you can navigate from the charts in the dashboard to view Logs by +clicking on the context menu in the upper-right corner. + +If you use the **Timeline zoom** function at the bottom of the chart, logs will narrow down to the time range you selected. + +### Timeline zoom and URL sharing + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/198910) in GitLab 12.8. + +You can use the **Timeline zoom** function at the bottom of a chart to zoom in +on a date and time of your choice. When you click and drag the sliders to select +a different beginning or end date of data to display, GitLab adds your selected start +and end times to the URL, enabling you to share specific timeframes more easily. + +### Downloading data as CSV + +Data from Prometheus charts on the metrics dashboard can be downloaded as CSV. + +## Dashboard Annotations + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/211330) in GitLab 12.10 (enabled by feature flag `metrics_dashboard_annotations`). +> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/215224) in GitLab 13.0. + +You can use **Metrics Dashboard Annotations** to mark any important events on +every metrics dashboard by adding annotations to it. While viewing a dashboard, +annotation entries assigned to the selected time range will be automatically +fetched and displayed on every chart within that dashboard. On mouse hover, each +annotation presents additional details, including the exact time of an event and +its description. + +You can create annotations by making requests to the +[Metrics dashboard annotations API](../../../api/metrics_dashboard_annotations.md) + + + +### Retention policy + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/211433) in GitLab 13.01. + +To avoid excessive storage space consumption by stale annotations, records attached +to time periods older than two weeks are removed daily. This recurring background +job runs at 1:00 a.m. local server time. + +## Add related links to custom dashboards + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/216385) in GitLab 13.1. + +You can embed links to other dashboards or external services in your custom +dashboard by adding **Related links** to your dashboard's YAML file. Related links +open in the same tab as the dashboard. Related links can be displayed in the +following locations on your dashboard: + +- At the top of your dashboard as the top level [`links` dashboard property](../../../operations/metrics/dashboards/yaml.md#dashboard-top-level-properties). +- In charts context menus as the [`links` property of a panel](../../../operations/metrics/dashboards/yaml.md#panel-panels-properties). + +Related links can contain the following attributes: + +- `url`: The full URL to the link. Required. +- `title`: A phrase describing the link. Optional. If this attribute is not set, + the full URL is used for the link title. +- `type`: A string declaring the type of link. Optional. If set to `grafana`, the + dashboard's time range values are converted to Grafana's time range format and + appended to the `url`. + +The dashboard's time range is appended to the `url` as URL parameters. + +The following example shows two related links (`GitLab.com` and `GitLab Documentation`) +added to a dashboard: + + + +### Links Syntax + +```yaml +links: + - title: GitLab.com + url: https://gitlab.com + - title: GitLab Documentation + url: https://docs.gitlab.com + - title: Public Grafana playground dashboard + url: https://play.grafana.org/d/000000012/grafana-play-home?orgId=1 + type: grafana +``` + +## Defining custom dashboards per project + +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/59974) in GitLab 12.1. + +By default, all projects include a GitLab-defined Prometheus dashboard, which +includes a few key metrics, but you can also define your own custom dashboards. + +You may create a new file from scratch or duplicate a GitLab-defined Prometheus +dashboard. + +NOTE: **Note:** +The metrics as defined below do not support alerts, unlike +[custom metrics](../index.md#adding-custom-metrics). + +### Adding a new dashboard to your project + +> UI option [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/223204) in GitLab 13.2. + +You can configure a custom dashboard by adding a new YAML file into your project's +`.gitlab/dashboards/` directory. In order for the dashboards to be displayed on +the project's **{cloud-gear}** **Operations > Metrics** page, the files must have a `.yml` +extension and should be present in the project's **default** branch. + +To create a new dashboard from the GitLab user interface: + +1. Sign in to GitLab as a user with Maintainer or Owner + [permissions](../../../user/permissions.md#project-members-permissions). +1. Navigate to your dashboard at **{cloud-gear}** **Operations > Metrics**. +1. In the top-right corner of your dashboard, click the **{file-addition-solid}** **Actions** menu, + and select **Create new**: +  +1. In the modal window, click **Open Repository**, then follow the instructions + for creating a new dashboard from the command line. + +To create a new dashboard from the command line: + +1. Create `.gitlab/dashboards/prom_alerts.yml` under your repository's root + directory. Each YAML file should define the layout of the dashboard and the + Prometheus queries used to populate data. This example dashboard displays a + single area chart: + + ```yaml + dashboard: 'Dashboard Title' + panel_groups: + - group: 'Group Title' + panels: + - type: area-chart + title: "Chart Title" + y_label: "Y-Axis" + y_axis: + format: number + precision: 0 + metrics: + - id: my_metric_id + query_range: 'http_requests_total' + label: "Instance: {{instance}}, method: {{method}}" + unit: "count" + ``` + +1. Save the file, commit, and push to your repository. The file must be present in your **default** branch. +1. Navigate to your project's **Operations > Metrics** and choose the custom + dashboard from the dropdown. + +NOTE: **Note:** +Configuration files nested under subdirectories of `.gitlab/dashboards` are not +supported and will not be available in the UI. + +### Navigating to a custom dashboard + +Custom dashboards are uniquely identified by their filenames. In order to quickly view the custom dashboard, +just use the dashboard filename in the URL this way: +`https://gitlab-instance.example.com/project/-/metrics/custom_dashboard_name.yml`. + +### Duplicating a GitLab-defined dashboard + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/37238) in GitLab 12.7. +> - From [GitLab 12.8 onwards](https://gitlab.com/gitlab-org/gitlab/-/issues/39505), custom metrics are also duplicated when you duplicate a dashboard. + +You can save a complete copy of a GitLab defined dashboard along with all custom metrics added to it. +Resulting `.yml` file can be customized and adapted to your project. +You can decide to save the dashboard `.yml` file in the project's **default** branch or in a +new branch. + +1. Click **Duplicate dashboard** in the dashboard dropdown or in the actions menu. + + NOTE: **Note:** + You can duplicate only GitLab-defined dashboards. + +1. Enter the file name and other information, such as the new commit's message, and click **Duplicate**. + +If you select your **default** branch, the new dashboard becomes immediately available. +If you select another branch, this branch should be merged to your **default** branch first. + +## Troubleshooting + +When troubleshooting issues with a managed Prometheus app, it is often useful to +[view the Prometheus UI](../../../development/prometheus.md#access-the-ui-of-a-prometheus-managed-application-in-kubernetes). + +### "No data found" error on Metrics dashboard page + +If the "No data found" screen continues to appear, it could be due to: + +- No successful deployments have occurred to this environment. +- Prometheus does not have performance data for this environment, or the metrics + are not labeled correctly. To test this, connect to the Prometheus server and + [run a query](../../../user/project/integrations/prometheus_library/kubernetes.md#metrics-supported), replacing `$CI_ENVIRONMENT_SLUG` + with the name of your environment. +- You may need to re-add the GitLab predefined common metrics. This can be done by running the [import common metrics Rake task](../../../administration/raketasks/maintenance.md#import-common-metrics). diff --git a/doc/operations/metrics/dashboards/panel_types.md b/doc/operations/metrics/dashboards/panel_types.md new file mode 100644 index 00000000000..0397218fe0e --- /dev/null +++ b/doc/operations/metrics/dashboards/panel_types.md @@ -0,0 +1,262 @@ +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Panel types for dashboards + +The below panel types are supported in monitoring dashboards. + +## Area or Line Chart + +To add an area chart panel type to a dashboard, look at the following sample dashboard file: + +```yaml +dashboard: 'Dashboard Title' +panel_groups: + - group: 'Group Title' + panels: + - type: area-chart # or line-chart + title: 'Area Chart Title' + y_label: "Y-Axis" + y_axis: + format: number + precision: 0 + metrics: + - id: area_http_requests_total + query_range: 'http_requests_total' + label: "Instance: {{instance}}, Method: {{method}}" + unit: "count" +``` + +Note the following properties: + +| Property | Type | Required | Description | +| ------ | ------ | ------ | ------ | +| type | string | no | Type of panel to be rendered. Optional for area panel types | +| query_range | string | required | For area panel types, you must use a [range query](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries) | + + + +Starting in [version 12.8](https://gitlab.com/gitlab-org/gitlab/-/issues/202696), the y-axis values will automatically scale according to the data. Previously, it always started from 0. + +## Anomaly chart + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/16530) in GitLab 12.5. + +To add an anomaly chart panel type to a dashboard, add a panel with *exactly* 3 metrics. + +The first metric represents the current state, and the second and third metrics represent the upper and lower limit respectively: + +```yaml +dashboard: 'Dashboard Title' +panel_groups: + - group: 'Group Title' + panels: + - type: anomaly-chart + title: "Chart Title" + y_label: "Y-Axis" + metrics: + - id: anomaly_requests_normal + query_range: 'http_requests_total' + label: "# of Requests" + unit: "count" + metrics: + - id: anomaly_requests_upper_limit + query_range: 10000 + label: "Max # of requests" + unit: "count" + metrics: + - id: anomaly_requests_lower_limit + query_range: 2000 + label: "Min # of requests" + unit: "count" +``` + +Note the following properties: + +| Property | Type | Required | Description | +| ------ | ------ | ------ | ------ | +| type | string | required | Must be `anomaly-chart` for anomaly panel types | +| query_range | yes | required | For anomaly panel types, you must use a [range query](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries) in every metric. | + + + +## Bar chart + +To add a bar chart to a dashboard, look at the following sample dashboard file: + +```yaml +dashboard: 'Dashboard Title' +panel_groups: + - group: 'Group title' + panels: + - type: bar + title: "Http Handlers" + x_label: 'Response Size' + y_axis: + name: "Handlers" + metrics: + - id: prometheus_http_response_size_bytes_bucket + query_range: "sum(increase(prometheus_http_response_size_bytes_bucket[1d])) by (handler)" + unit: 'Bytes' +``` + +Note the following properties: + +| Property | Type | Required | Description | +| ------ | ------ | ------ | ------ | +| `type` | string | yes | Type of panel to be rendered. For bar chart types, set to `bar` | +| `query_range` | yes | yes | For bar chart, you must use a [range query](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries) + + + +## Column chart + +To add a column panel type to a dashboard, look at the following sample dashboard file: + +```yaml +dashboard: 'Dashboard Title' +panel_groups: + - group: 'Group title' + panels: + - title: "Column" + type: "column" + metrics: + - id: 1024_memory + query: 'avg(sum(container_memory_usage_bytes{container_name!="POD",pod_name=~"^%{ci_environment_slug}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="%{kube_namespace}"}) by (job)) without (job) / count(avg(container_memory_usage_bytes{container_name!="POD",pod_name=~"^%{ci_environment_slug}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="%{kube_namespace}"}) without (job)) /1024/1024' + unit: MB + label: "Memory Usage" +``` + +Note the following properties: + +| Property | Type | Required | Description | +| ------ | ------ | ------ | ------ | +| type | string | yes | Type of panel to be rendered. For column panel types, set to `column` | +| query_range | yes | yes | For column panel types, you must use a [range query](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries) | + + + +## Stacked column + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/30583) in GitLab 12.8. + +To add a stacked column panel type to a dashboard, look at the following sample dashboard file: + +```yaml +dashboard: 'Dashboard title' +priority: 1 +panel_groups: + - group: 'Group Title' + priority: 5 + panels: + - type: 'stacked-column' + title: "Stacked column" + y_label: "y label" + x_label: 'x label' + metrics: + - id: memory_1 + query_range: 'memory_query' + label: "memory query 1" + unit: "count" + series_name: 'group 1' + - id: memory_2 + query_range: 'memory_query_2' + label: "memory query 2" + unit: "count" + series_name: 'group 2' +``` + + + +| Property | Type | Required | Description | +| ------ | ------ | ------ | ------ | +| `type` | string | yes | Type of panel to be rendered. For stacked column panel types, set to `stacked-column` | +| `query_range` | yes | yes | For stacked column panel types, you must use a [range query](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries) | + +## Single Stat + +To add a single stat panel type to a dashboard, look at the following sample dashboard file: + +```yaml +dashboard: 'Dashboard Title' +panel_groups: + - group: 'Group Title' + panels: + - title: "Single Stat" + type: "single-stat" + metrics: + - id: 10 + query: 'max(go_memstats_alloc_bytes{job="prometheus"})' + unit: MB + label: "Total" +``` + +Note the following properties: + +| Property | Type | Required | Description | +| ------ | ------ | ------ | ------ | +| type | string | yes | Type of panel to be rendered. For single stat panel types, set to `single-stat` | +| field | string | no | Panels display the value of a metric. For a panel to display the value of a label instead, put the name of the label in this key. | +| query | string | yes | For single stat panel types, you must use an [instant query](https://prometheus.io/docs/prometheus/latest/querying/api/#instant-queries) | + + + +## Percentile based results + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/201946) in GitLab 12.8. + +Query results sometimes need to be represented as a percentage value out of 100. You can use the `max_value` property at the root of the panel definition: + +```yaml +dashboard: 'Dashboard Title' +panel_groups: + - group: 'Group Title' + panels: + - title: "Single Stat" + type: "single-stat" + max_value: 100 + metrics: + - id: 10 + query: 'max(go_memstats_alloc_bytes{job="prometheus"})' + unit: '%' + label: "Total" +``` + +For example, if you have a query value of `53.6`, adding `%` as the unit results in a single stat value of `53.6%`, but if the maximum expected value of the query is `120`, the value would be `44.6%`. Adding the `max_value` causes the correct percentage value to display. + +## Heatmaps + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/30581) in GitLab 12.5. + +To add a heatmap panel type to a dashboard, look at the following sample dashboard file: + +```yaml +dashboard: 'Dashboard Title' +panel_groups: + - group: 'Group Title' + panels: + - title: "Heatmap" + type: "heatmap" + metrics: + - id: 10 + query: 'sum(rate(nginx_upstream_responses_total{upstream=~"%{kube_namespace}-%{ci_environment_slug}-.*"}[60m])) by (status_code)' + unit: req/sec + label: "Status code" +``` + +Note the following properties: + +| Property | Type | Required | Description | +| ------ | ------ | ------ | ------ | +| type | string | yes | Type of panel to be rendered. For heatmap panel types, set to `heatmap` | +| query_range | yes | yes | For area panel types, you must use a [range query](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries) | + + + +CAUTION: **Warning:** +When a query returns too many data points, the heatmap data bucket dimensions tend downwards to 0, making the chart's data invisible, as shown in the image below. To fix this problem, limit the amount of data returned by changing the time range filter on the metrics dashboard UI, or adding the **step** property to your dashboard's YAML file. + + diff --git a/doc/operations/metrics/dashboards/templating_variables.md b/doc/operations/metrics/dashboards/templating_variables.md new file mode 100644 index 00000000000..a515742ea92 --- /dev/null +++ b/doc/operations/metrics/dashboards/templating_variables.md @@ -0,0 +1,128 @@ +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Templating variables for metrics dashboards + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/214539) in GitLab 13.0. + +Templating variables can be used to make your metrics dashboard more versatile. + +`templating` is a top-level key in the +[dashboard YAML](yaml.md#dashboard-top-level-properties). +Define your variables in the `variables` key, under `templating`. The value of +the `variables` key should be a hash, and each key under `variables` +defines a templating variable on the dashboard, and may contain alphanumeric and underscore characters. + +A variable can be used in a Prometheus query in the same dashboard using the syntax +described [in Using Variables](variables.md). + +## `text` variable type + +CAUTION: **Warning:** +This variable type is an _alpha_ feature, and is subject to change at any time +without prior notice! + +For each `text` variable defined in the dashboard YAML, there will be a free text +box on the dashboard UI, allowing you to enter a value for each variable. + +The `text` variable type supports a simple and a full syntax. + +### Simple syntax + +This example creates a variable called `variable1`, with a default value +of `default value`: + +```yaml +templating: + variables: + variable1: 'default value' # `text` type variable with `default value` as its default. +``` + +### Full syntax + +This example creates a variable called `variable1`, with a default value of `default`. +The label for the text box on the UI will be the value of the `label` key: + +```yaml +templating: + variables: + variable1: # The variable name that can be used in queries. + label: 'Variable 1' # (Optional) label that will appear in the UI for this text box. + type: text + options: + default_value: 'default' # (Optional) default value. +``` + +## `custom` variable type + +CAUTION: **Warning:** +This variable type is an _alpha_ feature, and is subject to change at any time +without prior notice! + +Each `custom` variable defined in the dashboard YAML creates a dropdown +selector on the dashboard UI, allowing you to select a value for each variable. + +The `custom` variable type supports a simple and a full syntax. + +### Simple syntax + +This example creates a variable called `variable1`, with a default value of `value1`. +The dashboard UI will display a dropdown with `value1`, `value2` and `value3` +as the choices. + +```yaml +templating: + variables: + variable1: ['value1', 'value2', 'value3'] +``` + +### Full syntax + +This example creates a variable called `variable1`, with a default value of `value_option_2`. +The label for the text box on the UI will be the value of the `label` key. +The dashboard UI will display a dropdown with `Option 1` and `Option 2` +as the choices. + +If you select `Option 1` from the dropdown, the variable will be replaced with `value option 1`. +Similarly, if you select `Option 2`, the variable will be replaced with `value_option_2`: + +```yaml +templating: + variables: + variable1: # The variable name that can be used in queries. + label: 'Variable 1' # (Optional) label that will appear in the UI for this dropdown. + type: custom + options: + values: + - value: 'value option 1' # The value that will replace the variable in queries. + text: 'Option 1' # (Optional) Text that will appear in the UI dropdown. + - value: 'value_option_2' + text: 'Option 2' + default: true # (Optional) This option should be the default value of this variable. +``` + +## `metric_label_values` variable type + +CAUTION: **Warning:** +This variable type is an _alpha_ feature, and is subject to change at any time +without prior notice! + +### Full syntax + +This example creates a variable called `variable2`. The values of the dropdown will +be all the different values of the `backend` label in the Prometheus series described by +`up{env="production"}`. + +```yaml +templating: + variables: + variable2: # The variable name that can be interpolated in queries. + label: 'Variable 2' # (Optional) label that will appear in the UI for this dropdown. + type: metric_label_values + options: + series_selector: 'up{env="production"}' + label: 'backend' +``` diff --git a/doc/operations/metrics/dashboards/variables.md b/doc/operations/metrics/dashboards/variables.md new file mode 100644 index 00000000000..19b77a1ed87 --- /dev/null +++ b/doc/operations/metrics/dashboards/variables.md @@ -0,0 +1,59 @@ +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Using Variables + +## Query Variables + +Variables can be specified using double curly braces, such as `"{{ci_environment_slug}}"` ([added](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/20793) in GitLab 12.7). + +Support for the `"%{ci_environment_slug}"` format was +[removed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/31581) in GitLab 13.0. +Queries that continue to use the old format will show no data. + +## Predefined variables + +GitLab supports a limited set of [CI variables](../../../ci/variables/README.md) in the Prometheus query. This is particularly useful for identifying a specific environment, for example with `ci_environment_slug`. The supported variables are: + +- `ci_environment_slug` +- `kube_namespace` +- `ci_project_name` +- `ci_project_namespace` +- `ci_project_path` +- `ci_environment_name` +- `__range` + +NOTE: **Note:** +Variables for Prometheus queries must be lowercase. + +### __range + +The `__range` variable is useful in Prometheus +[range vector selectors](https://prometheus.io/docs/prometheus/latest/querying/basics/#range-vector-selectors). +Its value is the total number of seconds in the dashboard's time range. +For example, if the dashboard time range is set to 8 hours, the value of +`__range` is `28800s`. + +## User-defined variables + +[Variables can be defined](../../../operations/metrics/dashboards/yaml.md#templating-templating-properties) in a custom dashboard YAML file. + +## Query Variables from URL + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/214500) in GitLab 13.0. + +GitLab supports setting custom variables through URL parameters. Surround the variable +name with double curly braces (`{{example}}`) to interpolate the variable in a query: + +```plaintext +avg(sum(container_memory_usage_bytes{container_name!="{{pod}}"}) by (job)) without (job) /1024/1024/1024' +``` + +The URL for this query would be: + +```plaintext +http://gitlab.com/<user>/<project>/-/environments/<environment_id>/metrics?dashboard=.gitlab%2Fdashboards%2Fcustom.yml&pod=POD +``` diff --git a/doc/operations/metrics/dashboards/yaml.md b/doc/operations/metrics/dashboards/yaml.md new file mode 100644 index 00000000000..6a4158798bc --- /dev/null +++ b/doc/operations/metrics/dashboards/yaml.md @@ -0,0 +1,166 @@ +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Dashboard YAML properties + +Dashboards have several components: + +- Templating variables. +- Panel groups, which consist of panels. +- Panels, which support one or more metrics. + +The following tables outline the details of expected properties. + +## **Dashboard (top-level) properties** + +| Property | Type | Required | Description | +| ------ | ------ | ------ | ------ | +| `dashboard` | string | yes | Heading for the dashboard. Only one dashboard should be defined per file. | +| `panel_groups` | array | yes | The panel groups which should be on the dashboard. | +| `templating` | hash | no | Top level key under which templating related options can be added. | +| `links` | array | no | Add links to display on the dashboard. | + +## **Templating (`templating`) properties** + +| Property | Type | Required | Description | +| -------- | ---- | -------- | ----------- | +| `variables` | hash | yes | Variables can be defined here. | + +Read the documentation on [templating](templating_variables.md). + +## **Links (`links`) properties** + +| Property | Type | Required | Description | +| -------- | ---- | -------- | ----------- | +| `url` | string | yes | The address of the link. | +| `title` | string | no | Display title for the link. | +| `type` | string | no | Type of the link. Specifies the link type, can be: `grafana` | + +Read the documentation on [links](index.md#add-related-links-to-custom-dashboards). + +## **Panel group (`panel_groups`) properties** + +| Property | Type | Required | Description | +| ------ | ------ | ------ | ------ | +| `group` | string | required | Heading for the panel group. | +| `priority` | number | optional, defaults to order in file | Order to appear on the dashboard. Higher number means higher priority, which will be higher on the page. Numbers do not need to be consecutive. | +| `panels` | array | required | The panels which should be in the panel group. | + +Panels in a panel group are laid out in rows consisting of two panels per row. An exception to this rule are single panels on a row: these panels will take the full width of their containing row. + +## **Panel (`panels`) properties** + +| Property | Type | Required | Description | +| ------ | ------ | ------ | ------- | +| `type` | enum | no, defaults to `area-chart` | Specifies the panel type to use, for example `area-chart`, `line-chart` or `anomaly-chart`. [View documentation on all panel types.](panel_types.md) | +| `title` | string | yes | Heading for the panel. | +| `y_label` | string | no, but highly encouraged | Y-Axis label for the panel. | +| `y_axis` | string | no | Y-Axis configuration for the panel. | +| `max_value` | number | no | Denominator value used for calculating [percentile based results](panel_types.md#percentile-based-results) | +| `weight` | number | no, defaults to order in file | Order to appear within the grouping. Lower number means higher priority, which will be higher on the page. Numbers do not need to be consecutive. | +| `metrics` | array | yes | The metrics which should be displayed in the panel. Any number of metrics can be displayed when `type` is `area-chart` or `line-chart`, whereas only 3 can be displayed when `type` is `anomaly-chart`. | +| `links` | array | no | Add links to display on the chart's [context menu](index.md#chart-context-menu). | + +## **Axis (`panels[].y_axis`) properties** + +| Property | Type | Required | Description | +| ----------- | ------ | ----------------------------- | -------------------------------------------------------------------- | +| `name` | string | no, but highly encouraged | Y-Axis label for the panel. Replaces `y_label` if set. | +| `format` | string | no, defaults to `engineering` | Unit format used. See the [full list of units](yaml_number_format.md). | +| `precision` | number | no, defaults to `2` | Number of decimal places to display in the number. | | + +## **Metrics (`metrics`) properties** + +| Property | Type | Required | Description | +| ------ | ------ | ------ | ------ | +| `id` | string | no | Used for associating dashboard metrics with database records. Must be unique across dashboard configuration files. Required for [alerting](../alerts.md) (support not yet enabled, see [relevant issue](https://gitlab.com/gitlab-org/gitlab/-/issues/27980)). | +| `unit` | string | yes | Defines the unit of the query's return data. | +| `label` | string | no, but highly encouraged | Defines the legend-label for the query. Should be unique within the panel's metrics. Can contain time series labels as interpolated variables. | +| `query` | string | yes if `query_range` is not defined | Defines the Prometheus query to be used to populate the chart/panel. If defined, the `query` endpoint of the [Prometheus API](https://prometheus.io/docs/prometheus/latest/querying/api/) will be utilized. | +| `query_range` | string | yes if `query` is not defined | Defines the Prometheus query to be used to populate the chart/panel. If defined, the `query_range` endpoint of the [Prometheus API](https://prometheus.io/docs/prometheus/latest/querying/api/) will be utilized. | +| `step` | number | no, value is calculated if not defined | Defines query resolution step width in float number of seconds. Metrics on the same panel should use the same `step` value. | + +## Dynamic labels + +Dynamic labels are useful when multiple time series are returned from a Prometheus query. + +When a static label is used and a query returns multiple time series, then all the legend items will be labeled the same, which makes identifying each time series difficult: + +```yaml +metrics: + - id: my_metric_id + query_range: 'http_requests_total' + label: "Time Series" + unit: "count" +``` + +This may render a legend like this: + + + +For labels to be more explicit, using variables that reflect time series labels is a good practice. The variables will be replaced by the values of the time series labels when the legend is rendered: + +```yaml +metrics: + - id: my_metric_id + query_range: 'http_requests_total' + label: "Instance: {{instance}}, method: {{method}}" + unit: "count" +``` + +The resulting rendered legend will look like this: + + + +There is also a shorthand value for dynamic dashboard labels that make use of only one time series label: + +```yaml +metrics: + - id: my_metric_id + query_range: 'http_requests_total' + label: "Method" + unit: "count" +``` + +This works by lowercasing the value of `label` and, if there are more words separated by spaces, replacing those spaces with an underscore (`_`). The transformed value is then checked against the labels of the time series returned by the Prometheus query. If a time series label is found that is equal to the transformed value, then the label value will be used and rendered in the legend like this: + + + +## Dashboard YAML syntax validation + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/33202) in GitLab 13.1. + +To confirm your dashboard definition contains valid YAML syntax: + +1. Navigate to **{doc-text}** **Repository > Files**. +1. Navigate to your dashboard file in your repository. +1. Review the information pane about the file, displayed above the file contents. + +Files with valid syntax display **Metrics Dashboard YAML definition is valid**, +and files with invalid syntax display **Metrics Dashboard YAML definition is invalid**. + + + +When **Metrics Dashboard YAML definition is invalid** at least one of the following messages is displayed: + +1. `dashboard: can't be blank` [learn more](../../../operations/metrics/dashboards/yaml.md#dashboard-top-level-properties) +1. `panel_groups: should be an array of panel_groups objects` [learn more](../../../operations/metrics/dashboards/yaml.md#dashboard-top-level-properties) +1. `group: can't be blank` [learn more](../../../operations/metrics/dashboards/yaml.md#panel-group-panel_groups-properties) +1. `panels: should be an array of panels objects` [learn more](../../../operations/metrics/dashboards/yaml.md#panel-group-panel_groups-properties) +1. `title: can't be blank` [learn more](../../../operations/metrics/dashboards/yaml.md#panel-panels-properties) +1. `metrics: should be an array of metrics objects` [learn more](../../../operations/metrics/dashboards/yaml.md#panel-panels-properties) +1. `query: can't be blank` [learn more](../../../operations/metrics/dashboards/yaml.md#metrics-metrics-properties) +1. `query_range: can't be blank` [learn more](../../../operations/metrics/dashboards/yaml.md#metrics-metrics-properties) +1. `unit: can't be blank` [learn more](../../../operations/metrics/dashboards/yaml.md#metrics-metrics-properties) +1. `YAML syntax: The parsed YAML is too big` + + This is displayed when the YAML file is larger than 1 MB. + +1. `YAML syntax: Invalid configuration format` + + This is displayed when the YAML file is empty or does not contain valid YAML. + +Metrics Dashboard YAML definition validation information is also available as a [GraphQL API field](../../../api/graphql/reference/index.md#metricsdashboard) diff --git a/doc/operations/metrics/dashboards/yaml_number_format.md b/doc/operations/metrics/dashboards/yaml_number_format.md new file mode 100644 index 00000000000..ae0cd9fff64 --- /dev/null +++ b/doc/operations/metrics/dashboards/yaml_number_format.md @@ -0,0 +1,177 @@ +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Unit formats reference + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/201999) in GitLab 12.9. + +Format the data in your dashboard panels. + +You can select units to format your charts by adding `format` to your +[axis configuration](yaml.md). + +## Internationalization and localization + +Currently, your [internationalization and localization options](https://en.wikipedia.org/wiki/Internationalization_and_localization) for number formatting are dependent on the system you are using i.e. your OS or browser. + +## Engineering Notation + +For generic or default data, numbers are formatted according to the current locale in [engineering notation](https://en.wikipedia.org/wiki/Engineering_notation). + +While an [engineering notation exists for the web](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Intl/NumberFormat), GitLab uses a version based off the [scientific notation](https://en.wikipedia.org/wiki/Scientific_notation). GitLab formatting acts in accordance with SI prefixes. For example, using GitLab notation, `1500.00` becomes `1.5k` instead of `1.5E3`. Keep this distinction in mind when using the engineering notation for your metrics. + +Formats: `engineering` + +SI prefixes: + +| Name | Symbol | Value | +| ---------- | ------- | -------------------------- | +| `yotta` | Y | 1000000000000000000000000 | +| `zetta` | Z | 1000000000000000000000 | +| `exa` | E | 1000000000000000000 | +| `peta` | P | 1000000000000000 | +| `tera` | T | 1000000000000 | +| `giga` | G | 1000000000 | +| `mega` | M | 1000000 | +| `kilo` | k | 1000 | +| `milli` | m | 0.001 | +| `micro` | μ | 0.000001 | +| `nano` | n | 0.000000001 | +| `pico` | p | 0.000000000001 | +| `femto` | f | 0.000000000000001 | +| `atto` | a | 0.000000000000000001 | +| `zepto` | z | 0.000000000000000000001 | +| `yocto` | y | 0.000000000000000000000001 | + +**Examples:** + +| Data | Displayed | +| --------------------------------- | --------- | +| `0.000000000000000000000008` | 8y | +| `0.000000000000000000008` | 8z | +| `0.000000000000000008` | 8a | +| `0.000000000000008` | 8f | +| `0.000000000008` | 8p | +| `0.000000008` | 8n | +| `0.000008` | 8μ | +| `0.008` | 8m | +| `10` | 10 | +| `1080` | 1.08k | +| `18000` | 18k | +| `18888` | 18.9k | +| `188888` | 189k | +| `18888888` | 18.9M | +| `1888888888` | 1.89G | +| `1888888888888` | 1.89T | +| `1888888888888888` | 1.89P | +| `1888888888888888888` | 1.89E | +| `1888888888888888888888` | 1.89Z | +| `1888888888888888888888888` | 1.89Y | +| `1888888888888888888888888888` | 1.89e+27 | + +## Numbers + +For number data, numbers are formatted according to the current locale. + +Formats: `number` + +**Examples:** + +| Data | Displayed | +| ---------- | --------- | +| `10` | 1 | +| `1000` | 1,000 | +| `1000000` | 1,000,000 | + +## Percentage + +For percentage data, format numbers in the chart with a `%` symbol. + +Formats supported: `percent`, `percentHundred` + +**Examples:** + +| Format | Data | Displayed | +| ---------------- | ----- | --------- | +| `percent` | `0.5` | 50% | +| `percent` | `1` | 100% | +| `percent` | `2` | 200% | +| `percentHundred` | `50` | 50% | +| `percentHundred` | `100` | 100% | +| `percentHundred` | `200` | 200% | + +## Duration + +For time durations, format numbers in the chart with a time unit symbol. + +Formats supported: `milliseconds`, `seconds` + +**Examples:** + +| Format | Data | Displayed | +| -------------- | ------ | --------- | +| `milliseconds` | `10` | 10ms | +| `milliseconds` | `500` | 100ms | +| `milliseconds` | `1000` | 1000ms | +| `seconds` | `10` | 10s | +| `seconds` | `500` | 500s | +| `seconds` | `1000` | 1000s | + +## Digital (Metric) + +Converts a number of bytes using metric prefixes. It scales to +use the unit that's the best fit. + +Formats supported: + +- `decimalBytes` +- `kilobytes` +- `megabytes` +- `gigabytes` +- `terabytes` +- `petabytes` + +**Examples:** + +| Format | Data | Displayed | +| -------------- | --------- | --------- | +| `decimalBytes` | `1` | 1B | +| `decimalBytes` | `1000` | 1kB | +| `decimalBytes` | `1000000` | 1MB | +| `kilobytes` | `1` | 1kB | +| `kilobytes` | `1000` | 1MB | +| `kilobytes` | `1000000` | 1GB | +| `megabytes` | `1` | 1MB | +| `megabytes` | `1000` | 1GB | +| `megabytes` | `1000000` | 1TB | + +## Digital (IEC) + +Converts a number of bytes using binary prefixes. It scales to +use the unit that's the best fit. + +Formats supported: + +- `bytes` +- `kibibytes` +- `mebibytes` +- `gibibytes` +- `tebibytes` +- `pebibytes` + +**Examples:** + +| Format | Data | Displayed | +| ----------- | ------------- | --------- | +| `bytes` | `1` | 1B | +| `bytes` | `1024` | 1KiB | +| `bytes` | `1024 * 1024` | 1MiB | +| `kibibytes` | `1` | 1KiB | +| `kibibytes` | `1024` | 1MiB | +| `kibibytes` | `1024 * 1024` | 1GiB | +| `mebibytes` | `1` | 1MiB | +| `mebibytes` | `1024` | 1GiB | +| `mebibytes` | `1024 * 1024` | 1TiB | diff --git a/doc/operations/metrics/embed.md b/doc/operations/metrics/embed.md new file mode 100644 index 00000000000..5ee9b0859b9 --- /dev/null +++ b/doc/operations/metrics/embed.md @@ -0,0 +1,93 @@ +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Embedding metric charts within GitLab Flavored Markdown + +## Embedding GitLab-managed Kubernetes metrics + +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/29691) in GitLab 12.2. + +It is possible to display metrics charts within [GitLab Flavored Markdown](../../user/markdown.md#gitlab-flavored-markdown-gfm) fields such as issue or merge request descriptions. The maximum number of embedded charts allowed in a GitLab Flavored Markdown field is 100. + +This can be useful if you are sharing an application incident or performance +metrics to others and want to have relevant information directly available. + +NOTE: **Note:** +Requires [Kubernetes](../../user/project/integrations/prometheus_library/kubernetes.md) metrics. + +To display metric charts, include a link of the form `https://<root_url>/<project>/-/environments/<environment_id>/metrics`: + + + +GitLab unfurls the link as an embedded metrics panel: + + + +You can also embed a single chart. To get a link to a chart, click the +**{ellipsis_v}** **More actions** menu in the upper right corner of the chart, +and select **Copy link to chart**, as shown in this example: + + + +The following requirements must be met for the metric to unfurl: + +- The `<environment_id>` must correspond to a real environment. +- Prometheus must be monitoring the environment. +- The GitLab instance must be configured to receive data from the environment. +- The user must be allowed access to the monitoring dashboard for the environment ([Reporter or higher](../../user/permissions.md)). +- The dashboard must have data within the last 8 hours. + + If all of the above are true, then the metric will unfurl as seen below: + + + +Metric charts may also be hidden: + + + +You can open the link directly into your browser for a [detailed view of the data](dashboards/index.md#expand-panel). + +## Embedding metrics in issue templates + +It is also possible to embed either the default dashboard metrics or individual metrics in issue templates. For charts to render side-by-side, links to the entire metrics dashboard or individual metrics should be separated by either a comma or a space. + + + +## Embedding metrics based on alerts in incident issues + +For [GitLab-managed alerting rules](alerts.md), the issue will include an embedded chart for the query corresponding to the alert. The chart displays an hour of data surrounding the starting point of the incident, 30 minutes before and after. + +For [manually configured Prometheus instances](../../user/project/integrations/prometheus.md#manual-configuration-of-prometheus), a chart corresponding to the query can be included if these requirements are met: + +- The alert corresponds to an environment managed through GitLab. +- The alert corresponds to a [range query](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries). +- The alert contains the required attributes listed in the chart below. + +| Attributes | Required | Description | +| ---------- | -------- | ----------- | +| `annotations/gitlab_environment_name` | Yes | Name of the GitLab-managed environment corresponding to the alert | +| One of `annotations/title`, `annotations/summary`, `labels/alertname` | Yes | Will be used as the chart title | +| `annotations/gitlab_y_label` | No | Will be used as the chart's y-axis label | + +## Embedding cluster health charts + +> - [Introduced](<https://gitlab.com/gitlab-org/gitlab/-/issues/40997>) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.9. +> - [Moved](<https://gitlab.com/gitlab-org/gitlab/-/issues/208224>) to GitLab core in 13.2. + +[Cluster Health Metrics](../../user/project/clusters/index.md#visualizing-cluster-health) can also be embedded in [GitLab-flavored Markdown](../../user/markdown.md). + +To embed a metric chart, include a link to that chart in the form `https://<root_url>/<project>/-/cluster/<cluster_id>?<query_params>` anywhere that GitLab-flavored Markdown is supported. To generate and copy a link to the chart, follow the instructions in the [Cluster Health Metric documentation](../../user/project/clusters/index.md#visualizing-cluster-health). + +The following requirements must be met for the metric to unfurl: + +- The `<cluster_id>` must correspond to a real cluster. +- Prometheus must be monitoring the cluster. +- The user must be allowed access to the project cluster metrics. +- The dashboards must be reporting data on the [Cluster Health Page](../../user/project/clusters/index.md#visualizing-cluster-health) + + If the above requirements are met, then the metric will unfurl as seen below. + + diff --git a/doc/operations/metrics/embed_grafana.md b/doc/operations/metrics/embed_grafana.md new file mode 100644 index 00000000000..427ad866442 --- /dev/null +++ b/doc/operations/metrics/embed_grafana.md @@ -0,0 +1,65 @@ +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Embedding Grafana charts + +Grafana metrics can be embedded in [GitLab Flavored Markdown](../../user/markdown.md). + +## Embedding charts via Grafana Rendered Images + +It is possible to embed live [Grafana](https://docs.gitlab.com/omnibus/settings/grafana.html) charts in issues, as a [direct linked rendered image](https://grafana.com/docs/grafana/latest/reference/share_panel/#direct-link-rendered-image). + +The sharing dialog within Grafana provides the link, as highlighted below. + + + +NOTE: **Note:** +For this embed to display correctly, the Grafana instance must be available to the target user, either as a public dashboard, or on the same network. + +Copy the link and add an image tag as [inline HTML](../../user/markdown.md#inline-html) in your Markdown. You may tweak the query parameters as required. For instance, removing the `&from=` and `&to=` parameters will give you a live chart. Here is example markup for a live chart from GitLab's public dashboard: + +```html +<img src="https://dashboards.gitlab.com/d/RZmbBr7mk/gitlab-triage?orgId=1&refresh=30s&var-env=gprd&var-environment=gprd&var-prometheus=prometheus-01-inf-gprd&var-prometheus_app=prometheus-app-01-inf-gprd&var-backend=All&var-type=All&var-stage=main&from=1580444107655&to=1580465707655"/> +``` + +This will render like so: + + + +## Embedding charts via integration with Grafana HTTP API + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/31376) in GitLab 12.5. + +Each project can support integration with one Grafana instance. This configuration allows a user to copy a link to a panel in Grafana, then paste it into a GitLab Markdown field. The chart will be rendered in the GitLab chart format. + +Prerequisites for embedding from a Grafana instance: + +1. The datasource must be a Prometheus instance. +1. The datasource must be proxyable, so the HTTP Access setting should be set to `Server`. + + + +## Setting up the Grafana integration + +1. [Generate an Admin-level API Token in Grafana.](https://grafana.com/docs/grafana/latest/http_api/auth/#create-api-token) +1. In your GitLab project, navigate to **Settings > Operations > Grafana Authentication**. +1. To enable the integration, check the "Active" checkbox. +1. For "Grafana URL", enter the base URL of the Grafana instance. +1. For "API Token", enter the Admin API Token you just generated. +1. Click **Save Changes**. + +## Generating a link to a chart + +1. In Grafana, navigate to the dashboard you wish to embed a panel from. +  +1. In the upper-left corner of the page, select a specific value for each variable required for the queries in the chart. +  +1. In Grafana, click on a panel's title, then click **Share** to open the panel's sharing dialog to the **Link** tab. If you click the _dashboard's_ share panel instead, GitLab will attempt to embed the first supported panel on the dashboard (if available). +1. If your Prometheus queries use Grafana's custom template variables, ensure that the "Template variables" option is toggled to **On**. Of Grafana global template variables, only `$__interval`, `$__from`, and `$__to` are currently supported. Toggle **On** the "Current time range" option to specify the time range of the chart. Otherwise, the default range will be the last 8 hours. +  +1. Click **Copy** to copy the URL to the clipboard. +1. In GitLab, paste the URL into a Markdown field and save. The chart will take a few moments to render. +  diff --git a/doc/operations/metrics/img/example-dashboard_v13_1.png b/doc/operations/metrics/img/example-dashboard_v13_1.png Binary files differnew file mode 100644 index 00000000000..0cda4ece689 --- /dev/null +++ b/doc/operations/metrics/img/example-dashboard_v13_1.png diff --git a/doc/operations/metrics/index.md b/doc/operations/metrics/index.md new file mode 100644 index 00000000000..12088884f44 --- /dev/null +++ b/doc/operations/metrics/index.md @@ -0,0 +1,142 @@ +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Monitor metrics for your CI/CD environment + +After [configuring Prometheus for a cluster](../../user/project/integrations/prometheus.md), +GitLab attempts to retrieve performance metrics for any [environment](../../ci/environments/index.md) with +a successful deployment. + +GitLab scans the Prometheus server for metrics from known servers like Kubernetes +and NGINX, and attempts to identify individual environments. To learn more about +the supported metrics and scan processes, see the +[Prometheus Metrics Library documentation](../../user/project/integrations/prometheus_library/index.md). + +To view the metrics dashboard for an environment that has +[completed at least one deployment](#populate-your-metrics-dashboard): + +1. *If the metrics dashboard is only visible to project members,* sign in to + GitLab as a member of a project. Learn more about [metrics dashboard visibility](#metrics-dashboard-visibility). +1. In your project, navigate to **{cloud-gear}** **Operations > Metrics**. + +GitLab displays the default metrics dashboard for the environment, like the +following example: + + + +The top of the dashboard contains a navigation bar. From left to right, the +navigation bar contains: + +- **Dashboard** - A dropdown list of all dashboards available for the project, + with starred dashboards listed first. +- **Environment** - A dropdown list of all [environments](../index.md) that searches + through all environments as you type. +- **Range** - The time period of data to display. +- **Refresh dashboard** **{retry}** - Reload the dashboard with current data. +- **Set refresh rate** - Set a time frame for refreshing the data displayed. +- **Star dashboard** **{star-o}** - Click to mark a dashboard as a favorite. + Starred dashboards display a solid star **{star}** button, and display first + in the **Dashboard** dropdown list. + ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/214582) in GitLab 13.0.) +- **Create dashboard** **{file-addition-solid}** - Create a + [new custom dashboard for your project](dashboards/index.md#adding-a-new-dashboard-to-your-project). +- **Metrics settings** **{settings}** - Configure the + [settings for this dashboard](dashboards/index.md#manage-the-metrics-dashboard-settings). + +## Populate your metrics dashboard + +After [configuring Prometheus for a cluster](../../user/project/integrations/prometheus.md), +you must also deploy code for the **{cloud-gear}** **Operations > Metrics** page +to contain data. Setting up [Auto DevOps](../../topics/autodevops/index.md) +helps quickly create a deployment: + +1. Navigate to your project's **{cloud-gear}** **Operations > Kubernetes** page. +1. Ensure that, in addition to Prometheus, you also have Runner and Ingress + installed. +1. After installing Ingress, copy its endpoint. +1. Navigate to your project's **{settings}** **Settings > CI/CD** page. In the + **Auto DevOps** section, select a deployment strategy and save your changes. +1. On the same page, in the **Variables** section, add a variable named + `KUBE_INGRESS_BASE_DOMAIN` with the value of the Ingress endpoint you + copied previously. Leave the type as **Variable**. +1. Navigate to your project's **{rocket}** **CI/CD > Pipelines** page, and run a + pipeline on any branch. +1. When the pipeline has run successfully, graphs are available on the + **{cloud-gear}** **Operations > Metrics** page. + + + +## Customize your metrics dashboard + +After creating your dashboard, you can customize it to meet your needs: + +- **Add custom metrics**: In addition to the GitLab default metrics, you can + [create custom metrics](#adding-custom-metrics) and display them on your metrics dashboard. +- **Configure alerts for metrics**: [Configure custom alerts](alerts.md) for your team when + environment performance falls outside of the boundaries you set. +- **Trigger actions from alerts**: [Open new issues for your team](alerts.md#trigger-actions-from-alerts-ultimate) **(ULTIMATE)** + when environment performance falls outside of the boundaries you set. + +## Metrics dashboard visibility + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/201924) in GitLab 13.0. + +You can set the visibility of the metrics dashboard to **Only Project Members** +or **Everyone With Access**. When set to **Everyone with Access**, the metrics +dashboard is visible to authenticated and non-authenticated users. + +## Adding custom metrics + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/3799) in [GitLab Premium](https://about.gitlab.com/pricing/) 10.6. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/28527) to [GitLab Core](https://about.gitlab.com/pricing/) 12.10. + +Custom metrics can be monitored by adding them on the monitoring dashboard page. +After saving them, they display on the environment metrics dashboard provided that either: + +- A [connected Kubernetes cluster](../../user/project/clusters/add_remove_clusters.md) + with the matching [environment scope](../../ci/environments/index.md#scoping-environments-with-specs) is used and + [Prometheus installed on the cluster](../../user/project/integrations/prometheus.md#enabling-prometheus-integration). +- Prometheus is [manually configured](../../user/project/integrations/prometheus.md#manual-configuration-of-prometheus). + + + +A few fields are required: + +- **Name**: Chart title +- **Type**: Type of metric. Metrics of the same type will be shown together. +- **Query**: Valid [PromQL query](https://prometheus.io/docs/prometheus/latest/querying/basics/). +- **Y-axis label**: Y axis title to display on the dashboard. +- **Unit label**: Query units, for example `req / sec`. Shown next to the value. + +Multiple metrics can be displayed on the same chart if the fields **Name**, **Type**, +and **Y-axis label** match between metrics. For example, a metric with **Name** +`Requests Rate`, **Type** `Business`, and **Y-axis label** `rec / sec` would display +on the same chart as a second metric with the same values. A **Legend label** is +suggested if this feature is used. + +## Editing additional metrics from the dashboard + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/208976) in GitLab 12.9. + +You can edit existing additional custom metrics for your dashboard by clicking the +**{ellipsis_v}** **More actions** dropdown and selecting **Edit metric**. + + + +## Keyboard shortcuts for charts + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/202146) in GitLab 13.2. + +You can use keyboard shortcuts to interact more quickly with your currently-focused +chart panel. To activate keyboard shortcuts, use keyboard tabs to highlight the +**{ellipsis_v}** **More actions** dropdown menu, or hover over the dropdown menu +with your mouse, then press the key corresponding to your desired action: + +- **Expand panel** - <kbd>e</kbd> +- **View logs** - <kbd>l</kbd> (lowercase 'L') +- **Download CSV** - <kbd>d</kbd> +- **Copy link to chart** - <kbd>c</kbd> +- **Alerts** - <kbd>a</kbd> diff --git a/doc/operations/tracing.md b/doc/operations/tracing.md new file mode 100644 index 00000000000..07f60c37f1b --- /dev/null +++ b/doc/operations/tracing.md @@ -0,0 +1,40 @@ +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Tracing **(ULTIMATE)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/7903) in GitLab Ultimate 11.5. + +Tracing provides insight into the performance and health of a deployed application, +tracking each function or microservice which handles a given request. + +This makes it easy to +understand the end-to-end flow of a request, regardless of whether you are using a monolithic or distributed system. + +## Jaeger tracing + +[Jaeger](https://www.jaegertracing.io/) is an open source, end-to-end distributed +tracing system used for monitoring and troubleshooting microservices-based distributed +systems. + +### Deploying Jaeger + +To learn more about deploying Jaeger, read the official +[Getting Started documentation](https://www.jaegertracing.io/docs/latest/getting-started/). +There is an easy to use [all-in-one Docker image](https://www.jaegertracing.io/docs/latest/getting-started/#AllinoneDockerimage), +as well as deployment options for [Kubernetes](https://github.com/jaegertracing/jaeger-kubernetes) +and [OpenShift](https://github.com/jaegertracing/jaeger-openshift). + +### Enabling Jaeger + +GitLab provides an easy way to open the Jaeger UI from within your project: + +1. [Set up Jaeger](https://www.jaegertracing.io) and configure your application using one of the + [client libraries](https://www.jaegertracing.io/docs/latest/client-libraries/). +1. Navigate to your project's **Settings > Operations** and provide the Jaeger URL. +1. Click **Save changes** for the changes to take effect. +1. You can now visit **Operations > Tracing** in your project's sidebar and + GitLab will redirect you to the configured Jaeger URL. diff --git a/doc/policy/maintenance.md b/doc/policy/maintenance.md index bba16e491d0..5b3fc3e0021 100644 --- a/doc/policy/maintenance.md +++ b/doc/policy/maintenance.md @@ -2,15 +2,19 @@ type: concepts --- -# GitLab Release and Maintenance Policy +# GitLab release and maintenance policy GitLab has strict policies governing version naming, as well as release pace for major, minor, -patch, and security releases. New releases are usually announced on the [GitLab blog](https://about.gitlab.com/releases/categories/releases/). +patch, and security releases. New releases are announced on the [GitLab blog](https://about.gitlab.com/releases/categories/releases/). Our current policy is: - Backporting bug fixes for **only the current stable release** at any given time. (See [patch releases](#patch-releases).) -- Backporting **to the previous two monthly releases in addition to the current stable release**. (See [security releases](#security-releases).) +- Backporting security fixes **to the previous two monthly releases in addition to the current stable release**. (See [security releases](#security-releases).) + +In rare cases, release managers may make an exception and backport to more than +the last two monthly releases. See [Backporting to older +releases](#backporting-to-older-releases) for more information. ## Versioning @@ -19,8 +23,8 @@ GitLab uses [Semantic Versioning](https://semver.org/) for its releases: For example, for GitLab version 12.10.6: -- `12` represents the major version. The major release was 12.0.0, but often referred to as 12.0. -- `10` represents the minor version. The minor release was 12.10.0, but often referred to as 12.10. +- `12` represents the major version. The major release was 12.0.0 but often referred to as 12.0. +- `10` represents the minor version. The minor release was 12.10.0 but often referred to as 12.10. - `6` represents the patch number. Any part of the version number can increment into multiple digits, for example, 13.10.11. @@ -37,7 +41,7 @@ The following table describes the version types and their release cadence: We encourage everyone to run the [latest stable release](https://about.gitlab.com/releases/categories/releases/) to ensure that you can easily upgrade to the most secure and feature-rich GitLab experience. -In order to make sure you can easily run the most recent stable release, we are working +To make sure you can easily run the most recent stable release, we are working hard to keep the update process simple and reliable. If you are unable to follow our monthly release cycle, there are a couple of @@ -66,7 +70,8 @@ one major version. For example, it is safe to: - `9.5.5` -> `9.5.9` - `8.9.2` -> `8.9.6` -NOTE **Note** Version specific changes in Omnibus GitLab Linux packages can be found in [the Omnibus GitLab documentation](https://docs.gitlab.com/omnibus/update/README.html#version-specific-changes). +NOTE: **Note:** +Version specific changes in Omnibus GitLab Linux packages can be found in [the Omnibus GitLab documentation](https://docs.gitlab.com/omnibus/update/README.html#version-specific-changes). NOTE: **Note:** Instructions are available for downloading an Omnibus GitLab Linux package locally and [manually installing](https://docs.gitlab.com/omnibus/manual_install.html) it. @@ -79,7 +84,7 @@ We cannot guarantee that upgrading between major versions will be seamless. We suggest upgrading to the latest available *minor* version within your major version before proceeding to the next major version. Doing this will address any backward-incompatible changes or deprecations -to help ensure a successful upgrade to next major release. +to help ensure a successful upgrade to the next major release. It's also important to ensure that any background migrations have been fully completed before upgrading to a new major version. To see the current size of the `background_migration` queue, @@ -107,17 +112,16 @@ Please see the table below for some examples: | Target version | Your version | Recommended upgrade path | Note | | --------------------- | ------------ | ------------------------ | ---- | -| `13.2.0` | `11.5.0` | `11.5.0` -> `11.11.8` -> `12.0.12` -> `12.10.6` -> `13.0.0` -> `13.2.0` | Four intermediate versions are required: the final 11.11, 12.0, and 12.10 releases, plus 13.0. | +| `13.2.0` | `11.5.0` | `11.5.0` -> `11.11.8` -> `12.0.12` -> `12.10.6` -> `13.0.0` -> `13.2.0` | Four intermediate versions are required: the final `11.11`, `12.0`, and `12.10` releases, plus `13.0`. | | `13.0.1` | `11.10.8` | `11.10.5` -> `11.11.8` -> `12.0.12` -> `12.10.6` -> `13.0.1` | Three intermediate versions are required: `11.11`, `12.0`, and `12.10`. | | `12.10.6` | `11.3.4` | `11.3.4` -> `11.11.8` -> `12.0.12` -> `12.10.6` | Two intermediate versions are required: `11.11` and `12.0` | -| `12.9.5.` | `10.4.5` | `10.4.5` -> `10.8.7` -> `11.11.8` -> `12.0.12` -> `12.9.5` | Three intermediate versions are required: `10.8`, `11.11`, and `12.0`, then `12.9.5` | +| `12.9.5` | `10.4.5` | `10.4.5` -> `10.8.7` -> `11.11.8` -> `12.0.12` -> `12.9.5` | Three intermediate versions are required: `10.8`, `11.11`, and `12.0`, then `12.9.5` | | `12.2.5` | `9.2.6` | `9.2.6` -> `9.5.10` -> `10.8.7` -> `11.11.8` -> `12.0.12` -> `12.2.5` | Four intermediate versions are required: `9.5`, `10.8`, `11.11`, `12.0`, then `12.2`. | | `11.3.4` | `8.13.4` | `8.13.4` -> `8.17.7` -> `9.5.10` -> `10.8.7` -> `11.3.4` | `8.17.7` is the last version in version 8, `9.5.10` is the last version in version 9, `10.8.7` is the last version in version 10. | ### Upgrades from versions earlier than 8.12 -- `8.11.x` and earlier: you might have to upgrade to `8.12.0` specifically before you can - upgrade to `8.17.7`. This was [reported in an issue](https://gitlab.com/gitlab-org/gitlab/-/issues/207259). +- `8.11.x` and earlier: you might have to upgrade to `8.12.0` specifically before you can upgrade to `8.17.7`. This was [reported in an issue](https://gitlab.com/gitlab-org/gitlab/-/issues/207259). - [CI changes prior to version 8.0](https://docs.gitlab.com/omnibus/update/README.html#updating-gitlab-ci-from-prior-540-to-version-714-via-omnibus-gitlab) when it was merged into GitLab. @@ -155,11 +159,11 @@ and support cost. 1. Supporting parallel version discourages incremental upgrades which over time accumulate in complexity and create upgrade challenges for all users. GitLab has a dedicated team ensuring that incremental upgrades (and installations) are as simple as possible. -1. The number of changes created in the GitLab application is high, which contributes to backporting complexity to older releases. In number of cases, backporting has to go through the same +1. The number of changes created in the GitLab application is high, which contributes to backporting complexity to older releases. In several cases, backporting has to go through the same review process a new change goes through. -1. Ensuring that tests pass on older release is a considerable challenge in some cases, and as such is very time consuming. +1. Ensuring that tests pass on the older release is a considerable challenge in some cases, and as such is very time-consuming. -Including new features in patch releases is not possible as that would break [Semantic Versioning](https://semver.org/). +Including new features in a patch release is not possible as that would break [Semantic Versioning](https://semver.org/). Breaking [Semantic Versioning](https://semver.org/) has the following consequences for users that have to adhere to various internal requirements (for example, org. compliance, verifying new features, and similar): @@ -169,7 +173,7 @@ have to adhere to various internal requirements (for example, org. compliance, v In cases where a strategic user has a requirement to test a feature before it is officially released, we can offer to create a Release Candidate (RC) version that will -include the specific feature. This should be needed only in extreme cases, and can be requested for +include the specific feature. This should be needed only in extreme cases and can be requested for consideration by raising an issue in the [release/tasks](https://gitlab.com/gitlab-org/release/tasks/-/issues/new?issuable_template=Backporting-request) issue tracker. It is important to note that the Release Candidate will also contain other features and changes as it is not possible to easily isolate a specific feature (similar reasons as noted above). The @@ -178,8 +182,8 @@ accessible. ### Backporting to older releases -Backporting to more than one stable release is reserved for [security releases](#security-releases). -In some cases however, we may need to backport *a bug fix* to more than one stable +Backporting to more than one stable release is normally reserved for [security releases](#security-releases). +In some cases, however, we may need to backport *a bug fix* to more than one stable release, depending on the severity of the bug. The decision on whether backporting a change will be performed is done at the discretion of the @@ -189,16 +193,13 @@ based on *all* of the following: 1. Estimated [severity](../development/contributing/issue_workflow.md#severity-labels) of the bug: Highest possible impact to users based on the current definition of severity. - 1. Estimated [priority](../development/contributing/issue_workflow.md#priority-labels) of the bug: Immediate impact on all impacted users based on the above estimated severity. - 1. Potentially incurring data loss and/or security breach. - 1. Potentially affecting one or more strategic accounts due to a proven inability by the user to upgrade to the current stable version. If *all* of the above are satisfied, the backport releases can be created for -the current stable release, and two previous monthly releases. +the current stable release, and two previous monthly releases. In rare cases a release manager may grant an exception to backport to more than two previous monthly releases. For instance, if we release `11.2.1` with a fix for a severe bug introduced in `11.0.0`, we could backport the fix to a new `11.0.x`, and `11.1.x` patch release. @@ -220,12 +221,11 @@ This decision is made on a case-by-case basis. Check [our release posts](https://about.gitlab.com/releases/categories/releases/). Each month, we publish either a major or minor release of GitLab. At the end -of those release posts there are three sections to look for: Deprecations, Removals, and Important notes on upgrading. These will include: +of those release posts, there are three sections to look for: Deprecations, Removals, and Important notes on upgrading. These will include: - Steps you need to perform as part of an upgrade. For example [8.12](https://about.gitlab.com/releases/2016/09/22/gitlab-8-12-released/#upgrade-barometer) - required the Elasticsearch index to be recreated. Any older version of GitLab upgrading to 8.12 or higher - would require this. + required the Elasticsearch index to be recreated. Any older version of GitLab upgrading to 8.12 or higher would require this. - Changes to the versions of software we support such as [ceasing support for IE11 in GitLab 13](https://about.gitlab.com/releases/2020/03/22/gitlab-12-9-released/#ending-support-for-internet-explorer-11). diff --git a/doc/public_access/public_access.md b/doc/public_access/public_access.md index 3f93cd7f61f..848065de001 100644 --- a/doc/public_access/public_access.md +++ b/doc/public_access/public_access.md @@ -59,7 +59,7 @@ it. The restriction for visibility levels on the application setting level also applies to groups, so if that's set to internal, the explore page will be empty for anonymous users. The group page now has a visibility level icon. -Admin users cannot create subgroups or projects with higher visibility level than that of the parent group. +Admin users cannot create subgroups or projects with higher visibility level than that of the immediate parent group. ## Visibility of users diff --git a/doc/raketasks/README.md b/doc/raketasks/README.md index eaaf1ebed99..b7cfc18534b 100644 --- a/doc/raketasks/README.md +++ b/doc/raketasks/README.md @@ -29,9 +29,10 @@ The following are available Rake tasks: | [Import repositories](import.md) | Import bare repositories into your GitLab instance. | | [Import large project exports](../development/import_project.md#importing-via-a-rake-task) | Import large GitLab [project exports](../user/project/settings/import_export.md). | | [Integrity checks](../administration/raketasks/check.md) | Check the integrity of repositories, files, and LDAP. | -| [LDAP maintenance](../administration/raketasks/ldap.md) | [LDAP](../administration/auth/ldap/index.md)-related tasks. | +| [LDAP maintenance](../administration/raketasks/ldap.md) | [LDAP](../administration/auth/ldap/index.md)-related tasks. | | [List repositories](list_repos.md) | List of all GitLab-managed Git repositories on disk. | | [Migrate Snippets to Git](migrate_snippets.md) | Migrate GitLab Snippets to Git repositories and show migration status | +| [Praefect Rake tasks](../administration/raketasks/praefect.md) | [Praefect](../administration/gitaly/praefect.md)-related tasks. | | [Project import/export](../administration/raketasks/project_import_export.md) | Prepare for [project exports and imports](../user/project/settings/import_export.md). | | [Sample Prometheus data](generate_sample_prometheus_data.md) | Generate sample Prometheus data. | | [Repository storage](../administration/raketasks/storage.md) | List and migrate existing projects and attachments from legacy storage to hashed storage. | diff --git a/doc/raketasks/backup_restore.md b/doc/raketasks/backup_restore.md index 18c1cba54f7..d8522cc19a0 100644 --- a/doc/raketasks/backup_restore.md +++ b/doc/raketasks/backup_restore.md @@ -1,4 +1,4 @@ -# Backing up and restoring GitLab **(CORE ONLY)** +# Back up and restore GitLab **(CORE ONLY)** GitLab provides Rake tasks for backing up and restoring GitLab instances. @@ -26,14 +26,6 @@ installed on your system. sudo yum install rsync ``` -- **Tar**: Backup and restore tasks use `tar` under the hood to create and extract - archives. Ensure you have version 1.30 or above of `tar` available in your - system. To check the version, run: - - ```shell - tar --version - ``` - ## Backup timestamp NOTE: **Note:** @@ -75,7 +67,7 @@ Use this command if you've installed GitLab with the Omnibus package: sudo gitlab-backup create ``` -NOTE: **Note** +NOTE: **Note:** For GitLab 12.1 and earlier, use `gitlab-rake gitlab:backup:create`. Use this if you've installed GitLab from source: @@ -90,7 +82,7 @@ If you are running GitLab within a Docker container, you can run the backup from docker exec -t <container name> gitlab-backup create ``` -NOTE: **Note** +NOTE: **Note:** For GitLab 12.1 and earlier, use `gitlab-rake gitlab:backup:create`. If you are using the [GitLab Helm chart](https://gitlab.com/gitlab-org/charts/gitlab) on a @@ -206,7 +198,7 @@ To use the `copy` strategy instead of the default streaming strategy, specify sudo gitlab-backup create STRATEGY=copy ``` -NOTE: **Note** +NOTE: **Note:** For GitLab 12.1 and earlier, use `gitlab-rake gitlab:backup:create`. #### Backup filename @@ -221,7 +213,7 @@ By default a backup file is created according to the specification in [the Backu sudo gitlab-backup create BACKUP=dump ``` -NOTE: **Note** +NOTE: **Note:** For GitLab 12.1 and earlier, use `gitlab-rake gitlab:backup:create`. The resulting file will then be `dump_gitlab_backup.tar`. This is useful for systems that make use of rsync and incremental backups, and will result in considerably faster transfer speeds. @@ -236,7 +228,7 @@ Note that the `--rsyncable` option in `gzip` is not guaranteed to be available o sudo gitlab-backup create BACKUP=dump GZIP_RSYNCABLE=yes ``` -NOTE: **Note** +NOTE: **Note:** For GitLab 12.1 and earlier, use `gitlab-rake gitlab:backup:create`. #### Excluding specific directories from the backup @@ -264,7 +256,7 @@ For Omnibus GitLab packages: sudo gitlab-backup create SKIP=db,uploads ``` -NOTE: **Note** +NOTE: **Note:** For GitLab 12.1 and earlier, use `gitlab-rake gitlab:backup:create`. For installations from source: @@ -466,7 +458,13 @@ For Omnibus GitLab packages: gitlab_rails['backup_upload_connection'] = { 'provider' => 'Google', 'google_storage_access_key_id' => 'Access Key', - 'google_storage_secret_access_key' => 'Secret' + 'google_storage_secret_access_key' => 'Secret', + + ## If you have CNAME buckets (foo.example.com), you might run into SSL issues + ## when uploading backups ("hostname foo.example.com.storage.googleapis.com + ## does not match the server certificate"). In that case, uncomnent the following + ## setting. See: https://github.com/fog/fog/issues/2834 + #'path_style' => true } gitlab_rails['backup_upload_remote_directory'] = 'my.google.bucket' ``` @@ -499,7 +497,7 @@ sudo gitlab-backup create DIRECTORY=daily sudo gitlab-backup create DIRECTORY=weekly ``` -NOTE: **Note** +NOTE: **Note:** For GitLab 12.1 and earlier, use `gitlab-rake gitlab:backup:create`. #### Uploading to locally mounted shares @@ -517,7 +515,8 @@ backups will be copied to, and will be created if it does not exist. If the directory that you want to copy the tarballs to is the root of your mounted directory, just use `.` instead. -NOTE: **Note:** Since file system performance may affect GitLab's overall performance, we do not recommend using EFS for storage. See the [relevant documentation](../administration/high_availability/nfs.md#avoid-using-awss-elastic-file-system-efs) for more details. +NOTE: **Note:** +Since file system performance may affect GitLab's overall performance, we do not recommend using EFS for storage. See the [relevant documentation](../administration/high_availability/nfs.md#avoid-using-awss-elastic-file-system-efs) for more details. For Omnibus GitLab packages: @@ -605,7 +604,7 @@ For Omnibus GitLab packages: 0 2 * * * /opt/gitlab/bin/gitlab-backup create CRON=1 ``` - NOTE: **Note** + NOTE: **Note:** For GitLab 12.1 and earlier, use `gitlab-rake gitlab:backup:create`. For installations from source: @@ -679,7 +678,7 @@ You can only restore a backup to **exactly the same version and type (CE/EE)** o GitLab that you created it on, for example CE 9.1.0. If your backup is a different version than the current installation, you will -need to [downgrade your GitLab installation](https://docs.gitlab.com/omnibus/update/README.html#downgrading) +need to [downgrade your GitLab installation](https://docs.gitlab.com/omnibus/update/README.html#downgrade) before restoring the backup. ### Restore prerequisites @@ -806,7 +805,7 @@ restore: sudo gitlab-backup restore BACKUP=11493107454_2018_04_25_10.6.4-ce ``` -NOTE: **Note** +NOTE: **Note:** For GitLab 12.1 and earlier, use `gitlab-rake gitlab:backup:restore`. CAUTION: **Warning:** @@ -828,6 +827,12 @@ If there is a GitLab version mismatch between your backup tar file and the insta version of GitLab, the restore command will abort with an error. Install the [correct GitLab version](https://packages.gitlab.com/gitlab/) and try again. +NOTE: **Note:** +There is currently a [known issue](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/3470) for restore not working +with `pgbouncer`. In order to workaround the issue, the Rails node will need to bypass `pgbouncer` and connect +directly to the primary database node. This can be done by setting `gitlab_rails['db_host']` and `gitlab_rails['port']` +to connect to the primary database node and [reconfiguring GitLab](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure). + ### Restore for Docker image and GitLab Helm chart installations For GitLab installations using the Docker image or the GitLab Helm chart on @@ -845,10 +850,25 @@ backup location (default location is `/var/opt/gitlab/backups`). For Docker installations, the restore task can be run from host: ```shell -docker exec -it <name of container> gitlab-backup restore +# Stop the processes that are connected to the database +docker exec -it <name of container> gitlab-ctl stop unicorn +docker exec -it <name of container> gitlab-ctl stop puma +docker exec -it <name of container> gitlab-ctl stop sidekiq + +# Verify that the processes are all down before continuing +docker exec -it <name of container> gitlab-ctl status + +# Run the restore +docker exec -it <name of container> gitlab-backup restore BACKUP=11493107454_2018_04_25_10.6.4-ce + +# Restart the GitLab container +docker restart <name of container> + +# Check GitLab +docker exec -it <name of container> gitlab-rake gitlab:check SANITIZE=true ``` -NOTE: **Note** +NOTE: **Note:** For GitLab 12.1 and earlier, use `gitlab-rake gitlab:backup:restore`. CAUTION: **Warning:** @@ -859,6 +879,28 @@ use `gitlab-backup restore` to avoid this issue. The GitLab Helm chart uses a different process, documented in [restoring a GitLab Helm chart installation](https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/backup-restore/restore.md). +### Restoring only one or a few project(s) or group(s) from a backup + +While the Rake task used to restore a GitLab instance doesn't support +restoring a single project or group, you can use a workaround by +restoring your backup to a separate, temporary GitLab instance, and +export your project or group from there: + +1. [Install a new GitLab](../install/README.md) instance at the same version as + the backed-up instance from which you want to restore. +1. [Restore the backup](#restore-gitlab) into this new instance and + export your [project](../user/project/settings/import_export.md) + or [group](../user/group/settings/import_export.md). Make sure to + read the **Important Notes** on either export feature's documentation + to understand what will be exported and what not. +1. Once the export is complete, go to the old instance and import it. +1. After importing only the project(s) or group(s) that you wanted is complete, + you may delete the new, temporary GitLab instance. + +NOTE: **Note:** +A feature request to provide direct restore of individual projects or groups +is being discussed in [issue #17517](https://gitlab.com/gitlab-org/gitlab/-/issues/17517). + ## Alternative backup strategies If your GitLab server contains a lot of Git repository data you may find the GitLab backup script to be too slow. @@ -914,7 +956,7 @@ Be advised that, backup is successfully restored in spite of these warnings. The Rake task runs this as the `gitlab` user which does not have the superuser access to the database. When restore is initiated it will also run as `gitlab` user but it will also try to alter the objects it does not have access to. Those objects have no influence on the database backup/restore but they give this annoying warning. -For more information see similar questions on PostgreSQL issue tracker[here](https://www.postgresql.org/message-id/201110220712.30886.adrian.klaver@gmail.com) and [here](https://www.postgresql.org/message-id/2039.1177339749@sss.pgh.pa.us) as well as [stack overflow](https://stackoverflow.com/questions/4368789/error-must-be-owner-of-language-plpgsql). +For more information see similar questions on PostgreSQL issue tracker [here](https://www.postgresql.org/message-id/201110220712.30886.adrian.klaver@gmail.com) and [here](https://www.postgresql.org/message-id/2039.1177339749@sss.pgh.pa.us) as well as [stack overflow](https://stackoverflow.com/questions/4368789/error-must-be-owner-of-language-plpgsql). ### When the secrets file is lost diff --git a/doc/raketasks/cleanup.md b/doc/raketasks/cleanup.md index 5bdae998ec9..cf4edea383b 100644 --- a/doc/raketasks/cleanup.md +++ b/doc/raketasks/cleanup.md @@ -2,7 +2,7 @@ GitLab provides Rake tasks for cleaning up GitLab instances. -## Remove unreferenced LFS files from filesystem +## Remove unreferenced LFS files > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/36628) in GitLab 12.10. @@ -42,7 +42,7 @@ Note that this Rake task only removes the references to LFS files. Unreferenced later (once a day). If you need to garbage collect them immediately, run `rake gitlab:cleanup:orphan_lfs_files` described below. -## Remove unreferenced LFS files +### Remove unreferenced LFS files immediately > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/36628) in GitLab 12.10. @@ -64,7 +64,11 @@ $ sudo gitlab-rake gitlab:cleanup:orphan_lfs_files I, [2020-01-08T20:51:17.148765 #43765] INFO -- : Removed unreferenced LFS files: 12 ``` -## Remove garbage from filesystem +## Clean up project upload files + +Clean up project upload files if they don't exist in GitLab database. + +### Clean up project upload files from filesystem > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/20863) in GitLab 11.2. @@ -100,11 +104,11 @@ I, [2018-07-27T12:08:33.755624 #89817] INFO -- : Did fix /opt/gitlab/embedded/s I, [2018-07-27T12:08:33.760257 #89817] INFO -- : Did move to lost and found /opt/gitlab/embedded/service/gitlab-rails/public/uploads/foo/bar/1dd6f0f7eefd2acc4c2233f89a0f7b0b/image.png -> /opt/gitlab/embedded/service/gitlab-rails/public/uploads/-/project-lost-found/foo/bar/1dd6f0f7eefd2acc4c2233f89a0f7b0b/image.png ``` -## Remove garbage from object storage +### Clean up project upload files from object storage > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/20918) in GitLab 11.2. -Remove object store upload files if they don't exist in GitLab database. +Move object store upload files to a lost and found directory if they don't exist in GitLab database. ```shell # omnibus-gitlab @@ -142,7 +146,7 @@ When you notice there are more job artifacts files on disk than there should be, you can run: ```shell -gitlab-rake gitlab:cleanup:orphan_job_artifact_files +sudo gitlab-rake gitlab:cleanup:orphan_job_artifact_files ``` This command: @@ -156,13 +160,13 @@ delete. Run the command with `DRY_RUN=false` if you actually want to delete the files: ```shell -gitlab-rake gitlab:cleanup:orphan_job_artifact_files DRY_RUN=false +sudo gitlab-rake gitlab:cleanup:orphan_job_artifact_files DRY_RUN=false ``` You can also limit the number of files to delete with `LIMIT`: ```shell -gitlab-rake gitlab:cleanup:orphan_job_artifact_files LIMIT=100 +sudo gitlab-rake gitlab:cleanup:orphan_job_artifact_files LIMIT=100 ``` This will only delete up to 100 files from disk. You can use this to diff --git a/doc/security/rack_attack.md b/doc/security/rack_attack.md index 605b669d498..d3de2222c39 100644 --- a/doc/security/rack_attack.md +++ b/doc/security/rack_attack.md @@ -18,11 +18,13 @@ tracking. For more information on how to use these options see the [Rack Attack README](https://github.com/kickstarter/rack-attack/blob/master/README.md). -NOTE: **Note:** See +NOTE: **Note:** +See [User and IP rate limits](../user/admin_area/settings/user_and_ip_rate_limits.md) for simpler limits that are configured in the UI. -NOTE: **Note:** Starting with GitLab 11.2, Rack Attack is disabled by default. If your +NOTE: **Note:** +Starting with GitLab 11.2, Rack Attack is disabled by default. If your instance is not exposed to the public internet, it is recommended that you leave Rack Attack disabled. diff --git a/doc/security/rate_limits.md b/doc/security/rate_limits.md index d58eb130522..af2c14be2cd 100644 --- a/doc/security/rate_limits.md +++ b/doc/security/rate_limits.md @@ -26,6 +26,7 @@ similarly mitigated by a rate limit. - [User and IP rate limits](../user/admin_area/settings/user_and_ip_rate_limits.md). - [Raw endpoints rate limits](../user/admin_area/settings/rate_limits_on_raw_endpoints.md). - [Protected paths](../user/admin_area/settings/protected_paths.md). +- [Import/Export rate limits](../user/admin_area/settings/import_export_rate_limits.md). ## Rack Attack initializer diff --git a/doc/ssh/README.md b/doc/ssh/README.md index 5db1bde6413..980a20ade3e 100644 --- a/doc/ssh/README.md +++ b/doc/ssh/README.md @@ -412,7 +412,7 @@ are *explicitly not supported* and may stop working at any time. ### Options for Microsoft Windows -If you're running Windows 10, the [Windows Subsystem for Linux (WSL)](https://docs.microsoft.com/en-us/windows/wsl/install-win10), and its latest [WSL 2](https://docs.microsoft.com/en-us/install-win10) version, +If you're running Windows 10, the [Windows Subsystem for Linux (WSL)](https://docs.microsoft.com/en-us/windows/wsl/install-win10), and its latest [WSL 2](https://docs.microsoft.com/en-us/windows/wsl/install-win10#update-to-wsl-2) version, support the installation of different Linux distributions, which include the Git and SSH clients. For current versions of Windows, you can also install the Git and SSH clients with diff --git a/doc/subscriptions/index.md b/doc/subscriptions/index.md index 0425fe98d27..64bf2e0b50b 100644 --- a/doc/subscriptions/index.md +++ b/doc/subscriptions/index.md @@ -22,13 +22,13 @@ When choosing a subscription, consider the following factors: ### Choosing a GitLab tier -Pricing is [tier-based](https://about.gitlab.com/pricing/), allowing you to choose the features which fit your budget. See the [feature comparison](https://about.gitlab.com/pricing/gitlab-com/feature-comparison/) for information on what features are available at each tier. +Pricing is [tier-based](https://about.gitlab.com/pricing/), allowing you to choose the features which fit your budget. See the [GitLab.com feature comparison](https://about.gitlab.com/pricing/gitlab-com/feature-comparison/) and the [self-managed feature comparison](https://about.gitlab.com/pricing/self-managed/feature-comparison/) for information on what features are available at each tier for each product. ### Choosing between GitLab.com or self-managed There are some differences in how a subscription applies, depending if you use GitLab.com or a self-managed instance. -- [GitLab.com](#gitlabcom): GitLab's software-as-a-service offering. You don't need to install anything to use GitLab.com, you only need to [sign up](https://gitlab.com/users/sign_in) and start using GitLab straight away. +- [GitLab.com](#gitlabcom): GitLab's software-as-a-service offering. You don't need to install anything to use GitLab.com, you only need to [sign up](https://gitlab.com/users/sign_up) and start using GitLab straight away. - [GitLab self-managed](#self-managed): Install, administer, and maintain your own GitLab instance. On a self-managed instance, a GitLab subscription provides the same set of features for all users. On GitLab.com you can apply a subscription to either a group or a personal namespace. @@ -103,10 +103,11 @@ To subscribe to GitLab.com: 1. Select the **Bronze**, **Silver**, or **Gold** GitLab.com plan through the [Customers Portal](https://customers.gitlab.com/). 1. Link your GitLab.com account with your Customers Portal account. - Once signed into the Customers Portal, if your account is not + Once a plan has been selected, if your account is not already linked, you will be prompted to link your account with a - **Link my GitLab Account** button. -1. Associate the group with the subscription. + **Sign in to GitLab.com** button. +1. Select the namespace from the drop-down list to associate the subscription. +1. Proceed to checkout. TIP: **Tip:** You can also go to the [**My Account**](https://customers.gitlab.com/customers/edit) @@ -129,19 +130,20 @@ instance, ensure you're purchasing enough seats to With the [Customers Portal](https://customers.gitlab.com/) you can: -- [Change billing information](#change-billing-information) +- [Change billing and company information](#change-billing-information) - [Change the payment method](#change-payment-method) - [Change the linked account](#change-the-linked-account) - [Change the associated namespace](#change-the-associated-namespace) +- [Change customers portal account password](#change-customer-portal-account-password) ### Change billing information To change billing information: 1. Log in to the [Customers Portal](https://customers.gitlab.com/customers/sign_in). -1. Go to the **My Account** page. +1. Select the **My account** drop-down and click on **Payment methods**. 1. Make the required changes to the **Account Details** information. -1. Click **Update Account**. +1. Click **Save changes**. NOTE: **Note:** Future purchases will use the information in this section. @@ -159,17 +161,17 @@ To change payment method or update credit card information: ### Change the linked account -To change the GitLab.com account associated with a Customers Portal +To change the GitLab.com account associated with your Customers Portal account: 1. Log in to the [Customers Portal](https://customers.gitlab.com/customers/sign_in). -1. Go to [GitLab.com](https://gitlab.com) in a separate browser tab. Ensure you +1. In a separate browser tab, go to [GitLab.com](https://gitlab.com) and ensure you are not logged in. -1. On the Customers Portal page, click - [**My Account**](https://customers.gitlab.com/customers/edit) in the top menu. -1. Under **Your GitLab.com account**, click **Change linked account** button. -1. Log in to the [GitLab.com](https://gitlab.com) account you want to link to the Customers Portal. +1. On the Customers Portal page, click **My account > Account details**. +1. Under **Your GitLab.com account**, click **Change linked account**. +1. Log in to the [GitLab.com](https://gitlab.com) account you want to link to the Customers Portal + account. ### Change the associated namespace @@ -177,12 +179,21 @@ With a linked GitLab.com account: 1. Log in to the [Customers Portal](https://customers.gitlab.com/customers/sign_in). 1. Navigate to the **Manage Purchases** page. -1. Click **Change linked group**. +1. Click **Change linked namespace**. 1. Select the desired group from the **This subscription is for** dropdown. 1. Click **Proceed to checkout**. Subscription charges are calculated based on the total number of users in a group, including its subgroups and nested projects. If the total number of users exceeds the number of seats in your subscription, you will be charged for the additional users. +### Change customer portal account password + +To change the password for this customers portal account: + +1. Log in to the [Customers Portal](https://customers.gitlab.com/customers/sign_in). +1. Select the **My account** drop-down and click on **Account details**. +1. Make the required changes to the **Your password** section. +1. Click **Save changes**. + ## View your subscription ### View your GitLab.com subscription @@ -224,8 +235,8 @@ To renew your subscription, [prepare for renewal by reviewing your account](#pre The [Customers Portal](https://customers.gitlab.com/customers/sign_in) is your tool for renewing and modifying your subscription. Before going ahead with renewal, log in and verify or update: -- The invoice contact details on the **My Account** page. -- The credit card on file in the **Payment Methods** page. +- The invoice contact details on the **Account details** page. +- The credit card on file on the **Payment Methods** page. TIP: **Tip:** Contact our [support team](https://support.gitlab.com/hc/en-us/requests/new?ticket_form_id=360000071293) if you need assistance accessing the Customers Portal or if you need to change the contact person who manages your subscription. @@ -241,7 +252,7 @@ A GitLab subscription is valid for a specific number of users. For details, see ##### Purchase additional seats for GitLab.com -There is no self-service option for purchasing additional seats. You must request a quotation from GitLab Sales. To do so, contact GitLab via our [support form](https://support.gitlab.com/hc/en-us/requests/new) and select **Licensing and Renewals Problems** from the menu. +There is no self-service option for purchasing additional seats. You must request a quotation from GitLab Sales. To do so, contact GitLab via our [support form](https://support.gitlab.com/hc/en-us/requests/new?ticket_form_id=360000071293). The amount charged per seat is calculated by one of the following methods: @@ -255,8 +266,8 @@ Self-managed instances can add users to a subscription any time during the subsc To add users to a subscription: 1. Log in to the [Customers Portal](https://customers.gitlab.com/). -1. Select **Manage Purchases**. -1. Select **Add more seats**. +1. Navigate to the **Manage Purchases** page. +1. Select **Add more seats** on the relevant subscription card. 1. Enter the number of additional users. 1. Select **Proceed to checkout**. 1. Review the **Subscription Upgrade Detail**. The system lists the total price for all users on the system and a credit for what you've already paid. You will only be charged for the net change. @@ -264,7 +275,7 @@ To add users to a subscription: The following will be emailed to you: -- A payment receipt. You can also access this information in the Customers Portal under **Payment History**. +- A payment receipt. You can also access this information in the Customers Portal under [**View invoices**](https://customers.gitlab.com/receipts). - A new license. [Upload this license](../user/admin_area/license.md#uploading-your-license) to your instance to use it. ### Seat Link @@ -273,7 +284,7 @@ The following will be emailed to you: Seat Link allows us to provide our self-managed customers with prorated charges for user growth throughout the year using a quarterly reconciliation process. -Seat Link sends to GitLab daily a count of all users in connected self-managed instances. That information is used to automate prorated reconciliations. The data is sent securely through an encrypted HTTPS connection. +Seat Link daily sends a count of all users in connected self-managed instances to GitLab. That information is used to automate prorated reconciliations. The data is sent securely through an encrypted HTTPS connection. Seat Link provides **only** the following information to GitLab: @@ -333,7 +344,7 @@ Sg0KU1hNMGExaE9SVGR2V2pKQlBUMWNiaUo5DQo=', You can view the exact JSON payload in the administration panel. To view the payload: -1. Navigate to **Admin Area > Settings > Metrics and profiling** and expand **Seat Links**. +1. Navigate to **Admin Area > Settings > Metrics and profiling** and expand **Seat Link**. 1. Click **Preview payload**. #### Disable Seat Link @@ -343,7 +354,7 @@ You can view the exact JSON payload in the administration panel. To view the pay Seat Link is enabled by default. To disable this feature, go to -**{admin}** **Admin Area > Settings > Metrics and profiling** and clear the **Seat Link** checkbox. +**{admin}** **Admin Area > Settings > Metrics and profiling**, uncheck the **Enable Seat Link** checkbox > **Save changes**. To disable Seat Link in an Omnibus GitLab installation, and prevent it from being configured in the future through the administration panel, set the following in @@ -379,7 +390,7 @@ To view or change automatic subscription renewal (at the same tier as the previo - If you see **Cancel subscription**, your subscription is set to automatically renew at the end of the subscription period. Click it to cancel automatic renewal. With automatic renewal enabled, the subscription will automatically renew on the expiration date and there will be no gap in available service. -An invoice will be generated for the renewal and available for viewing or download in the [Payment History](https://customers.gitlab.com/receipts) page. If you have difficulty during the renewal process, contact our [support team](https://support.gitlab.com/hc/en-us/requests/new?ticket_form_id=360000071293) for assistance. +An invoice will be generated for the renewal and available for viewing or download in the [View invoices](https://customers.gitlab.com/receipts) page. If you have difficulty during the renewal process, contact our [support team](https://support.gitlab.com/hc/en-us/requests/new?ticket_form_id=360000071293) for assistance. ### Renew a self-managed subscription @@ -410,10 +421,10 @@ We recommend following these steps during renewal: | Users over license | The number of users that exceed the `Users in License` for the current license term. Charges for this number of users will be incurred at the next renewal. | 1. Review your renewal details and complete the payment process. -1. A license for the renewal term will be available on the [Manage Purchases](https://customers.gitlab.com/subscriptions) page beneath your new subscription details. -1. [Upload](../user/admin_area/license.md) your new license to your instance. +1. A license for the renewal term will be available for download on the [Manage Purchases](https://customers.gitlab.com/subscriptions) page on the relevant subscription card. Select **Copy license to clipboard** or **Download license** to get a copy. +1. [Upload](../user/admin_area/license.md#uploading-your-license) your new license to your instance. -An invoice will be generated for the renewal and available for viewing or download in the [Payment History](https://customers.gitlab.com/receipts) page. If you have difficulty during the renewal process, contact our [support team](https://support.gitlab.com/hc/en-us/requests/new?ticket_form_id=360000071293) for assistance. +An invoice will be generated for the renewal and available for viewing or download on the [View invoices](https://customers.gitlab.com/receipts) page. If you have difficulty during the renewal process, contact our [support team](https://support.gitlab.com/hc/en-us/requests/new?ticket_form_id=360000071293) for assistance. ## Upgrade your subscription tier @@ -424,7 +435,7 @@ The process for upgrading differs depending on whether you're a GitLab.com or se To upgrade your [GitLab tier](https://about.gitlab.com/pricing/): 1. Log in to the [Customers Portal](https://customers.gitlab.com/customers/sign_in). -1. Select **Upgrade** under your subscription on the [My Account](https://customers.gitlab.com/subscriptions) page. +1. Select the **Upgrade** button on the relevant subscription card on the [Manage purchases](https://customers.gitlab.com/subscriptions) page. 1. Select the desired upgrade. 1. Confirm the active form of payment, or add a new form of payment. 1. Check the **I accept the Privacy Policy and Terms of Service** checkbox. @@ -436,8 +447,7 @@ When the purchase has been processed, you receive confirmation of your new subsc To upgrade your [GitLab tier](https://about.gitlab.com/pricing/), contact our sales team as this can't be done in the Customers Portal. You can either send an email to `renewals@gitlab.com`, or -complete the [**Contact Sales**](https://about.gitlab.com/sales/) form. Include in your message -details of which subscription you want to upgrade, and the desired tier. +complete the [**Contact Sales**](https://about.gitlab.com/sales/) form. Include details of which subscription you want to upgrade and the desired tier in your message. After messaging the sales team, the workflow is as follows: @@ -458,7 +468,7 @@ If you renew or upgrade, your data will again be accessible. ### Self-managed GitLab data -For self-managed customers, there is a two-week grace period when your features +For self-managed customers, there is a 14-day grace period when your features will continue to work as-is, after which the entire instance will become read only. @@ -467,19 +477,24 @@ features, and the instance will be read / write again. ## CI pipeline minutes -CI pipeline minutes are the execution time for your [pipelines](../ci/pipelines/index.md) on GitLab's shared runners. Each [GitLab.com tier](https://about.gitlab.com/pricing/) includes a monthly quota of CI pipeline minutes. +CI pipeline minutes are the execution time for your [pipelines](../ci/pipelines/index.md) on GitLab's shared runners. Each [GitLab.com tier](https://about.gitlab.com/pricing/) includes a monthly quota of CI pipeline minutes: + +- Free: 2,000 minutes +- Bronze: 2,000 minutes +- Silver: 10,000 minutes +- Gold: 50,000 minutes Quotas apply to: -- Groups, where the minutes are shared across all members of the group, its subgroups, and nested projects. To view the group's usage, navigate to the group, then **{settings}** **Settings > Usage Quotas**. -- Your personal account, where the minutes are available for your personal projects. To view and buy personal minutes, click your avatar, then **{settings}** **Settings > Pipeline quota**. +- Groups, where the minutes are shared across all members of the group, its subgroups, and nested projects. To view the group's usage, navigate to the group, then **{settings}** **Settings** > **Usage Quotas**. +- Your personal account, where the minutes are available for your personal projects. To view and buy personal minutes, click your avatar, then **{settings}** **Settings** > **[Usage Quotas](https://gitlab.com/profile/usage_quotas#pipelines-quota-tab)**. Only pipeline minutes for GitLab shared runners are restricted. If you have a specific runner set up for your projects, there is no limit to your build time on GitLab.com. The available quota is reset on the first of each calendar month at midnight UTC. When the CI minutes are depleted, an email is sent automatically to notify the owner(s) -of the group/namespace. You can [purchase additional CI minutes](#purchasing-additional-ci-minutes), or upgrade your account to [Silver or Gold](https://about.gitlab.com/pricing/). Your own runners can still be used even if you reach your limits. +of the namespace. You can [purchase additional CI minutes](#purchasing-additional-ci-minutes), or upgrade your account to [Silver or Gold](https://about.gitlab.com/pricing/). Your own runners can still be used even if you reach your limits. ### Purchasing additional CI minutes @@ -493,16 +508,20 @@ main quota. You can find pricing for additional CI/CD minutes in the [GitLab Cus To purchase additional minutes for your group on GitLab.com: 1. From your group, go to **{settings}** **Settings > Usage Quotas**. +1. Select **Buy additional minutes** and you will be directed to the Customers Portal. 1. Locate the subscription card that's linked to your group on GitLab.com, click **Buy more CI minutes**, and complete the details about the transaction. -1. Once we have processed your payment, the extra CI minutes will be synced to your group. +1. Once we have processed your payment, the extra CI minutes will be synced to your group namespace. 1. To confirm the available CI minutes, go to your group, then **{settings}** **Settings > Usage Quotas**. + The **Additional minutes** displayed now includes the purchased additional CI minutes, plus any minutes rolled over from last month. To purchase additional minutes for your personal namespace: -1. Click your avatar, then go to **Settings > Pipeline quota**. -1. Locate the subscription card that's linked to your personal namespace on GitLab.com, click **Buy more CI minutes**, and complete the details about the transaction. Once we have processed your payment, the extra CI minutes will be synced to your Group. -1. To confirm the available CI minutes for your personal projects, click your avatar, then go to **Settings > Pipeline quota**. +1. Click your avatar, then go to **Settings > Usage Quotas**. +1. Select **Buy additional minutes** and you will be directed to the Customers Portal. +1. Locate the subscription card that's linked to your personal namespace on GitLab.com, click **Buy more CI minutes**, and complete the details about the transaction. Once we have processed your payment, the extra CI minutes will be synced to your personal namespace. +1. To confirm the available CI minutes for your personal projects, click your avatar, then go to **Settings > Usage Quotas**. + The **Additional minutes** displayed now includes the purchased additional CI minutes, plus any minutes rolled over from last month. Be aware that: diff --git a/doc/tools/email.md b/doc/tools/email.md index eacf63ecdcd..69aca200311 100644 --- a/doc/tools/email.md +++ b/doc/tools/email.md @@ -22,7 +22,9 @@ at their primary email address.  1. Compose an email and choose where it will be sent (all users or users of a - chosen group or project): + chosen group or project). The email body only supports plain text messages. + HTML, Markdown, and other rich text formats are not supported, and will be + sent as plain text to users.  diff --git a/doc/topics/autodevops/customize.md b/doc/topics/autodevops/customize.md index 253d5e56463..2d6da4d322b 100644 --- a/doc/topics/autodevops/customize.md +++ b/doc/topics/autodevops/customize.md @@ -41,11 +41,16 @@ If your goal is to use only a single custom buildpack, you should provide the pr ## Custom `Dockerfile` +> Support for `DOCKERFILE_PATH` was [added in GitLab 13.2](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/35662) + If your project has a `Dockerfile` in the root of the project repository, Auto DevOps builds a Docker image based on the Dockerfile, rather than using buildpacks. This can be much faster and result in smaller images, especially if your Dockerfile is based on [Alpine](https://hub.docker.com/_/alpine/). +If you set the `DOCKERFILE_PATH` CI variable, Auto Build looks for a Dockerfile there +instead. + ## Passing arguments to `docker build` Arguments can be passed to the `docker build` command using the @@ -213,7 +218,7 @@ include: See the [Auto DevOps template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml) for information on available jobs. -CAUTION: **Deprecation** +CAUTION: **Deprecation:** Auto DevOps templates using the [`only`](../../ci/yaml/README.md#onlyexcept-basic) or [`except`](../../ci/yaml/README.md#onlyexcept-basic) syntax will switch to the [`rules`](../../ci/yaml/README.md#rules) syntax, starting in @@ -238,7 +243,7 @@ postgres://user:password@postgres-host:postgres-port/postgres-database ### Upgrading PostgresSQL -CAUTION: **Deprecation** +CAUTION: **Deprecation:** The variable `AUTO_DEVOPS_POSTGRES_CHANNEL` that controls default provisioned PostgreSQL was changed to `2` in [GitLab 13.0](https://gitlab.com/gitlab-org/gitlab/-/issues/210499). To keep using the old PostgreSQL, set the `AUTO_DEVOPS_POSTGRES_CHANNEL` variable to @@ -306,11 +311,13 @@ applications. | `AUTO_DEVOPS_CHART_REPOSITORY_USERNAME` | From GitLab 11.11, used to set a username to connect to the Helm repository. Defaults to no credentials. Also set `AUTO_DEVOPS_CHART_REPOSITORY_PASSWORD`. | | `AUTO_DEVOPS_CHART_REPOSITORY_PASSWORD` | From GitLab 11.11, used to set a password to connect to the Helm repository. Defaults to no credentials. Also set `AUTO_DEVOPS_CHART_REPOSITORY_USERNAME`. | | `AUTO_DEVOPS_DEPLOY_DEBUG` | From GitLab 13.1, if this variable is present, Helm will output debug logs. | +| `AUTO_DEVOPS_ALLOW_TO_FORCE_DEPLOY_V<N>` | From [auto-deploy-image](https://gitlab.com/gitlab-org/cluster-integration/auto-deploy-image) v1.0.0, if this variable is present, a new major version of chart is forcibly deployed. [More details](upgrading_chart.md#ignore-warning-and-continue-deploying) | | `AUTO_DEVOPS_MODSECURITY_SEC_RULE_ENGINE` | From GitLab 12.5, used in combination with [ModSecurity feature flag](../../user/clusters/applications.md#web-application-firewall-modsecurity) to toggle [ModSecurity's `SecRuleEngine`](https://github.com/SpiderLabs/ModSecurity/wiki/Reference-Manual-(v2.x)#SecRuleEngine) behavior. Defaults to `DetectionOnly`. | | `BUILDPACK_URL` | Buildpack's full URL. Can point to either [a Git repository URL or a tarball URL](#custom-buildpacks). | | `CANARY_ENABLED` | From GitLab 11.0, used to define a [deploy policy for canary environments](#deploy-policy-for-canary-environments-premium). | | `CANARY_PRODUCTION_REPLICAS` | Number of canary replicas to deploy for [Canary Deployments](../../user/project/canary_deployments.md) in the production environment. Takes precedence over `CANARY_REPLICAS`. Defaults to 1. | | `CANARY_REPLICAS` | Number of canary replicas to deploy for [Canary Deployments](../../user/project/canary_deployments.md). Defaults to 1. | +| `DOCKERFILE_PATH` | From GitLab 13.2, allows overriding the [default Dockerfile path for the build stage](#custom-dockerfile) | | `HELM_RELEASE_NAME` | From GitLab 12.1, allows the `helm` release name to be overridden. Can be used to assign unique release names when deploying multiple projects to a single namespace. | | `HELM_UPGRADE_VALUES_FILE` | From GitLab 12.6, allows the `helm upgrade` values file to be overridden. Defaults to `.gitlab/auto-deploy-values.yaml`. | | `HELM_UPGRADE_EXTRA_ARGS` | From GitLab 11.11, allows extra arguments in `helm` commands when deploying the application. Note that using quotes won't prevent word splitting. | @@ -358,7 +365,8 @@ The following table lists variables used to disable jobs. | `DAST_DISABLED` | From GitLab 11.0, used to disable the `dast` job. If the variable is present, the job won't be created. | | `DEPENDENCY_SCANNING_DISABLED` | From GitLab 11.0, used to disable the `dependency_scanning` job. If the variable is present, the job won't be created. | | `LICENSE_MANAGEMENT_DISABLED` | From GitLab 11.0, used to disable the `license_management` job. If the variable is present, the job won't be created. | -| `PERFORMANCE_DISABLED` | From GitLab 11.0, used to disable the `performance` job. If the variable is present, the job won't be created. | +| `PERFORMANCE_DISABLED` | From GitLab 11.0, used to disable the browser `performance` job. If the variable is present, the job won't be created. | +| `LOAD_PERFORMANCE_DISABLED` | From GitLab 13.2, used to disable the `load_performance` job. If the variable is present, the job won't be created. | | `REVIEW_DISABLED` | From GitLab 11.0, used to disable the `review` and the manual `review:stop` job. If the variable is present, these jobs won't be created. | | `SAST_DISABLED` | From GitLab 11.0, used to disable the `sast` job. If the variable is present, the job won't be created. | | `TEST_DISABLED` | From GitLab 11.0, used to disable the `test` job. If the variable is present, the job won't be created. | @@ -445,7 +453,7 @@ QA testing: environment: name: qa script: - - deploy foo + - deploy foo ``` The track `foo` being referenced must also be defined in the application's Helm chart, like: diff --git a/doc/topics/autodevops/index.md b/doc/topics/autodevops/index.md index 767ea5ee7b7..01e61095fe2 100644 --- a/doc/topics/autodevops/index.md +++ b/doc/topics/autodevops/index.md @@ -104,16 +104,20 @@ knowledge of the following: - [GitLab Runner](https://docs.gitlab.com/runner/) - [Prometheus](https://prometheus.io/docs/introduction/overview/) -Auto DevOps provides great defaults for all the stages; you can, however, +Auto DevOps provides great defaults for all the stages and makes use of [CI templates](https://gitlab.com/gitlab-org/gitlab/-/tree/master/lib/gitlab/ci/templates); you can, however, [customize](customize.md) almost everything to your needs. For an overview on the creation of Auto DevOps, read more [in this blog post](https://about.gitlab.com/blog/2017/06/29/whats-next-for-gitlab-ci/). -NOTE: **Note** +NOTE: **Note:** Kubernetes clusters can [be used without](../../user/project/clusters/index.md) Auto DevOps. +## Kubernetes requirements + +See [Auto DevOps requirements for Kubernetes](requirements.md#auto-devops-requirements-for-kubernetes). + ## Auto DevOps base domain The Auto DevOps base domain is required to use @@ -163,6 +167,10 @@ set the Auto DevOps base domain to `1.2.3.4.nip.io`. After completing setup, all requests hit the load balancer, which routes requests to the Kubernetes pods running your application. +### AWS ECS + +See [Auto DevOps requirements for Amazon ECS](requirements.md#auto-devops-requirements-for-amazon-ecs). + ## Enabling/Disabling Auto DevOps When first using Auto DevOps, review the [requirements](#requirements) to ensure @@ -185,7 +193,7 @@ If enabling, check that your project does not have a `.gitlab-ci.yml`, or if one and choose the [deployment strategy](#deployment-strategy). 1. Click **Save changes** for the changes to take effect. -After enabling the feature, an Auto DevOps pipeline is triggered on the default branch. +After enabling the feature, an Auto DevOps pipeline is triggered on the `master` branch. ### At the group level @@ -240,11 +248,11 @@ TIP: **Tip:** Use the [blue-green deployment](../../ci/environments/incremental_rollouts.md#blue-green-deployment) technique to minimize downtime and risk. -## Using multiple Kubernetes clusters **(PREMIUM)** +## Using multiple Kubernetes clusters When using Auto DevOps, you can deploy different environments to different Kubernetes clusters, due to the 1:1 connection -[existing between them](../../user/project/clusters/index.md#multiple-kubernetes-clusters-premium). +[existing between them](../../user/project/clusters/index.md#multiple-kubernetes-clusters). The [Deploy Job template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml) used by Auto DevOps currently defines 3 environment names: @@ -271,8 +279,8 @@ To add a different cluster for each environment: 1. Navigate to your project's **{cloud-gear}** **Operations > Kubernetes**. 1. Create the Kubernetes clusters with their respective environment scope, as described from the table above. -1. After creating the clusters, navigate to each cluster and install Helm Tiller - and Ingress. Wait for the Ingress IP address to be assigned. +1. After creating the clusters, navigate to each cluster and install + Ingress. Wait for the Ingress IP address to be assigned. 1. Make sure you've [configured your DNS](#auto-devops-base-domain) with the specified Auto DevOps domains. 1. Navigate to each cluster's page, through **{cloud-gear}** **Operations > Kubernetes**, @@ -293,9 +301,9 @@ No documented way of using private container registry with Auto DevOps exists. We strongly advise using GitLab Container Registry with Auto DevOps to simplify configuration and prevent any unforeseen issues. -### Installing Helm behind a proxy +### Install applications behind a proxy -GitLab does not support installing [Helm as a GitLab-managed App](../../user/clusters/applications.md#helm) when +GitLab's Helm integration does not support installing applications when behind a proxy. Users who want to do so must inject their proxy settings into the installation pods at runtime, such as by using a [`PodPreset`](https://kubernetes.io/docs/concepts/workloads/pods/podpreset/): @@ -358,6 +366,55 @@ Auto Deploy will fail if GitLab can't create a Kubernetes namespace and service account for your project. For help debugging this issue, see [Troubleshooting failed deployment jobs](../../user/project/clusters/index.md#troubleshooting). +### Detected an existing PostgreSQL database + +After upgrading to GitLab 13.0, you may encounter this message when deploying +with Auto DevOps: + +```plaintext +Detected an existing PostgreSQL database installed on the +deprecated channel 1, but the current channel is set to 2. The default +channel changed to 2 in of GitLab 13.0. +[...] +``` + +Auto DevOps, by default, installs an in-cluster PostgreSQL database alongside +your application. The default installation method changed in GitLab 13.0, and +upgrading existing databases requires user involvement. The two installation +methods are: + +- **channel 1 (deprecated):** Pulls in the database as a dependency of the associated + Helm chart. Only supports Kubernetes versions up to version 1.15. +- **channel 2 (current):** Installs the database as an independent Helm chart. Required + for using the in-cluster database feature with Kubernetes versions 1.16 and greater. + +If you receive this error, you can do one of the following actions: + +- You can *safely* ignore the warning and continue using the channel 1 PostgreSQL + database by setting `AUTO_DEVOPS_POSTGRES_CHANNEL` to `1` and redeploying. + +- You can delete the channel 1 PostgreSQL database and install a fresh channel 2 + database by setting `AUTO_DEVOPS_POSTGRES_DELETE_V1` to a non-empty value and + redeploying. + + DANGER: **Danger:** + Deleting the channel 1 PostgreSQL database permanently deletes the existing + channel 1 database and all its data. See + [Upgrading PostgreSQL](upgrading_postgresql.md) + for more information on backing up and upgrading your database. + +- If you are not using the in-cluster database, you can set + `POSTGRES_ENABLED` to `false` and re-deploy. This option is especially relevant to + users of *custom charts without the in-chart PostgreSQL dependency*. + Database auto-detection is based on the `postgresql.enabled` Helm value for + your release. This value is set based on the `POSTGRES_ENABLED` CI variable + and persisted by Helm, regardless of whether or not your chart uses the + variable. + +DANGER: **Danger:** +Setting `POSTGRES_ENABLED` to `false` permanently deletes any existing +channel 1 database for your environment. + ## Development guides [Development guide for Auto DevOps](../../development/auto_devops.md) diff --git a/doc/topics/autodevops/quick_start_guide.md b/doc/topics/autodevops/quick_start_guide.md index ec5191dd4ac..4f8074f047e 100644 --- a/doc/topics/autodevops/quick_start_guide.md +++ b/doc/topics/autodevops/quick_start_guide.md @@ -98,24 +98,10 @@ status on your [GCP dashboard](https://console.cloud.google.com/kubernetes). Next, you will install some applications on your cluster that are needed to take full advantage of Auto DevOps. -## Install the package manager - -After creating your Kubernetes cluster, GitLab's Kubernetes integration provides -[pre-defined applications](../../user/project/clusters/index.md#installing-applications) -for you to install. To install them, you must next install Helm Tiller, the -Kubernetes package manager for Kubernetes, to enable the installation of other applications. - -Next to **Helm Tiller**, click **Install**. - - - -After installation completes, the page reloads, and you can install other -applications. - ## Install Ingress and Prometheus -After installing **Helm Tiller**, you can install other applications that rely on it, -including Ingress and Prometheus, which we will install in this quick start guide: +After your cluster is running, you can install your first applications. +In this guide, we will install Ingress and Prometheus: - Ingress - Provides load balancing, SSL termination, and name-based virtual hosting, using NGINX behind the scenes. @@ -305,7 +291,7 @@ all within GitLab. Despite its automatic nature, Auto DevOps can also be configu and customized to fit your workflow. Here are some helpful resources for further reading: 1. [Auto DevOps](index.md) -1. [Multiple Kubernetes clusters](index.md#using-multiple-kubernetes-clusters-premium) **(PREMIUM)** +1. [Multiple Kubernetes clusters](index.md#using-multiple-kubernetes-clusters) 1. [Incremental rollout to production](customize.md#incremental-rollout-to-production-premium) **(PREMIUM)** 1. [Disable jobs you don't need with environment variables](customize.md#environment-variables) 1. [Use a static IP for your cluster](../../user/clusters/applications.md#using-a-static-ip) diff --git a/doc/topics/autodevops/requirements.md b/doc/topics/autodevops/requirements.md index b09a571fd16..33db94be97e 100644 --- a/doc/topics/autodevops/requirements.md +++ b/doc/topics/autodevops/requirements.md @@ -103,22 +103,26 @@ After all requirements are met, you can [enable Auto DevOps](index.md#enablingdi > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/208132) in GitLab 13.0. -You can choose to target [Amazon Elastic Container Service (ECS)](../../ci/cloud_deployment/index.md) as a deployment platform instead of using Kubernetes. +You can choose to target [AWS ECS](../../ci/cloud_deployment/index.md) as a deployment platform instead of using Kubernetes. -To get started on Auto DevOps to Amazon ECS, you'll have to add a specific Environment +To get started on Auto DevOps to AWS ECS, you'll have to add a specific Environment Variable. To do so, follow these steps: 1. In your project, go to **Settings > CI / CD** and expand the **Variables** section. 1. Specify which AWS platform to target during the Auto DevOps deployment - by adding the `AUTO_DEVOPS_PLATFORM_TARGET` variable. + by adding the `AUTO_DEVOPS_PLATFORM_TARGET` variable with one of the following values: + - `FARGATE` if the service you're targeting must be of launch type FARGATE. + - `ECS` if you're not enforcing any launch type check when deploying to ECS. -1. Give this variable the value `ECS` before saving it. - -When you trigger a pipeline, if Auto DevOps is enabled and if you've correctly +When you trigger a pipeline, if you have Auto DevOps enabled and if you have correctly [entered AWS credentials as environment variables](../../ci/cloud_deployment/index.md#deploy-your-application-to-the-aws-elastic-container-service-ecs), -your application will be deployed to Amazon ECS. +your application will be deployed to AWS ECS. + +NOTE: **Note:** +[GitLab Managed Apps](../../user/clusters/applications.md) are not available when deploying to AWS ECS. +You must manually configure your application (such as Ingress or Help) on AWS ECS. NOTE: **Note:** If you have both a valid `AUTO_DEVOPS_PLATFORM_TARGET` variable and a Kubernetes cluster tied to your project, @@ -130,5 +134,5 @@ defined in the [`Jobs/Deploy/ECS.gitlab-ci.yml` template](https://gitlab.com/git However, it's not recommended to [include](../../ci/yaml/README.md#includetemplate) it on its own. This template is designed to be used with Auto DevOps only. It may change unexpectedly causing your pipeline to fail if included on its own. Also, the job -names within this template may also change. Don't override these jobs' names in your +names within this template may also change. Do not override these jobs' names in your own pipeline, as the override will stop working when the name changes. diff --git a/doc/topics/autodevops/stages.md b/doc/topics/autodevops/stages.md index 0c7c4919431..bf1594130f4 100644 --- a/doc/topics/autodevops/stages.md +++ b/doc/topics/autodevops/stages.md @@ -72,7 +72,8 @@ Heroku buildpacks, with the following caveats: - The `/bin/herokuish` command is not present in the resulting image, and prefixing commands with `/bin/herokuish procfile exec` is no longer required (nor possible). -NOTE: **Note**: Auto Test still uses Herokuish, as test suite detection is not +NOTE: **Note:** +Auto Test still uses Herokuish, as test suite detection is not yet part of the Cloud Native Buildpack specification. For more information, see [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/212689). @@ -294,7 +295,8 @@ You can disable DAST: > Introduced in [GitLab Premium](https://about.gitlab.com/pricing/) 10.4. -Auto Browser Performance Testing measures the performance of a web page with the +Auto [Browser Performance Testing](../../user/project/merge_requests/browser_performance_testing.md) +measures the browser performance of a web page with the [Sitespeed.io container](https://hub.docker.com/r/sitespeedio/sitespeed.io/), creates a JSON report including the overall performance score for each page, and uploads the report as an artifact. By default, it tests the root page of your Review and @@ -307,9 +309,26 @@ file named `.gitlab-urls.txt` in the root directory, one file per line. For exam /direction ``` -Any performance differences between the source and target branches are also +Any browser performance differences between the source and target branches are also [shown in the merge request widget](../../user/project/merge_requests/browser_performance_testing.md). +## Auto Load Performance Testing **(PREMIUM)** + +> Introduced in [GitLab Premium](https://about.gitlab.com/pricing/) 13.2. + +Auto [Load Performance Testing](../../user/project/merge_requests/load_performance_testing.md) +measures the server performance of an application with the +[k6 container](https://hub.docker.com/r/loadimpact/k6/), +creates a JSON report including several key result metrics, and +uploads the report as an artifact. + +Some initial setup is required. A [k6](https://k6.io/) test needs to be +written that's tailored to your specific application. The test also needs to be +configured so it can pick up the environment's dynamic URL via an environment variable. + +Any load performance test result differences between the source and target branches are also +[shown in the merge request widget](../../user/project/merge_requests/load_performance_testing.md). + ## Auto Deploy This is an optional step, since many projects don't have a Kubernetes cluster @@ -372,7 +391,7 @@ as it attempts to fetch the image using `CI_REGISTRY_PASSWORD`. > - Support for deploying a PostgreSQL version that supports Kubernetes 1.16+ was [introduced](https://gitlab.com/gitlab-org/cluster-integration/auto-deploy-image/-/merge_requests/49) in GitLab 12.9. > - Supported out of the box for new deployments as of GitLab 13.0. -CAUTION: **Deprecation** +CAUTION: **Deprecation:** The default value for the `deploymentApiVersion` setting was changed from `extensions/v1beta` to `apps/v1` in [GitLab 13.0](https://gitlab.com/gitlab-org/charts/auto-deploy-app/-/issues/47). @@ -398,7 +417,8 @@ To use Auto Deploy on a Kubernetes 1.16+ cluster: 1. If you are deploying your application for the first time and are using GitLab 12.9 or 12.10, set `AUTO_DEVOPS_POSTGRES_CHANNEL` to `2`. -DANGER: **Danger:** On GitLab 12.9 and 12.10, opting into +DANGER: **Danger:** +On GitLab 12.9 and 12.10, opting into `AUTO_DEVOPS_POSTGRES_CHANNEL` version `2` deletes the version `1` PostgreSQL database. Follow the [guide to upgrading PostgreSQL](upgrading_postgresql.md) to back up and restore your database before opting into version `2` (On @@ -437,6 +457,10 @@ Herokuish, and you must prefix commands run in these images with `/bin/herokuish procfile exec` to replicate the environment where your application will run. +### Upgrade auto-deploy-app Chart + +You can upgrade auto-deploy-app chart by following the [upgrade guide](upgrading_chart.md). + ### Workers Some web applications must run extra deployments for "worker processes". For @@ -469,16 +493,16 @@ workers: sidekiq: replicaCount: 1 command: - - /bin/herokuish - - procfile - - exec - - sidekiq + - /bin/herokuish + - procfile + - exec + - sidekiq preStopCommand: - - /bin/herokuish - - procfile - - exec - - sidekiqctl - - quiet + - /bin/herokuish + - procfile + - exec + - sidekiqctl + - quiet terminationGracePeriodSeconds: 60 ``` @@ -524,12 +548,12 @@ networkPolicy: matchLabels: app.gitlab.com/env: staging ingress: - - from: - - podSelector: - matchLabels: {} - - namespaceSelector: - matchLabels: - app.gitlab.com/managed_by: gitlab + - from: + - podSelector: + matchLabels: {} + - namespaceSelector: + matchLabels: + app.gitlab.com/managed_by: gitlab ``` For more information on installing Network Policies, see diff --git a/doc/topics/autodevops/upgrading_chart.md b/doc/topics/autodevops/upgrading_chart.md new file mode 100644 index 00000000000..e4dacdfcf5b --- /dev/null +++ b/doc/topics/autodevops/upgrading_chart.md @@ -0,0 +1,72 @@ +# Upgrading auto-deploy-app chart for Auto DevOps + +Auto DevOps provides the auto-deploy-app chart for deploying your application to the +Kubernetes cluster with Helm/Tiller. Major version changes of this chart could have +a significantly different resource architecture, and may not be backwards compatible. + +This guide provides instructions on how to upgrade your deployments to use the latest +chart and resource architecture. + +## Compatibility + +The following table lists the version compatibility between GitLab and [auto-deploy-image](https://gitlab.com/gitlab-org/cluster-integration/auto-deploy-image) (with the [auto-deploy-app chart](https://gitlab.com/gitlab-org/cluster-integration/auto-deploy-image/-/tree/master/assets/auto-deploy-app)). + +| GitLab version | auto-deploy-image version | Notes | +|------------------|---------------------------|--------------------------------------------| +| v10.0 and higher | v0.1.0 and higher | v0 and v1 charts are backwards compatible. | + +## Upgrade Guide + +The Auto DevOps project must use the unmodified chart managed by GitLab. +[Customized charts](customize.md#custom-helm-chart) are unsupported. + +### v1 chart + +The v1 chart is backward compatible with the v0 chart, so no configuration changes are needed. + +## Troubleshooting + +### Major version mismatch warning + +If deploying a chart that has a major version that is different from the previous one, +the new chart might not be correctly deployed. This could be due to an architectural +change. If that happens, the deployment job fails with a message similar to: + +```plaintext +************************************************************************************* + [WARNING] +Detected a major version difference between the the chart that is currently deploying (auto-deploy-app-v0.7.0), and the previously deployed chart (auto-deploy-app-v1.0.0). +A new major version might not be backward compatible with the current release (production). The deployment could fail or be stuck in an unrecoverable status. +... +``` + +To clear this error message and resume deployments, you must do one of the following: + +- Manually [upgrade the chart version](#upgrade-guide). +- [Use a specific chart version](#use-a-specific-chart-version). + +#### Use a specific chart version + +To use a specific chart version, you must specify a corresponding version of [auto-deploy-image](https://gitlab.com/gitlab-org/cluster-integration/auto-deploy-image). +Do this by [customizing the image in your `.gitlab-ci.yml`](customize.md#customizing-gitlab-ciyml). + +For example, create the following `.gitlab-ci.yml` file in the project. It configures Auto DevOps +to use [auto-deploy-image](https://gitlab.com/gitlab-org/cluster-integration/auto-deploy-image) version `v0.17.0` +for deployment jobs. It will download the chart from [chart repository](https://charts.gitlab.io/): + +```yaml +include: + - template: Auto-DevOps.gitlab-ci.yml + +.auto-deploy: + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.17.0" +``` + +### Ignore warning and continue deploying + +If you are certain that the new chart version is safe to be deployed, +you can add the `AUTO_DEVOPS_ALLOW_TO_FORCE_DEPLOY_V<N>` [environment variable](customize.md#build-and-deployment) +to force the deployment to continue, where `<N>` is the major version. + +For example, if you want to deploy the v2.0.0 chart on a deployment that previously +used the v0.17.0 chart, add `AUTO_DEVOPS_ALLOW_TO_FORCE_DEPLOY_V2`. diff --git a/doc/topics/autodevops/upgrading_postgresql.md b/doc/topics/autodevops/upgrading_postgresql.md index 893f7ba7cde..2ebe362280f 100644 --- a/doc/topics/autodevops/upgrading_postgresql.md +++ b/doc/topics/autodevops/upgrading_postgresql.md @@ -26,8 +26,12 @@ involves: This varies based on Kubernetes providers. 1. Prepare for downtime. The steps below include taking the application offline so that the in-cluster database does not get modified after the database dump is created. +1. Ensure you have not set `POSTGRES_ENABLED` to `false`, as this setting deletes + any existing channel 1 database. For more information, see + [Detected an existing PostgreSQL database](index.md#detected-an-existing-postgresql-database). -TIP: **Tip:** If you have configured Auto DevOps to have staging, +TIP: **Tip:** +If you have configured Auto DevOps to have staging, consider trying out the backup and restore steps on staging first, or trying this out on a review app. @@ -158,12 +162,13 @@ pvc-9085e3d3-5239-11ea-9c8d-42010a8e0096 8Gi RWO Retain ## Install new PostgreSQL -CAUTION: **Caution:** Using the newer version of PostgreSQL will delete +CAUTION: **Caution:** +Using the newer version of PostgreSQL will delete the older 0.7.1 PostgreSQL. To prevent the underlying data from being -deleted, you can choose to retain the [persistent -volume](#retain-persistent-volumes). +deleted, you can choose to retain the [persistent volume](#retain-persistent-volumes). -TIP: **Tip:** You can also +TIP: **Tip:** +You can also [scope](../../ci/environments/index.md#scoping-environments-with-specs) the `AUTO_DEVOPS_POSTGRES_CHANNEL`, `AUTO_DEVOPS_POSTGRES_DELETE_V1` and `POSTGRES_VERSION` variables to specific environments, e.g. `staging`. diff --git a/doc/topics/git/index.md b/doc/topics/git/index.md index 2e36fea14bf..89da3dfdbd0 100644 --- a/doc/topics/git/index.md +++ b/doc/topics/git/index.md @@ -26,8 +26,9 @@ The following resources will help you get started with Git: - [Git Basics](https://git-scm.com/book/en/v2/Getting-Started-Git-Basics) - [Git on the Server - GitLab](https://git-scm.com/book/en/v2/Git-on-the-Server-GitLab) - [How to install Git](how_to_install_git/index.md) +- [Git terminology](../../gitlab-basics/start-using-git.md#git-terminology) - [Start using Git on the command line](../../gitlab-basics/start-using-git.md) -- [Command line file editing basic commands](../../gitlab-basics/command-line-commands.md) +- [Edit files through the command line](../../gitlab-basics/command-line-commands.md) - [GitLab Git Cheat Sheet (download)](https://about.gitlab.com/images/press/git-cheat-sheet.pdf) - Commits: - [Revert a commit](../../user/project/merge_requests/revert_changes.md#reverting-a-commit) diff --git a/doc/topics/git/lfs/index.md b/doc/topics/git/lfs/index.md index 706d3c3eddf..1e2f45fd67b 100644 --- a/doc/topics/git/lfs/index.md +++ b/doc/topics/git/lfs/index.md @@ -89,6 +89,9 @@ that are on the remote repository, such as for a branch from origin: git lfs fetch origin master ``` +Make sure your files aren't listed in `.gitignore`, otherwise, they will be ignored by Git thus will not +be pushed to the remote repository. + ### Migrate an existing repo to Git LFS Read the documentation on how to [migrate an existing Git repository with Git LFS](migrate_to_git_lfs.md). diff --git a/doc/topics/git/lfs/migrate_to_git_lfs.md b/doc/topics/git/lfs/migrate_to_git_lfs.md index a64639a9238..3e287c0816d 100644 --- a/doc/topics/git/lfs/migrate_to_git_lfs.md +++ b/doc/topics/git/lfs/migrate_to_git_lfs.md @@ -21,7 +21,7 @@ and lastly create LFS tracking rules to prevent new binary files from being added. This tutorial was inspired by the guide -[Use BFG to migrate a repository to Git LFS](https://confluence.atlassian.com/bitbucket/use-bfg-to-migrate-a-repo-to-git-lfs-834233484.html). +[Use BFG to migrate a repository to Git LFS](https://support.atlassian.com/bitbucket-cloud/docs/use-bfg-to-migrate-a-repo-to-git-lfs/). For more information on Git LFS, see the [references](#references) below. diff --git a/doc/topics/git/partial_clone.md b/doc/topics/git/partial_clone.md index 7462406cad3..de25c8a3283 100644 --- a/doc/topics/git/partial_clone.md +++ b/doc/topics/git/partial_clone.md @@ -84,7 +84,7 @@ Updating files: 100% (28/28), done. $ cd www-gitlab-com -$ git sparse-checkout init --cone +$ git sparse-checkout init --clone $ git sparse-checkout add data remote: Enumerating objects: 301, done. @@ -113,9 +113,6 @@ file to specify which files should be included when cloning and fetching. For more details, see the Git documentation for [`rev-list-options`](https://gitlab.com/gitlab-org/git/-/blob/9fadedd637b312089337d73c3ed8447e9f0aa775/Documentation/rev-list-options.txt#L735-780). -With the `uploadpack.allowFilter` and `uploadpack.allowAnySHA1InWant` options -enabled on the Git server: - 1. **Create a filter spec.** For example, consider a monolithic repository with many applications, each in a different subdirectory in the root. Create a file `shiny-app/.filterspec` using the GitLab web interface: diff --git a/doc/topics/gitlab_flow.md b/doc/topics/gitlab_flow.md index 6382ac0957a..04c942ab532 100644 --- a/doc/topics/gitlab_flow.md +++ b/doc/topics/gitlab_flow.md @@ -176,9 +176,9 @@ The name of a branch might be dictated by organizational standards. When you are done or want to discuss the code, open a merge request. A merge request is an online place to discuss the change and review the code. -If you open the merge request but do not assign it to anyone, it is a "Work In Progress" merge request. +If you open the merge request but do not assign it to anyone, it is a [draft merge request](../user/project/merge_requests/work_in_progress_merge_requests.md). These are used to discuss the proposed implementation but are not ready for inclusion in the `master` branch yet. -Start the title of the merge request with `[WIP]` or `WIP:` to prevent it from being merged before it's ready. +Start the title of the merge request with `[Draft]`, `Draft:` or `(Draft)` to prevent it from being merged before it's ready. When you think the code is ready, assign the merge request to a reviewer. The reviewer can merge the changes when they think the code is ready for inclusion in the `master` branch. diff --git a/doc/topics/web_application_firewall/index.md b/doc/topics/web_application_firewall/index.md index 57043bf73b3..5ce7c0779bb 100644 --- a/doc/topics/web_application_firewall/index.md +++ b/doc/topics/web_application_firewall/index.md @@ -1,3 +1,10 @@ +--- +stage: Defend +group: Container Security +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + + # Web Application Firewall - ModSecurity A web application firewall (or WAF) filters, monitors, and blocks HTTP traffic to @@ -18,7 +25,7 @@ applications which have an Ingress. The ModSecurity module runs with the [OWASP Core Rule Set (CRS)](https://coreruleset.org/) by default. The OWASP CRS will detect and log a wide range of common attacks. -NOTE: **Note** +NOTE: **Note:** The WAF is deployed in "Detection-only mode" by default and will only log attack attempts. @@ -53,7 +60,7 @@ If you are using a self-managed instance of GitLab, you need to configure the [Google OAuth2 OmniAuth Provider](../../integration/google.md) before you can configure a cluster on GKE. Once this is set up, you can follow the steps on the [quick start guide](quick_start_guide.md) to get started. -NOTE: **Note** +NOTE: **Note:** This guide shows how the WAF can be deployed using Auto DevOps. The WAF is available by default to all applications no matter how they are deployed, as long as they are using Ingress. diff --git a/doc/topics/web_application_firewall/quick_start_guide.md b/doc/topics/web_application_firewall/quick_start_guide.md index ec6702bb457..9e69bc7e7c7 100644 --- a/doc/topics/web_application_firewall/quick_start_guide.md +++ b/doc/topics/web_application_firewall/quick_start_guide.md @@ -1,3 +1,9 @@ +--- +stage: Defend +group: Container Security +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Getting started with the Web Application Firewall This is a step-by-step guide that will help you use GitLab's [Web Application Firewall](index.md) after @@ -11,7 +17,7 @@ These instructions will also work for a self-managed GitLab instance. However, y need to ensure your own [Runners are configured](../../ci/runners/README.md) and [Google OAuth is enabled](../../integration/google.md). -**Note**: GitLab's Web Application Firewall is deployed with [Ingress](../../user/clusters/applications.md#Ingress), +**Note**: GitLab's Web Application Firewall is deployed with [Ingress](../../user/clusters/applications.md#ingress), so it will be available to your applications no matter how you deploy them to Kubernetes. ## Configuring your Google account @@ -86,7 +92,7 @@ status on your [GCP dashboard](https://console.cloud.google.com/kubernetes). The next step is to install some applications on your cluster that are needed to take full advantage of Auto DevOps. -## Installing Helm and Ingress +## Install Ingress GitLab's Kubernetes integration comes with some [pre-defined applications](../../user/project/clusters/index.md#installing-applications) @@ -94,12 +100,6 @@ for you to install.  -The first one to install is Helm Tiller, a package manager for Kubernetes, which -is needed in order to install the rest of the applications. Go ahead and click -its **Install** button. -Once it is installed, the other applications that rely on it will each have their -**Install** buttons enabled. - For this guide, we need to install Ingress. Ingress provides load balancing, SSL termination, and name-based virtual hosting, using NGINX behind the scenes. Make sure to switch the toogle to the enabled position before installing. diff --git a/doc/university/README.md b/doc/university/README.md index df41167dbe9..2a9111276d3 100644 --- a/doc/university/README.md +++ b/doc/university/README.md @@ -72,7 +72,7 @@ The GitLab University curriculum is composed of GitLab videos, screencasts, pres 1. [Getting Help](https://about.gitlab.com/get-help/) - Proposing Features and Reporting and Tracking bugs for GitLab - - The GitLab IRC channel, Gitter Chat Room, Community Forum and Mailing List + - The GitLab IRC channel, Gitter Chat Room, Community Forum, and Mailing List - Getting Technical Support - Being part of our Great Community and Contributing to GitLab 1. [Getting Started with the GitLab Development Kit (GDK)](https://about.gitlab.com/blog/2016/06/08/getting-started-with-gitlab-development-kit/) @@ -140,11 +140,11 @@ The GitLab University curriculum is composed of GitLab videos, screencasts, pres ## 3. GitLab Advanced -### 3.1. Dev Ops +### 3.1. DevOps -1. [Xebia Labs: Dev Ops Terminology](https://xebialabs.com/glossary/) -1. [Xebia Labs: Periodic Table of DevOps Tools](https://xebialabs.com/periodic-table-of-devops-tools/) -1. [Puppet Labs: State of Dev Ops 2016 - Book](https://puppet.com/resources/report/2016-state-devops-report/) +1. [XebiaLabs: DevOps Terminology](https://xebialabs.com/glossary/) +1. [XebiaLabs: Periodic Table of DevOps Tools](https://digital.ai/periodic-table-of-devops-tools) +1. [Puppet Labs: State of DevOps 2016 - Book](https://puppet.com/resources/report/2016-state-devops-report/) ### 3.2. Installing GitLab with Omnibus diff --git a/doc/university/bookclub/index.md b/doc/university/bookclub/index.md index c6dad663c0e..71dfe7fc3cb 100644 --- a/doc/university/bookclub/index.md +++ b/doc/university/bookclub/index.md @@ -15,7 +15,7 @@ See the [book list](booklist.md) for additional recommendations. 1. **Remote: Office not required** David Heinemeier Hansson and Jason Fried, 2013 - ([amazon](https://www.amazon.co.uk/Remote-Required-David-Heinemeier-Hansson/dp/0091954673)) + ([Amazon](https://www.amazon.co.uk/dp/0091954673/ref=cm_sw_r_tw_dp_x_0yy9EbZ2WXJ6Y)) 1. **The Year Without Pants** diff --git a/doc/university/training/end-user/README.md b/doc/university/training/end-user/README.md index 0465f48bb34..8d25b865855 100644 --- a/doc/university/training/end-user/README.md +++ b/doc/university/training/end-user/README.md @@ -159,7 +159,7 @@ git push origin squash_some_bugs - When you want feedback create a merge request - Target is the ‘default’ branch (usually master) - Assign or mention the person you would like to review -- Add `WIP` to the title if it's a work in progress +- Add `Draft:` to the title if it's a work in progress - When accepting, always delete the branch - Anyone can comment, not just the assignee - Push corrections to the same branch diff --git a/doc/university/training/topics/merge_requests.md b/doc/university/training/topics/merge_requests.md index 0ea8b707ea9..ea679a7d66f 100644 --- a/doc/university/training/topics/merge_requests.md +++ b/doc/university/training/topics/merge_requests.md @@ -7,7 +7,7 @@ comments: false - When you want feedback create a merge request - Target is the default branch (usually master) - Assign or mention the person you would like to review -- Add 'WIP' to the title if it's a work in progress +- Add `[Draft]` to the title if it's a work in progress - When accepting, always delete the branch - Anyone can comment, not just the assignee - Push corrections to the same branch diff --git a/doc/university/training/user_training.md b/doc/university/training/user_training.md index d2def1f3199..43de90010b1 100644 --- a/doc/university/training/user_training.md +++ b/doc/university/training/user_training.md @@ -186,7 +186,7 @@ git push origin squash_some_bugs - When you want feedback create a merge request. - Target is the ‘default’ branch (usually master). - Assign or mention the person you would like to review. -- Add 'WIP' to the title if it's a work in progress. +- Add `[Draft]` to the title if it's a work in progress. - When accepting, always delete the branch. - Anyone can comment, not just the assignee. - Push corrections to the same branch. diff --git a/doc/update/README.md b/doc/update/README.md index f36a304495c..26e52229fd2 100644 --- a/doc/update/README.md +++ b/doc/update/README.md @@ -147,8 +147,37 @@ puts Sidekiq::Queue.new("background_migration").size Sidekiq::ScheduledSet.new.select { |r| r.klass == 'BackgroundMigrationWorker' }.size ``` -There is also a [Rake task](../administration/raketasks/maintenance.md#display-status-of-database-migrations) -for displaying the status of each database migration. +### What do I do if my background migrations are stuck? + +CAUTION: **Warning:** +The following operations can disrupt your GitLab performance. + +NOTE: **Note:** +It is safe to re-execute these commands, especially if you have 1000+ pending jobs which would likely overflow your runtime memory. + +**For Omnibus installations** + +```shell +# Start the rails console +sudo gitlab-rails c + +# Execute the following in the rails console +scheduled_queue = Sidekiq::ScheduledSet.new +pending_job_classes = scheduled_queue.select { |job| job["class"] == "BackgroundMigrationWorker" }.map { |job| job["args"].first }.uniq +pending_job_classes.each { |job_class| Gitlab::BackgroundMigration.steal(job_class) } +``` + +**For installations from source** + +```shell +# Start the rails console +sudo -u git -H bundle exec rails RAILS_ENV=production + +# Execute the following in the rails console +scheduled_queue = Sidekiq::ScheduledSet.new +pending_job_classes = scheduled_queue.select { |job| job["class"] == "BackgroundMigrationWorker" }.map { |job| job["args"].first }.uniq +pending_job_classes.each { |job_class| Gitlab::BackgroundMigration.steal(job_class) } +``` ## Upgrading to a new major version @@ -192,6 +221,12 @@ possible. ## Version specific upgrading instructions +### 13.2.0 + +GitLab installations that have multiple web nodes will need to be +[upgraded to 13.1](#1310) before upgrading to 13.2 (and later) due to a +breaking change in Rails that can result in authorization issues. + ### 13.1.0 In 13.1.0, you must upgrade to either: @@ -202,6 +237,27 @@ In 13.1.0, you must upgrade to either: Failure to do so will result in internal errors in the Gitaly service in some RPCs due to the use of the new `--end-of-options` Git flag. +Additionally, in GitLab 13.1.0, the version of [Rails was upgraded from 6.0.3 to +6.0.3.1](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/33454). +The Rails upgrade included a change to CSRF token generation which is +not backwards-compatible - GitLab servers with the new Rails version +will generate CSRF tokens that are not recognizable by GitLab servers +with the older Rails version - which could cause non-GET requests to +fail for [multi-node GitLab installations](https://docs.gitlab.com/omnibus/update/#multi-node--ha-deployment). + +So, if you are using multiple Rails servers and specifically upgrading from 13.0, +all servers must first be upgraded to 13.1.0 before upgrading to later versions: + +1. Ensure all GitLab web nodes are on GitLab 13.1.0. +1. Optionally, enable the `global_csrf_token` feature flag to enable new + method of CSRF token generation: + + ```ruby + Feature.enable(:global_csrf_token) + ``` + +1. Only then, continue to upgrade to later versions of GitLab. + ### 12.2.0 In 12.2.0, we enabled Rails' authenticated cookie encryption. Old sessions are diff --git a/doc/update/restore_after_failure.md b/doc/update/restore_after_failure.md index c850bbff1cf..2c70e38d041 100644 --- a/doc/update/restore_after_failure.md +++ b/doc/update/restore_after_failure.md @@ -10,7 +10,9 @@ the previous version you were using. First, roll back the code or package. For source installations this involves checking out the older version (branch or tag). For Omnibus installations this -means installing the older `.deb` or `.rpm` package. Then, restore from a backup. +means installing the older +[`.deb` or `.rpm` package](https://packages.gitlab.com/gitlab). Then, restore from a +backup. Follow the instructions in the [Backup and Restore](../raketasks/backup_restore.md#restore-gitlab) documentation. diff --git a/doc/update/upgrading_from_ce_to_ee.md b/doc/update/upgrading_from_ce_to_ee.md index 78227f18457..f82f5001c89 100644 --- a/doc/update/upgrading_from_ce_to_ee.md +++ b/doc/update/upgrading_from_ce_to_ee.md @@ -4,7 +4,8 @@ comments: false # Upgrading from Community Edition to Enterprise Edition from source -NOTE: **NOTE** In the past we used separate documents for upgrading from +NOTE: **Note:** +In the past we used separate documents for upgrading from Community Edition to Enterprise Edition. These documents can be found in the [`doc/update` directory of Enterprise Edition's source code](https://gitlab.com/gitlab-org/gitlab/tree/11-8-stable-ee/doc/update). diff --git a/doc/user/admin_area/activating_deactivating_users.md b/doc/user/admin_area/activating_deactivating_users.md index 06a26737495..448c65038c2 100644 --- a/doc/user/admin_area/activating_deactivating_users.md +++ b/doc/user/admin_area/activating_deactivating_users.md @@ -39,7 +39,7 @@ A user can be deactivated from the Admin Area. To do this: Please note that for the deactivation option to be visible to an admin, the user: - Must be currently active. -- Must not have any signins or activity in the last 180 days. +- Must not have signed in, or have any activity, in the last 180 days. Users can also be deactivated using the [GitLab API](../../api/users.md#deactivate-user). diff --git a/doc/user/admin_area/credentials_inventory.md b/doc/user/admin_area/credentials_inventory.md index 5eecfbb73c6..9259c93cfa3 100644 --- a/doc/user/admin_area/credentials_inventory.md +++ b/doc/user/admin_area/credentials_inventory.md @@ -18,9 +18,11 @@ Using Credentials inventory, GitLab administrators can see all the personal acce - Who they belong to. - Their access scope. - Their usage pattern. +- When they expire. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/214809) in GitLab 13.2. +- When they were revoked. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/214809) in GitLab 13.2. To access the Credentials inventory, navigate to **Admin Area > Credentials**. The following is an example of the Credentials inventory page: - + diff --git a/doc/user/admin_area/img/credentials_inventory_v12_6.png b/doc/user/admin_area/img/credentials_inventory_v12_6.png Binary files differdeleted file mode 100644 index 5c16781cb2d..00000000000 --- a/doc/user/admin_area/img/credentials_inventory_v12_6.png +++ /dev/null diff --git a/doc/user/admin_area/img/credentials_inventory_v13_2.png b/doc/user/admin_area/img/credentials_inventory_v13_2.png Binary files differnew file mode 100644 index 00000000000..5b56422a0a3 --- /dev/null +++ b/doc/user/admin_area/img/credentials_inventory_v13_2.png diff --git a/doc/user/admin_area/img/mr_approval_settings_compliance_project_v13_1.png b/doc/user/admin_area/img/mr_approval_settings_compliance_project_v13_1.png Binary files differnew file mode 100644 index 00000000000..04d01f2662f --- /dev/null +++ b/doc/user/admin_area/img/mr_approval_settings_compliance_project_v13_1.png diff --git a/doc/user/admin_area/img/scope_mr_approval_settings_v13_1.png b/doc/user/admin_area/img/scope_mr_approval_settings_v13_1.png Binary files differnew file mode 100644 index 00000000000..c6ca2bac83c --- /dev/null +++ b/doc/user/admin_area/img/scope_mr_approval_settings_v13_1.png diff --git a/doc/user/admin_area/license.md b/doc/user/admin_area/license.md index 1ffaf4e0678..c5e29612596 100644 --- a/doc/user/admin_area/license.md +++ b/doc/user/admin_area/license.md @@ -5,7 +5,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w type: howto --- -# Activate all GitLab Enterprise Edition functionality with a license **(STARTER ONLY)** +# Activate GitLab EE with a license **(STARTER ONLY)** To activate all GitLab Enterprise Edition (EE) functionality, you need to upload a license. Once you've received your license from GitLab Inc., you can upload it @@ -107,14 +107,23 @@ expired license(s). It's possible to upload and view more than one license, but only the latest license will be used as the active license. -<!-- ## Troubleshooting +## Troubleshooting -Include any troubleshooting steps that you can foresee. If you know beforehand what issues -one might have when setting this up, or when something is changed, or on upgrading, it's -important to describe those, too. Think of things that may go wrong and include them here. -This is important to minimize requests for support, and to avoid doc comments with -questions that you know someone might ask. +### There is no License tab in the Admin Area -Each scenario can be a third-level heading, e.g. `### Getting error message X`. -If you have none to add when creating a doc, leave this section in place -but commented out to help encourage others to add to it in the future. --> +If you originally installed Community Edition rather than Enterprise Edition you will need to +[upgrade to Enterprise Edition](../../update/README.md#community-to-enterprise-edition) +before uploading your license. + +GitLab.com users cannot upload and use a self-managed license. If you +wish to use paid features on GitLab.com, a separate subscription may be +[purchased](../../subscriptions/index.md#subscribe-to-gitlabcom). + +### Users exceed license limit upon renewal + +If you've added new users to your GitLab instance prior to renewal you may need to +purchase additional seats to cover those users. If this is the case and a license +without enough users is uploaded a message is displayed prompting you to purchase +additional users. More information on how to determine the required number of users +and how to add additional seats can be found in the +[licensing FAQ](https://about.gitlab.com/pricing/licensing-faq/). diff --git a/doc/user/admin_area/merge_requests_approvals.md b/doc/user/admin_area/merge_requests_approvals.md index 153ccfc128a..6d9d634ce14 100644 --- a/doc/user/admin_area/merge_requests_approvals.md +++ b/doc/user/admin_area/merge_requests_approvals.md @@ -34,3 +34,22 @@ Merge request approval rules that can be set at an instance level are: - **Prevent users from modifying merge request approvers list**. Prevents project maintainers from allowing users to modify the approvers list in project settings or in individual merge requests. + +## Scope rules to compliance-labeled projects + +> Introduced in [GitLab Premium](https://gitlab.com/groups/gitlab-org/-/epics/3432) 13.2. + +Merge request approval rules can be further scoped to specific compliance frameworks. + +When the compliance framework label is selected and the project is assigned the compliance +label, the instance-level MR approval settings will take effect and the +[project-level settings](../project/merge_requests/merge_request_approvals.md#adding--editing-a-default-approval-rule) +is locked for modification. + +When the compliance framework label is not selected or the project is not assigned the +compliance label, the project-level MR approval settings will take effect and the users with +Maintainer role and above can modify these. + +| Instance-level | Project-level | +| -------------- | ------------- | +|  |  | diff --git a/doc/user/admin_area/monitoring/health_check.md b/doc/user/admin_area/monitoring/health_check.md index 91e29118e3e..329b6ff5bb0 100644 --- a/doc/user/admin_area/monitoring/health_check.md +++ b/doc/user/admin_area/monitoring/health_check.md @@ -137,8 +137,8 @@ This check is being exempt from Rack Attack. ## Access token (Deprecated) -> NOTE: **Note:** -> Access token has been deprecated in GitLab 9.4 in favor of [IP whitelist](#ip-whitelist). +NOTE: **Note:** +Access token has been deprecated in GitLab 9.4 in favor of [IP whitelist](#ip-whitelist). An access token needs to be provided while accessing the probe endpoints. The current accepted token can be found under the **Admin Area > Monitoring > Health check** @@ -152,6 +152,10 @@ The access token can be passed as a URL parameter: https://gitlab.example.com/-/readiness?token=ACCESS_TOKEN ``` +NOTE: **Note:** +In case the database or Redis service are unaccessible, the probe endpoints response is not guaranteed to be correct. +You should switch to [IP whitelist](#ip-whitelist) from deprecated access token to avoid it. + <!-- ## Troubleshooting Include any troubleshooting steps that you can foresee. If you know beforehand what issues diff --git a/doc/user/admin_area/settings/account_and_limit_settings.md b/doc/user/admin_area/settings/account_and_limit_settings.md index b4fb7a524da..167016f1cb5 100644 --- a/doc/user/admin_area/settings/account_and_limit_settings.md +++ b/doc/user/admin_area/settings/account_and_limit_settings.md @@ -30,11 +30,12 @@ details. This sets a maximum size limit on each namespace. The following are included in the namespace size: -- repository -- wiki +- Repository +- Wiki - LFS objects -- build artifacts -- packages +- Build artifacts +- Packages +- Snippets NOTE: **Note:** This limit is not currently enforced but will be in a future release. @@ -140,6 +141,39 @@ Once a lifetime for personal access tokens is set, GitLab will: allowed lifetime. Three hours is given to allow administrators to change the allowed lifetime, or remove it, before revocation takes place. +## Optional enforcement of Personal Access Token expiry **(ULTIMATE ONLY)** + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/214723) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.1. +> - It is deployed behind a feature flag, disabled by default. +> - It is disabled on GitLab.com. +> - It is not recommended for production use. +> - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enable-or-disable-optional-enforcement-of-personal-access-token-expiry-feature-core-only). **(CORE ONLY)** + +GitLab administrators can choose to prevent personal access tokens from expiring automatically. The tokens will be usable after the expiry date, unless they are revoked explicitly. + +To do this: + +1. Navigate to **Admin Area > Settings > General**. +1. Expand the **Account and limit** section. +1. Uncheck the **Enforce personal access token expiration** checkbox. + +### Enable or disable optional enforcement of Personal Access Token expiry Feature **(CORE ONLY)** + +Optional Enforcement of Personal Access Token Expiry is under development and not ready for production use. It is deployed behind a feature flag that is **disabled by default**. +[GitLab administrators with access to the GitLab Rails console](../../../administration/feature_flags.md) can enable it for your instance from the [rails console](../../../administration/feature_flags.md#start-the-gitlab-rails-console). + +To enable it: + +```ruby +Feature.enable(:enforce_personal_access_token_expiration) +``` + +To disable it: + +```ruby +Feature.disable(:enforce_personal_access_token_expiration) +``` + ## Disabling user profile name changes **(PREMIUM ONLY)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/24605) in GitLab 12.7. diff --git a/doc/user/admin_area/settings/continuous_integration.md b/doc/user/admin_area/settings/continuous_integration.md index 3a287f29a0a..0479da7fb52 100644 --- a/doc/user/admin_area/settings/continuous_integration.md +++ b/doc/user/admin_area/settings/continuous_integration.md @@ -17,8 +17,8 @@ You can find it in the **Admin Area > Settings > CI/CD**. To enable (or disable) [Auto DevOps](../../../topics/autodevops/index.md) for all projects: -1. Go to **Admin Area > Settings > CI/CD** -1. Check (or uncheck to disable) the box that says "Default to Auto DevOps pipeline for all projects" +1. Go to **Admin Area > Settings > CI/CD**. +1. Check (or uncheck to disable) the box that says **Default to Auto DevOps pipeline for all projects**. 1. Optionally, set up the [Auto DevOps base domain](../../../topics/autodevops/index.md#auto-devops-base-domain) which is going to be used for Auto Deploy and Auto Review Apps. 1. Hit **Save changes** for the changes to take effect. @@ -48,21 +48,21 @@ To change it at the: 1. Go to **Admin Area > Settings > CI/CD**. 1. Change the value of maximum artifacts size (in MB). - 1. Hit **Save changes** for the changes to take effect. + 1. Click **Save changes** for the changes to take effect. - [Group level](../../group/index.md#group-settings) (this will override the instance setting): 1. Go to the group's **Settings > CI / CD > General Pipelines**. 1. Change the value of **maximum artifacts size (in MB)**. - 1. Press **Save changes** for the changes to take effect. + 1. Click **Save changes** for the changes to take effect. - [Project level](../../../ci/pipelines/settings.md) (this will override the instance and group settings): 1. Go to the project's **Settings > CI / CD > General Pipelines**. 1. Change the value of **maximum artifacts size (in MB)**. - 1. Press **Save changes** for the changes to take effect. + 1. Click **Save changes** for the changes to take effect. -NOTE: **Note** +NOTE: **Note:** The setting at all levels is only available to GitLab administrators. ## Default artifacts expiration **(CORE ONLY)** @@ -70,18 +70,17 @@ The setting at all levels is only available to GitLab administrators. The default expiration time of the [job artifacts](../../../administration/job_artifacts.md) can be set in the Admin Area of your GitLab instance. The syntax of duration is described in [`artifacts:expire_in`](../../../ci/yaml/README.md#artifactsexpire_in) -and the default value is `30 days`. On GitLab.com they -[never expire](../../gitlab_com/index.md#gitlab-cicd). +and the default value is `30 days`. 1. Go to **Admin Area > Settings > CI/CD**. 1. Change the value of default expiration time. -1. Hit **Save changes** for the changes to take effect. +1. Click **Save changes** for the changes to take effect. This setting is set per job and can be overridden in [`.gitlab-ci.yml`](../../../ci/yaml/README.md#artifactsexpire_in). To disable the expiration, set it to `0`. The default unit is in seconds. -NOTE: **Note** +NOTE: **Note:** Any changes to this setting will apply to new artifacts only. The expiration time will not be updated for artifacts created before this setting was changed. The administrator may need to manually search for and expire previously-created @@ -101,9 +100,10 @@ On GitLab.com, the quota is calculated based on your To change the pipelines minutes quota: -1. Go to **Admin Area > Settings > CI/CD** -1. Set the pipeline minutes quota limit. -1. Hit **Save changes** for the changes to take effect +1. Go to **Admin Area > Settings > CI/CD**. +1. Expand **Continuous Integration and Deployment**. +1. In the **Pipeline minutes quota** box, enter the maximum number of minutes. +1. Click **Save changes** for the changes to take effect. --- @@ -112,8 +112,8 @@ also change each group's pipeline minutes quota to override the global value. 1. Navigate to the **Admin Area > Overview > Groups** and hit the **Edit** button for the group you wish to change the pipeline minutes quota. -1. Set the pipeline minutes quota to the desired value -1. Hit **Save changes** for the changes to take effect. +1. In the **Pipeline Minutes Quota** box, enter the maximum number of minutes. +1. Click **Save changes** for the changes to take effect. Once saved, you can see the build quota in the group admin view. The quota can also be viewed in the project admin view if shared Runners @@ -143,6 +143,8 @@ Once that time passes, the jobs will be archived and no longer able to be retried. Make it empty to never expire jobs. It has to be no less than 1 day, for example: <code>15 days</code>, <code>1 month</code>, <code>2 years</code>. +As of June 22, 2020 the [value is set](../../gitlab_com/index.md#gitlab-cicd) to 3 months on GitLab.com. Jobs created before that date will be archived after September 22, 2020. + ## Default CI configuration path > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/18073) in GitLab 12.5. diff --git a/doc/user/admin_area/settings/img/email_notification_for_unknown_sign_ins_v13_2.png b/doc/user/admin_area/settings/img/email_notification_for_unknown_sign_ins_v13_2.png Binary files differnew file mode 100644 index 00000000000..fdcc542c4d7 --- /dev/null +++ b/doc/user/admin_area/settings/img/email_notification_for_unknown_sign_ins_v13_2.png diff --git a/doc/user/admin_area/settings/img/import_export_rate_limits_v13_2.png b/doc/user/admin_area/settings/img/import_export_rate_limits_v13_2.png Binary files differnew file mode 100644 index 00000000000..78c94b3803c --- /dev/null +++ b/doc/user/admin_area/settings/img/import_export_rate_limits_v13_2.png diff --git a/doc/user/admin_area/settings/import_export_rate_limits.md b/doc/user/admin_area/settings/import_export_rate_limits.md new file mode 100644 index 00000000000..92cb2a1a109 --- /dev/null +++ b/doc/user/admin_area/settings/import_export_rate_limits.md @@ -0,0 +1,32 @@ +--- +type: reference +stage: Manage +group: Import +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Project/Group Import/Export rate limits + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/35728) in GitLab 13.2. + +The following table includes configurable rate limits. The following table includes limits on a +per minute per user basis: + +| Limit | Default (per minute per user) | +|--------------------------|-------------------------------| +| Project Import | 6 | +| Project Export | 6 | +| Project Export Download | 1 | +| Group Import | 6 | +| Group Export | 6 | +| Group Export Download | 1 | + +All rate limits are: + +- Configurable at **(admin)** **Admin Area > Settings > Network > Import/Export Rate Limits** +- Applied per minute per user +- Not applied per IP address +- Active by default. To disable, set the option to `0` +- Logged to `auth.log` file if exceed rate limit + + diff --git a/doc/user/admin_area/settings/index.md b/doc/user/admin_area/settings/index.md index df087722fcf..8c1e82f838b 100644 --- a/doc/user/admin_area/settings/index.md +++ b/doc/user/admin_area/settings/index.md @@ -43,6 +43,7 @@ Access the default page for admin area settings by navigating to | Option | Description | | ------ | ----------- | +| [Repository's custom initial branch name](../../project/repository/branches/index.md#custom-initial-branch-name-core-only) | Set a custom branch name rather than master for all the new repositories created within your instance. | | [Repository mirror](visibility_and_access_controls.md#allow-mirrors-to-be-set-up-for-projects) | Configure repository mirroring. | | [Repository storage](../../../administration/repository_storage_types.md) | Configure storage path settings. | | Repository maintenance | ([Repository checks](../../../administration/repository_checks.md) and [Housekeeping](../../../administration/housekeeping.md)). Configure automatic Git checks and housekeeping on repositories. | diff --git a/doc/user/admin_area/settings/rate_limit_on_issues_creation.md b/doc/user/admin_area/settings/rate_limit_on_issues_creation.md index bae60aba15f..3f8ef7c9d01 100644 --- a/doc/user/admin_area/settings/rate_limit_on_issues_creation.md +++ b/doc/user/admin_area/settings/rate_limit_on_issues_creation.md @@ -10,10 +10,9 @@ info: To determine the technical writer assigned to the Stage/Group associated w > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/28129) in GitLab 12.10. This setting allows you to rate limit the requests to the issue creation endpoint. -It defaults to 300 requests per minute. -You can change it in **Admin Area > Settings > Network > Performance Optimization**. +You can change its value in **Admin Area > Settings > Network > Issues Rate Limits**. -For example, requests using the +For example, if you set a limit of 300, requests using the [Projects::IssuesController#create](https://gitlab.com/gitlab-org/gitlab/raw/master/app/controllers/projects/issues_controller.rb) action exceeding a rate of 300 per minute are blocked. Access to the endpoint is allowed after one minute. @@ -23,6 +22,6 @@ This limit is: - Applied independently per project and per user. - Not applied per IP address. -- Active by default. To disable it, set the option to `0`. +- Disabled by default. To enable it, set the option to any value other than `0`. Requests over the rate limit are logged into the `auth.log` file. diff --git a/doc/user/admin_area/settings/sign_in_restrictions.md b/doc/user/admin_area/settings/sign_in_restrictions.md index 1da93c7005f..311b79af7e3 100644 --- a/doc/user/admin_area/settings/sign_in_restrictions.md +++ b/doc/user/admin_area/settings/sign_in_restrictions.md @@ -4,9 +4,14 @@ type: reference # Sign-in restrictions **(CORE ONLY)** -You can use sign-in restrictions to limit the authentication with password -for web interface and Git over HTTP(S), two-factor authentication enforcing, as well as -as configuring the home page URL and after sign-out path. +You can use **Sign-in restrictions** to customize authentication restrictions for web interfaces as well as Git over HTTP(S). + +## Settings + +To access sign-in restriction settings: + +1. Navigate to the **Admin Area > Settings > General**. +1. Expand the **Sign-in restrictions** section. ## Password authentication enabled @@ -25,6 +30,15 @@ period in hours.  +## Email notification for unknown sign-ins + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/218457) in GitLab 13.2. + +When enabled, GitLab notifies users of sign-ins from unknown IP addresses or devices. For more information, +see [Email notification for unknown sign-ins](../../profile/unknown_sign_in_notification.md). + + + ## Sign-in information All users that are not logged-in will be redirected to the page represented by the configured @@ -36,13 +50,6 @@ after sign out if value is not empty. If a "Sign in text" in Markdown format is provided, then every user will be presented with this message after logging-in. -## Settings - -To access this feature: - -1. Navigate to the **Admin Area > Settings > General**. -1. Expand the **Sign-in restrictions** section. - <!-- ## Troubleshooting Include any troubleshooting steps that you can foresee. If you know beforehand what issues diff --git a/doc/user/admin_area/settings/visibility_and_access_controls.md b/doc/user/admin_area/settings/visibility_and_access_controls.md index d9ca4a0881a..92eeb6a04b7 100644 --- a/doc/user/admin_area/settings/visibility_and_access_controls.md +++ b/doc/user/admin_area/settings/visibility_and_access_controls.md @@ -68,8 +68,16 @@ To ensure only admin users can delete projects: > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/32935) in GitLab 12.6. -By default, a project or group marked for removal will be permanently removed after 7 days. -This period may be changed, and setting this period to 0 will enable immediate removal +By default, a project marked for deletion will be permanently removed with immediate effect. +By default, a group marked for deletion will be permanently removed after 7 days. + +CAUTION: **Warning:** +The default behavior of [Delayed Project deletion](https://gitlab.com/gitlab-org/gitlab/-/issues/32935) in GitLab 12.6 was changed to +[Immediate deletion](https://gitlab.com/gitlab-org/gitlab/-/issues/220382) in GitLab 13.2. + +Projects within a group can be deleted after a delayed period, by [configuring in Group Settings](../../group/index.md#enabling-delayed-project-removal-premium). + +The default period is 7 days, and can be changed. Setting this period to 0 will enable immediate removal of projects or groups. To change this period: diff --git a/doc/user/analytics/value_stream_analytics.md b/doc/user/analytics/value_stream_analytics.md index 0efe28ac5f7..b11bae98af3 100644 --- a/doc/user/analytics/value_stream_analytics.md +++ b/doc/user/analytics/value_stream_analytics.md @@ -332,12 +332,6 @@ The current permissions on the Project Value Stream Analytics dashboard are: You can [read more about permissions](../../ci/yaml/README.md) in general. -NOTE: **Note:** -As of GitLab 12.3, the project-level page is deprecated. You should access -project-level Value Stream Analytics from **Analytics > Value Stream Analytics** in the top -navigation bar. We will ensure that the same project-level functionality is available -to CE users in the new analytics space. - For Value Stream Analytics functionality introduced in GitLab 12.3 and later: - Users must have Reporter access or above. diff --git a/doc/user/application_security/configuration/index.md b/doc/user/application_security/configuration/index.md index f0fcd0c4419..229a8572206 100644 --- a/doc/user/application_security/configuration/index.md +++ b/doc/user/application_security/configuration/index.md @@ -14,17 +14,23 @@ info: To determine the technical writer assigned to the Stage/Group associated w The security configuration page displays the configuration state of each of the security features and can be accessed through a project's sidebar nav. - + The page uses the project's latest default branch [CI pipeline](../../../ci/pipelines/index.md) to determine the configuration state of each feature. If a job with the expected security report artifact exists in the pipeline, the feature is considered configured. -NOTE: **Note:** if the latest pipeline used [Auto DevOps](../../../topics/autodevops/index.md), +NOTE: **Note:** +If the latest pipeline used [Auto DevOps](../../../topics/autodevops/index.md), all security features will be configured by default. ## Limitations -It is not possible to enable or disable a feature using the configuration page. -However, instructions on how to enable or disable a feature can be found through -the links next to each feature on that page. +It is not yet possible to enable or disable most features using the +configuration page. However, instructions on how to enable or disable a feature +can be found through the links next to each feature on that page. + +If a project does not have an existing CI configuration, then the SAST feature +can be enabled by clicking on the "Enable with Merge Request" button under the +"Manage" column. Future work will expand this to editing _existing_ CI +configurations, and to other security features. diff --git a/doc/user/application_security/container_scanning/img/container_scanning_v13_0.png b/doc/user/application_security/container_scanning/img/container_scanning_v13_0.png Binary files differdeleted file mode 100644 index 7a079a65072..00000000000 --- a/doc/user/application_security/container_scanning/img/container_scanning_v13_0.png +++ /dev/null diff --git a/doc/user/application_security/container_scanning/img/container_scanning_v13_2.png b/doc/user/application_security/container_scanning/img/container_scanning_v13_2.png Binary files differnew file mode 100644 index 00000000000..254ea1dcf5d --- /dev/null +++ b/doc/user/application_security/container_scanning/img/container_scanning_v13_2.png diff --git a/doc/user/application_security/container_scanning/index.md b/doc/user/application_security/container_scanning/index.md index 0ffe83cdfc9..7bc8b62825c 100644 --- a/doc/user/application_security/container_scanning/index.md +++ b/doc/user/application_security/container_scanning/index.md @@ -32,7 +32,7 @@ You can enable container scanning by doing one of the following: GitLab compares the found vulnerabilities between the source and target branches, and shows the information directly in the merge request. - + <!-- NOTE: The container scanning tool references the following heading in the code, so if you make a change to this heading, make sure to update the documentation URLs used in the @@ -58,10 +58,10 @@ To enable Container Scanning in your pipeline, you need the following: ```yaml build: - image: docker:19.03.11 + image: docker:19.03.12 stage: build services: - - docker:19.03.11-dind + - docker:19.03.12-dind variables: IMAGE_TAG: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG:$CI_COMMIT_SHA script: @@ -114,7 +114,7 @@ build: image: docker:stable stage: build services: - - docker:19.03.11-dind + - docker:19.03.12-dind variables: IMAGE: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG:$CI_COMMIT_SHA script: @@ -141,7 +141,7 @@ enables verbose output from Clair by setting the `CLAIR_OUTPUT` environment vari ```yaml include: - template: Container-Scanning.gitlab-ci.yml + - template: Container-Scanning.gitlab-ci.yml variables: CLAIR_OUTPUT: High @@ -174,6 +174,7 @@ using environment variables. | `CLAIR_DB_IMAGE_TAG` | (**DEPRECATED - use `CLAIR_DB_IMAGE` instead**) The Docker image tag for the [PostgreSQL server hosting the vulnerabilities definitions](https://hub.docker.com/r/arminc/clair-db). It can be useful to override this value with a specific version, for example, to provide a consistent set of vulnerabilities for integration testing purposes. | `latest` | | `DOCKERFILE_PATH` | The path to the `Dockerfile` to be used for generating remediations. By default, the scanner will look for a file named `Dockerfile` in the root directory of the project, so this variable should only be configured if your `Dockerfile` is in a non-standard location, such as a subdirectory. See [Solutions for vulnerabilities](#solutions-for-vulnerabilities-auto-remediation) for more details. | `Dockerfile` | | `ADDITIONAL_CA_CERT_BUNDLE` | Bundle of CA certs that you want to trust. | "" | +| `SECURE_LOG_LEVEL` | The log levels available are: `fatal`, `error`, `warn`, `info`, `debug` | `info` | ### Overriding the Container Scanning template @@ -183,7 +184,7 @@ specify any additional keys. For example: ```yaml include: - template: Container-Scanning.gitlab-ci.yml + - template: Container-Scanning.gitlab-ci.yml container_scanning: variables: @@ -195,15 +196,15 @@ GitLab 13.0 and later doesn't support [`only` and `except`](../../../ci/yaml/REA When overriding the template, you must use [`rules`](../../../ci/yaml/README.md#rules) instead. -### Vulnerability whitelisting +### Vulnerability allowlisting -To whitelist specific vulnerabilities, follow these steps: +To allowlist specific vulnerabilities, follow these steps: 1. Set `GIT_STRATEGY: fetch` in your `.gitlab-ci.yml` file by following the instructions in [overriding the Container Scanning template](#overriding-the-container-scanning-template). -1. Define the whitelisted vulnerabilities in a YAML file named `clair-whitelist.yml`. This must use - the format described in the [whitelist example file](https://github.com/arminc/clair-scanner/blob/v12/example-whitelist.yaml). -1. Add the `clair-whitelist.yml` file to your project's Git repository. +1. Define the allowlisted vulnerabilities in a YAML file named `vulnerability-allowlist.yml`. This must use + the format described in the [allowlist example file](https://gitlab.com/gitlab-org/security-products/analyzers/klar/-/raw/master/testdata/vulnerability-allowlist.yml). +1. Add the `vulnerability-allowlist.yml` file to your project's Git repository. ### Running Container Scanning in an offline environment @@ -282,7 +283,7 @@ stages: build_latest_vulnerabilities: stage: build services: - - docker:19.03.11-dind + - docker:19.03.12-dind script: - docker pull arminc/clair-db:latest - docker tag arminc/clair-db:latest $CI_REGISTRY/namespace/clair-vulnerabilities-db diff --git a/doc/user/application_security/coverage_fuzzing/index.md b/doc/user/application_security/coverage_fuzzing/index.md new file mode 100644 index 00000000000..85da7d85506 --- /dev/null +++ b/doc/user/application_security/coverage_fuzzing/index.md @@ -0,0 +1,117 @@ +--- +stage: Secure +group: Fuzz Testing +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: reference, howto +--- + +# Coverage Guided Fuzz Testing **(ULTIMATE)** + +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3226) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.2 as an [Alpha feature](https://about.gitlab.com/handbook/product/gitlab-the-product/#alpha). + +GitLab allows you to add coverage-guided fuzz testing to your pipelines. This helps you discover +bugs and potential security issues that other QA processes may miss. Coverage-guided fuzzing sends +random inputs to an instrumented version of your application in an effort to cause unexpected +behavior, such as a crash. Such behavior indicates a bug that you should address. + +We recommend that you use fuzz testing in addition to the other security scanners in [GitLab Secure](../index.md) +and your own test processes. If you're using [GitLab CI/CD](../../../ci/README.md), +you can run your coverage guided fuzz tests as part your CI/CD workflow. You can take advantage of +Coverage Guided Fuzzing by including the CI job in your existing `.gitlab-ci.yml` file. + +## Supported fuzzing engines and languages + +GitLab supports these languages through the fuzzing engine listed for each. We currently provide a Docker image for apps written in Go, but you can test the other languages below by providing a Docker image with the fuzz engine to run your app. + +| Language | Fuzzing Engine | Example | +|----------|---------------------------------------------------------------------------|---------| +| C/C++ | [libFuzzer](https://llvm.org/docs/LibFuzzer.html) | | +| GoLang | [go-fuzz (libFuzzer support)](https://github.com/dvyukov/go-fuzz) | | +| Rust | [cargo-fuzz (libFuzzer support)](https://github.com/rust-fuzz/cargo-fuzz) | | + +## Configuration + +To enable fuzzing, you must +[include](../../../ci/yaml/README.md#includetemplate) +the [`Coverage-Fuzzing.gitlab-ci.yml` template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml) +provided as part of your GitLab installation. + +To do so, add the following to your `.gitlab-ci.yml` file: + +```yaml +include: + - template: Coverage-Fuzzing.gitlab-ci.yml +``` + +The included template makes available the [hidden job](../../../ci/yaml/README.md#hide-jobs) +`.fuzz_base`, which you must [extend](../../../ci/yaml/README.md#extends) for each of your fuzz +targets. Each fuzz target **must** have a separate job. For example, the +[go-fuzzing-example project](https://gitlab.com/gitlab-org/security-products/demos/go-fuzzing-example) +contains one job that extends `.fuzz_base` for its single fuzz target. + +The `my_fuzz_target` job (the separate job for your fuzz target) does the following: + +- Extends `.fuzz_base`. +- Compiles the fuzz target with [go-fuzz](https://github.com/dvyukov/go-fuzz). +- Runs the target with the `gitlab-cov-fuzz` command, which is available to each job that extends + `.fuzz_base`. +- Runs on a fuzz stage that usually comes after a test stage. + +The `gitlab-cov-fuzz` is a command-line tool that runs the instrumented application. It parses and +analyzes the exception information that the fuzzer outputs. It also downloads the [corpus](#glossary) +and crash events from previous pipelines automatically. This helps your fuzz targets build on the progress of +previous fuzzing jobs. The parsed crash events and data are written to +`gl-coverage-fuzzing-report.json`. + +### Artifacts + +Each fuzzing step outputs these artifacts: + +- `gl-coverage-fuzzing-report.json`: This file's format may change in future releases. +- `artifacts.zip`: This file contains two directories: + - `corpus`: Holds all test cases generated by the current and all previous jobs. + - `crashes`: Holds all crash events the current job encountered as well as those not fixed in + previous jobs. + +### Types of Fuzzing Jobs + +There are two types of jobs: + +- Fuzzing: Standard fuzzing session. You can configure a long session through a user defined + timeout. +- Regression: Run the fuzz targets through the accumulated test cases generated by previous fuzzing + sessions plus fixed crashes from previous sessions. This is usually very quick. + +Here's our current suggestion for configuring your fuzz target's timeout: + +- Set `COVERAGE_FUZZING_BRANCH` to the branch where you want to run long-running (async) fuzzing + jobs. This is `master` by default. +- Use regression or short-running fuzzing jobs for other branches or merge requests. + +This suggestion helps find new bugs on the development branch and catch old bugs in merge requests +(like unit tests). + +You can configure this by passing `--regression=false/true` to `gitlab-cov-fuzz` as the [Go example](https://gitlab.com/gitlab-org/security-products/demos/go-fuzzing-example/-/blob/master/.gitlab-ci.yml) +shows. Also note that `gitlab-cov-fuzz` is a wrapper, so you can pass those arguments to configure +any option available in the underlying fuzzing engine. + +### Available variables + +| Environment variable | Description | +|---------------------------|--------------------------------------------------------------------| +| `COVERAGE_FUZZING_BRANCH` | The branch for long-running fuzzing jobs. The default is `master`. | + +### Additional Configuration + +The `gitlab-cov-fuzz` command passes all arguments it receives to the underlying fuzzing engine. You +can therefore use all the options available in that fuzzing engine. For more information on these +options, see the underlying fuzzing engine's documentation. + +### Glossary + +- Seed corpus: The set of test cases given as initial input to the fuzz target. This usually speeds + up the fuzz target substantially. This can be either manually created test cases or auto-generated + with the fuzz target itself from previous runs. +- Corpus: The set of meaningful test cases that are generated while the fuzzer is running. Each + meaningful test case produces new coverage in the tested program. It's advised to re-use the + corpus and pass it to subsequent runs. diff --git a/doc/user/application_security/dast/img/dast_all_v13_0.png b/doc/user/application_security/dast/img/dast_all_v13_0.png Binary files differdeleted file mode 100644 index 7b67fc44fae..00000000000 --- a/doc/user/application_security/dast/img/dast_all_v13_0.png +++ /dev/null diff --git a/doc/user/application_security/dast/img/dast_on_demand_v13_2.png b/doc/user/application_security/dast/img/dast_on_demand_v13_2.png Binary files differnew file mode 100644 index 00000000000..8a733c27be1 --- /dev/null +++ b/doc/user/application_security/dast/img/dast_on_demand_v13_2.png diff --git a/doc/user/application_security/dast/img/dast_v13_2.png b/doc/user/application_security/dast/img/dast_v13_2.png Binary files differnew file mode 100644 index 00000000000..bbf7944eb40 --- /dev/null +++ b/doc/user/application_security/dast/img/dast_v13_2.png diff --git a/doc/user/application_security/dast/index.md b/doc/user/application_security/dast/index.md index 256daae46d7..d68928d858b 100644 --- a/doc/user/application_security/dast/index.md +++ b/doc/user/application_security/dast/index.md @@ -9,9 +9,9 @@ type: reference, howto > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/4348) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 10.4. -NOTE: **4 of the top 6 attacks were application based.** -Download our whitepaper, -["A Seismic Shift in Application Security"](https://about.gitlab.com/resources/whitepaper-seismic-shift-application-security/) +NOTE: **Note:** +The whitepaper ["A Seismic Shift in Application Security"](https://about.gitlab.com/resources/whitepaper-seismic-shift-application-security/) +explains how **4 of the top 6 attacks were application based**. Download it to learn how to protect your organization. Running [static checks](../sast/index.md) on your code is the first step to detect @@ -36,7 +36,7 @@ NOTE: **Note:** This comparison logic uses only the latest pipeline executed for the target branch's base commit. Running the pipeline on any other commit has no effect on the merge request. - + By clicking on one of the detected linked vulnerabilities, you can see the details and the URL(s) affected. @@ -44,10 +44,10 @@ see the details and the URL(s) affected.  [Dynamic Application Security Testing (DAST)](https://en.wikipedia.org/wiki/Dynamic_Application_Security_Testing) -uses the popular open source tool [OWASP ZAProxy](https://github.com/zaproxy/zaproxy) +uses the popular open source tool [OWASP Zed Attack Proxy](https://www.zaproxy.org/) to perform an analysis on your running web application. -By default, DAST executes [ZAP Baseline Scan](https://github.com/zaproxy/zaproxy/wiki/ZAP-Baseline-Scan) +By default, DAST executes [ZAP Baseline Scan](https://www.zaproxy.org/docs/docker/baseline-scan/) and performs passive scanning only. It won't actively attack your application. However, DAST can be [configured](#full-scan) to also perform an *active scan*: attack your application and produce a more extensive security report. @@ -143,6 +143,22 @@ The only changes to the site should be from the DAST scanner. Be aware that any changes that users, scheduled tasks, database changes, code changes, other pipelines, or other scanners make to the site during a scan could lead to inaccurate results. +### Hide sensitive information + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/36332) in GitLab 13.1. + +HTTP request and response headers may contain sensitive information, including cookies and +authorization credentials. By default, the following headers are masked: + +- `Authorization`. +- `Proxy-Authorization`. +- `Set-Cookie` (values only). +- `Cookie` (values only). + +Using the [`DAST_MASK_HTTP_HEADERS` variable](#available-variables), you can list the +headers whose values you want masked. For details on how to mask headers, see +[Customizing the DAST settings](#customizing-the-dast-settings). + ### Authentication It's also possible to authenticate the user before performing the DAST checks. @@ -398,6 +414,10 @@ variables: ### Customizing the DAST settings +CAUTION: **Deprecation:** +Beginning in GitLab 13.0, the use of [`only` and `except`](../../../ci/yaml/README.md#onlyexcept-basic) +is no longer supported. When overriding the template, you must use [`rules`](../../../ci/yaml/README.md#rules) instead. + The DAST settings can be changed through environment variables by using the [`variables`](../../../ci/yaml/README.md#variables) parameter in `.gitlab-ci.yml`. These variables are documented in [available variables](#available-variables). @@ -410,68 +430,43 @@ include: variables: DAST_WEBSITE: https://example.com - DAST_TARGET_AVAILABILITY_TIMEOUT: 120 + DAST_SPIDER_MINS: 120 ``` Because the template is [evaluated before](../../../ci/yaml/README.md#include) the pipeline configuration, the last mention of the variable will take precedence. -### Overriding the DAST template - -CAUTION: **Deprecation:** -Beginning in GitLab 13.0, the use of [`only` and `except`](../../../ci/yaml/README.md#onlyexcept-basic) -is no longer supported. When overriding the template, you must use [`rules`](../../../ci/yaml/README.md#rules) instead. - -If you want to override the job definition (for example, change properties like -`variables` or `dependencies`), you need to declare a `dast` job after the -template inclusion and specify any additional keys under it. For example: - -```yaml -include: - - template: DAST.gitlab-ci.yml - -dast: - stage: dast # IMPORTANT: don't forget to add this - variables: - DAST_WEBSITE: https://example.com - CI_DEBUG_TRACE: "true" -``` - -As the DAST job belongs to a separate `dast` stage that runs after all -[default stages](../../../ci/yaml/README.md#stages), -don't forget to add `stage: dast` when you override the template job definition. - ### Available variables DAST can be [configured](#customizing-the-dast-settings) using environment variables. -| Environment variable | Required | Description | +| Environment variable | Type | Description | |-----------------------------| -----------|--------------------------------------------------------------------------------| -| `SECURE_ANALYZERS_PREFIX` | no | Set the Docker registry base address from which to download the analyzer. | -| `DAST_WEBSITE` | no| The URL of the website to scan. `DAST_API_SPECIFICATION` must be specified if this is omitted. | -| `DAST_API_SPECIFICATION` | no | The API specification to import. `DAST_WEBSITE` must be specified if this is omitted. | -| `DAST_AUTH_URL` | no | The authentication URL of the website to scan. Not supported for API scans. | -| `DAST_USERNAME` | no | The username to authenticate to in the website. | -| `DAST_PASSWORD` | no | The password to authenticate to in the website. | -| `DAST_USERNAME_FIELD` | no | The name of username field at the sign-in HTML form. | -| `DAST_PASSWORD_FIELD` | no | The name of password field at the sign-in HTML form. | -| `DAST_AUTH_EXCLUDE_URLS` | no | The URLs to skip during the authenticated scan; comma-separated, no spaces in between. Not supported for API scans. | -| `DAST_TARGET_AVAILABILITY_TIMEOUT` | no | Time limit in seconds to wait for target availability. Scan is attempted nevertheless if it runs out. Integer. Defaults to `60`. | -| `DAST_FULL_SCAN_ENABLED` | no | Switches the tool to execute [ZAP Full Scan](https://github.com/zaproxy/zaproxy/wiki/ZAP-Full-Scan) instead of [ZAP Baseline Scan](https://github.com/zaproxy/zaproxy/wiki/ZAP-Baseline-Scan). Boolean. `true`, `True`, or `1` are considered as true value, otherwise false. Defaults to `false`. | -| `DAST_FULL_SCAN_DOMAIN_VALIDATION_REQUIRED` | no | Requires [domain validation](#domain-validation) when running DAST full scans. Boolean. `true`, `True`, or `1` are considered as true value, otherwise false. Defaults to `false`. Not supported for API scans. | -| `DAST_AUTO_UPDATE_ADDONS` | no | By default the versions of ZAP add-ons are pinned to those provided with the DAST image. Set to `true` to allow ZAP to download the latest versions. | -| `DAST_API_HOST_OVERRIDE` | no | Used to override domains defined in API specification files. | -| `DAST_EXCLUDE_RULES` | no | Set to a comma-separated list of Vulnerability Rule IDs to exclude them from the scan report. Currently, excluded rules will get executed but the alerts from them will be suppressed. Rule IDs are numbers and can be found from the DAST log or on the [ZAP project](https://github.com/zaproxy/zaproxy/blob/develop/docs/scanners.md). For example, `HTTP Parameter Override` has a rule ID of `10026`. | -| `DAST_REQUEST_HEADERS` | no | Set to a comma-separated list of request header names and values. For example, `Cache-control: no-cache,User-Agent: DAST/1.0` | -| `DAST_DEBUG` | no | Enable debug message output. Boolean. `true`, `True`, or `1` are considered as true value, otherwise false. Defaults to `false`. | -| `DAST_SPIDER_MINS` | no | The maximum duration of the spider scan in minutes. Set to zero for unlimited. Defaults to one minute, or unlimited when the scan is a full scan. | -| `DAST_HTML_REPORT` | no | The file name of the HTML report written at the end of a scan. | -| `DAST_MARKDOWN_REPORT` | no | The file name of the Markdown report written at the end of a scan. | -| `DAST_XML_REPORT` | no | The file name of the XML report written at the end of a scan. | -| `DAST_INCLUDE_ALPHA_VULNERABILITIES` | no | Include alpha passive and active scan rules. Boolean. `true`, `True`, or `1` are considered as true value, otherwise false. Defaults to `false`. | -| `DAST_USE_AJAX_SPIDER` | no | Use the AJAX spider in addition to the traditional spider, useful for crawling sites that require JavaScript. Boolean. `true`, `True`, or `1` are considered as true value, otherwise false. Defaults to `false`. | -| `DAST_ZAP_CLI_OPTIONS` | no | ZAP server command-line options. For example, `-Xmx3072m` would set the Java maximum memory allocation pool size. | -| `DAST_ZAP_LOG_CONFIGURATION` | no | Set to a semicolon-separated list of additional log4j properties for the ZAP Server. For example, `log4j.logger.org.parosproxy.paros.network.HttpSender=DEBUG` | +| `SECURE_ANALYZERS_PREFIX` | URL | Set the Docker registry base address from which to download the analyzer. | +| `DAST_WEBSITE` | URL | The URL of the website to scan. `DAST_API_SPECIFICATION` must be specified if this is omitted. | +| `DAST_API_SPECIFICATION` | URL or string | The API specification to import. The specification can be hosted at a URL, or the name of a file present in the `/zap/wrk` directory. `DAST_WEBSITE` must be specified if this is omitted. | +| `DAST_AUTH_URL` | URL | The URL of the page containing the sign-in HTML form on the target website. `DAST_USERNAME` and `DAST_PASSWORD` will be submitted with the login form to create an authenticated scan. Not supported for API scans. | +| `DAST_USERNAME` | string | The username to authenticate to in the website. | +| `DAST_PASSWORD` | string | The password to authenticate to in the website. | +| `DAST_USERNAME_FIELD` | string | The name of username field at the sign-in HTML form. | +| `DAST_PASSWORD_FIELD` | string | The name of password field at the sign-in HTML form. | +| `DAST_MASK_HTTP_HEADERS` | string | Comma-separated list of request and response headers to be masked (introduced in GitLab 13.1). Must contain **all** headers to be masked. Refer to [list of headers that are masked by default](#hide-sensitive-information). | +| `DAST_AUTH_EXCLUDE_URLS` | URLs | The URLs to skip during the authenticated scan; comma-separated, no spaces in between. Not supported for API scans. | +| `DAST_FULL_SCAN_ENABLED` | boolean | Set to `true` to run a [ZAP Full Scan](https://github.com/zaproxy/zaproxy/wiki/ZAP-Full-Scan) instead of a [ZAP Baseline Scan](https://github.com/zaproxy/zaproxy/wiki/ZAP-Baseline-Scan). Default: `false` | +| `DAST_FULL_SCAN_DOMAIN_VALIDATION_REQUIRED` | boolean | Set to `true` to require [domain validation](#domain-validation) when running DAST full scans. Not supported for API scans. Default: `false` | +| `DAST_AUTO_UPDATE_ADDONS` | boolean | ZAP add-ons are pinned to specific versions in the DAST Docker image. Set to `true` to download the latest versions when the scan starts. Default: `false` | +| `DAST_API_HOST_OVERRIDE` | string | Used to override domains defined in API specification files. Example: `example.com:8080` | +| `DAST_EXCLUDE_RULES` | string | Set to a comma-separated list of Vulnerability Rule IDs to exclude them from running during the scan. Rule IDs are numbers and can be found from the DAST log or on the [ZAP project](https://github.com/zaproxy/zaproxy/blob/develop/docs/scanners.md). For example, `HTTP Parameter Override` has a rule ID of `10026`. **Note:** In earlier versions of GitLab the excluded rules were executed but alerts they generated were supressed. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/118641) in GitLab 12.10. | +| `DAST_REQUEST_HEADERS` | string | Set to a comma-separated list of request header names and values. Headers will be added to every request made by DAST. For example, `Cache-control: no-cache,User-Agent: DAST/1.0` | +| `DAST_DEBUG` | boolean | Enable debug message output. Default: `false` | +| `DAST_SPIDER_MINS` | number | The maximum duration of the spider scan in minutes. Set to `0` for unlimited. Default: One minute, or unlimited when the scan is a full scan. | +| `DAST_HTML_REPORT` | string | The file name of the HTML report written at the end of a scan. | +| `DAST_MARKDOWN_REPORT` | string | The file name of the Markdown report written at the end of a scan. | +| `DAST_XML_REPORT` | string | The file name of the XML report written at the end of a scan. | +| `DAST_INCLUDE_ALPHA_VULNERABILITIES` | boolean | Set to `true` to include alpha passive and active scan rules. Default: `false` | +| `DAST_USE_AJAX_SPIDER` | boolean | Set to `true` to use the AJAX spider in addition to the traditional spider, useful for crawling sites that require JavaScript. Default: `false` | +| `DAST_ZAP_CLI_OPTIONS` | string | ZAP server command-line options. For example, `-Xmx3072m` would set the Java maximum memory allocation pool size. | +| `DAST_ZAP_LOG_CONFIGURATION` | string | Set to a semicolon-separated list of additional log4j properties for the ZAP Server. For example, `log4j.logger.org.parosproxy.paros.network.HttpSender=DEBUG;log4j.logger.com.crawljax=DEBUG` | ### DAST command-line options @@ -532,19 +527,20 @@ A DAST job has two executing processes: Debug mode of the scripts can be enabled by using the `DAST_DEBUG` environment variable. This can help when troubleshooting the job, and will output statements indicating what percentage of the scan is complete. -For details on using variables, see [Overriding the DAST template](#overriding-the-dast-template). +For details on using variables, see [Overriding the DAST template](#customizing-the-dast-settings). Debug mode of the ZAP server can be enabled using the `DAST_ZAP_LOG_CONFIGURATION` environment variable. The following table outlines examples of values that can be set and the effect that they have on the output that is logged. Multiple values can be specified, separated by semicolons. -| Log configuration value | Effect | -|-------------------------------------------------- | ----------------------------------------------------------------- | -| `log4j.rootLogger=DEBUG` | Enable all debug logging statements. | -| `log4j.logger.org.apache.commons.httpclient=DEBUG` | Log every HTTP request and response made by the ZAP server. | -| `log4j.logger.com.crawljax=DEBUG` | Enable Ajax Crawler debug logging statements. | -| `log4j.logger.org.parosproxy.paros=DEBUG` | Enable ZAP server proxy debug logging statements. | -| `log4j.logger.org.zaproxy.zap=DEBUG` | Enable debug logging statements of the general ZAP server code. | +| Log configuration value | Effect | +|-------------------------------------------------- | ----------------------------------------------------------------- | +| `log4j.rootLogger=DEBUG` | Enable all debug logging statements. | +| `log4j.logger.org.apache.commons.httpclient=DEBUG` | Log every HTTP request and response made by the ZAP server. | +| `log4j.logger.org.zaproxy.zap.spider.SpiderController=DEBUG` | Log URLs found during the spider scan of the target. | +| `log4j.logger.com.crawljax=DEBUG` | Enable Ajax Crawler debug logging statements. | +| `log4j.logger.org.parosproxy.paros=DEBUG` | Enable ZAP server proxy debug logging statements. | +| `log4j.logger.org.zaproxy.zap=DEBUG` | Enable debug logging statements of the general ZAP server code. | ## Running DAST in an offline environment @@ -604,6 +600,44 @@ security reports without requiring internet access. Alternatively, you can use the variable `SECURE_ANALYZERS_PREFIX` to override the base registry address of the `dast` image. +## On-Demand Scans + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/218465) in GitLab 13.2. +> - It's deployed behind a feature flag, disabled by default. +> - It's disabled on GitLab.com. +> - It's able to be enabled or disabled per-project. +> - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enable-or-disable-on-demand-scans). + +Passive DAST scans may be run on demand against a target website, outside the DevOps lifecycle. These scans will +always be associated with the default or `master` branch of your project and the results can be seen in the project dashboard. + + + +### Enable or disable On-Demand Scans + +On-Demand Scans is under development and not ready for production use. It is +deployed behind a feature flag that is **disabled by default**. +[GitLab administrators with access to the GitLab Rails console](../../../administration/feature_flags.md) +can enable it for your instance. On-Demand Scans can be enabled or disabled per-project + +To enable it: + +```ruby +# Instance-wide +Feature.enable(:security_on_demand_scans_feature_flag) +# or by project +Feature.enable(:security_on_demand_scans_feature_flag, Project.find(<project id>)) +``` + +To disable it: + +```ruby +# Instance-wide +Feature.disable(:security_on_demand_scans_feature_flag) +# or by project +Feature.disable(:security_on_demand_scans_feature_flag, Project.find(<project id>)) +``` + ## Reports The DAST tool outputs a report file in JSON format by default. However, this tool can also generate reports in @@ -683,18 +717,6 @@ Once a vulnerability is found, you can interact with it. Read more on how to For more information about the vulnerabilities database update, check the [maintenance table](../index.md#maintenance-and-update-of-the-vulnerabilities-database). -<!-- ## Troubleshooting - -Include any troubleshooting steps that you can foresee. If you know beforehand what issues -one might have when setting this up, or when something is changed, or on upgrading, it's -important to describe those, too. Think of things that may go wrong and include them here. -This is important to minimize requests for support, and to avoid doc comments with -questions that you know someone might ask. - -Each scenario can be a third-level heading, e.g. `### Getting error message X`. -If you have none to add when creating a doc, leave this section in place -but commented out to help encourage others to add to it in the future. --> - ## Optimizing DAST By default, DAST will download all artifacts defined by previous jobs in the pipeline. If @@ -734,3 +756,15 @@ variables: Here, DAST is being allocated 3072 MB. Change the number after `-Xmx` to the required memory amount. + +<!-- ## Troubleshooting + +Include any troubleshooting steps that you can foresee. If you know beforehand what issues +one might have when setting this up, or when something is changed, or on upgrading, it's +important to describe those, too. Think of things that may go wrong and include them here. +This is important to minimize requests for support, and to avoid doc comments with +questions that you know someone might ask. + +Each scenario can be a third-level heading, e.g. `### Getting error message X`. +If you have none to add when creating a doc, leave this section in place +but commented out to help encourage others to add to it in the future. --> diff --git a/doc/user/application_security/dependency_scanning/analyzers.md b/doc/user/application_security/dependency_scanning/analyzers.md index 474f9339d0b..ca2b212ffc3 100644 --- a/doc/user/application_security/dependency_scanning/analyzers.md +++ b/doc/user/application_security/dependency_scanning/analyzers.md @@ -1,3 +1,10 @@ +--- +type: reference, howto +stage: Secure +group: Composition Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Dependency Scanning Analyzers **(ULTIMATE)** Dependency Scanning relies on underlying third party tools that are wrapped into diff --git a/doc/user/application_security/dependency_scanning/img/dependency_scanning_v13_0.png b/doc/user/application_security/dependency_scanning/img/dependency_scanning_v13_0.png Binary files differdeleted file mode 100644 index 9f3990df957..00000000000 --- a/doc/user/application_security/dependency_scanning/img/dependency_scanning_v13_0.png +++ /dev/null diff --git a/doc/user/application_security/dependency_scanning/img/dependency_scanning_v13_2.png b/doc/user/application_security/dependency_scanning/img/dependency_scanning_v13_2.png Binary files differnew file mode 100644 index 00000000000..28c4eb85b7c --- /dev/null +++ b/doc/user/application_security/dependency_scanning/img/dependency_scanning_v13_2.png diff --git a/doc/user/application_security/dependency_scanning/index.md b/doc/user/application_security/dependency_scanning/index.md index 84ec0ec976d..57b4fae3230 100644 --- a/doc/user/application_security/dependency_scanning/index.md +++ b/doc/user/application_security/dependency_scanning/index.md @@ -27,7 +27,7 @@ GitLab checks the Dependency Scanning report, compares the found vulnerabilities between the source and target branches, and shows the information on the merge request. - + The results are sorted by the severity of the vulnerability: @@ -61,7 +61,7 @@ The following languages and dependency managers are supported: | Language (package managers) | Supported files | Scan tool(s) | |----------------------------- | --------------- | ------------ | | Java ([Gradle](https://gradle.org/), [Maven](https://maven.apache.org/)) | `build.gradle`, `build.gradle.kts`, `pom.xml` | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | -| JavaScript ([npm](https://www.npmjs.com/), [yarn](https://yarnpkg.com/en/)) | `package-lock.json`, `npm-shrinkwrap.json`, `yarn.lock` | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium), [Retire.js](https://retirejs.github.io/retire.js) | +| JavaScript ([npm](https://www.npmjs.com/), [yarn](https://classic.yarnpkg.com/en/)) | `package-lock.json`, `npm-shrinkwrap.json`, `yarn.lock` | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium), [Retire.js](https://retirejs.github.io/retire.js/) | | Go ([Golang](https://golang.org/)) | `go.sum` | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | | PHP ([Composer](https://getcomposer.org/)) | `composer.lock` | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | | Python ([setuptools](https://setuptools.readthedocs.io/en/latest/), [pip](https://pip.pypa.io/en/stable/), [Pipenv](https://pipenv.pypa.io/en/latest/)) | `setup.py`, `requirements.txt`, `requirements.pip`, `requires.txt`, `Pipfile` | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | @@ -72,7 +72,7 @@ Plans are underway for supporting the following languages, dependency managers, | Language (package managers) | Supported files | Scan tool(s) | Issue | |----------------------------- | --------------- | ------------ | ----- | -| Python ([Poetry](https://poetry.eustace.io/)) | `poetry.lock` | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | [GitLab#7006](https://gitlab.com/gitlab-org/gitlab/issues/7006) | +| Python ([Poetry](https://python-poetry.org/)) | `poetry.lock` | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | [GitLab#7006](https://gitlab.com/gitlab-org/gitlab/-/issues/7006) | | Python ([Pipenv](https://pipenv.pypa.io/en/latest/)) | `Pipfile.lock` | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | [GitLab#11756](https://gitlab.com/gitlab-org/gitlab/-/issues/11756) | ## Contribute your scanner @@ -151,11 +151,11 @@ The following variables allow configuration of global dependency scanning settin | Environment variable | Description | | --------------------------------------- |------------ | | `SECURE_ANALYZERS_PREFIX` | Override the name of the Docker registry providing the official default images (proxy). Read more about [customizing analyzers](analyzers.md). | -| `DS_ANALYZER_IMAGE_PREFIX` | **DEPRECATED:** Use `SECURE_ANALYZERS_PREFIX` instead. | | `DS_DEFAULT_ANALYZERS` | Override the names of the official default images. Read more about [customizing analyzers](analyzers.md). | | `DS_DISABLE_DIND` | Disable Docker-in-Docker and run analyzers [individually](#enabling-docker-in-docker). This variable is `true` by default. | | `ADDITIONAL_CA_CERT_BUNDLE` | Bundle of CA certs to trust. | | `DS_EXCLUDED_PATHS` | Exclude vulnerabilities from output based on the paths. A comma-separated list of patterns. Patterns can be globs, or file or folder paths (for example, `doc,spec`). Parent directories also match patterns. Default: `"spec, test, tests, tmp"` | +| `SECURE_LOG_LEVEL` | Default log level is `info`, you can set it to any of the following strings: `fatal`, `error`, `warn`, `info`, `debug`. | #### Configuring Docker-in-Docker orchestrator @@ -186,6 +186,7 @@ The following variables are used for configuring specific analyzers (used for a | `DS_PIP_VERSION` | `gemnasium-python` | | Force the install of a specific pip version (example: `"19.3"`), otherwise the pip installed in the Docker image is used. ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12811) in GitLab 12.7) | | `DS_PIP_DEPENDENCY_PATH` | `gemnasium-python` | | Path to load Python pip dependencies from. ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12412) in GitLab 12.2) | | `DS_PYTHON_VERSION` | `retire.js` | | Version of Python. If set to 2, dependencies are installed using Python 2.7 instead of Python 3.6. ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12296) in GitLab 12.1)| +| `DS_JAVA_VERSION` | `gemnasium-maven` | `11` | Version of Java. Available versions: `8`, `11`, `13`, `14`. Maven and Gradle will use the Java version specified by this value. | | `MAVEN_CLI_OPTS` | `gemnasium-maven` | `"-DskipTests --batch-mode"` | List of command line arguments that will be passed to `maven` by the analyzer. See an example for [using private repositories](../index.md#using-private-maven-repos). | | `GRADLE_CLI_OPTS` | `gemnasium-maven` | | List of command line arguments that will be passed to `gradle` by the analyzer. | | `SBT_CLI_OPTS` | `gemnasium-maven` | | List of command-line arguments that the analyzer will pass to `sbt`. | @@ -428,14 +429,14 @@ For details on saving and transporting Docker images as a file, see Docker's doc ### Set Dependency Scanning CI job variables to use local Dependency Scanning analyzers Add the following configuration to your `.gitlab-ci.yml` file. You must replace -`DS_ANALYZER_IMAGE_PREFIX` to refer to your local Docker container registry: +`SECURE_ANALYZERS_PREFIX` to refer to your local Docker container registry: ```yaml include: - template: Dependency-Scanning.gitlab-ci.yml variables: - DS_ANALYZER_IMAGE_PREFIX: "docker-registry.example.com/analyzers" + SECURE_ANALYZERS_PREFIX: "docker-registry.example.com/analyzers" GEMNASIUM_DB_REMOTE_URL: "gitlab.example.com/gemnasium-db.git" GIT_SSL_NO_VERIFY: "true" ``` diff --git a/doc/user/application_security/img/security_configuration_page_v13_1.png b/doc/user/application_security/img/security_configuration_page_v13_1.png Binary files differdeleted file mode 100644 index 176c64a9e87..00000000000 --- a/doc/user/application_security/img/security_configuration_page_v13_1.png +++ /dev/null diff --git a/doc/user/application_security/img/security_configuration_page_v13_2.png b/doc/user/application_security/img/security_configuration_page_v13_2.png Binary files differnew file mode 100644 index 00000000000..016328948cc --- /dev/null +++ b/doc/user/application_security/img/security_configuration_page_v13_2.png diff --git a/doc/user/application_security/index.md b/doc/user/application_security/index.md index 49580f494a2..3aca4c59423 100644 --- a/doc/user/application_security/index.md +++ b/doc/user/application_security/index.md @@ -116,6 +116,44 @@ information with several options:  +### View details of a DAST vulnerability + +Vulnerabilities detected by DAST occur in the live web application. Rectification of these types of +vulnerabilities requires specific information. DAST provides the information required to +investigate and rectify the underlying cause. + +To view details of DAST vulnerabilities: + +1. To see all vulnerabilities detected: + + - In a project, go to the project's **{shield}** **Security & Compliance** page. + - Only in a merge request, go the merge request's **Security** tab. + +1. Click on the vulnerability's description. The following details are provided: + + | Field | Description | +|:-----------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Description | Description of the vulnerability. | +| Project | Namespace and project in which the vulnerability was detected. | +| Method | HTTP method used to detect the vulnerability. | +| URL | URL at which the vulnerability was detected. | +| Request Headers | Headers of the request. | +| Response Status | Response status received from the application. | +| Response Headers | Headers of the response received from the application. | +| Evidence | Evidence of the data found that verified the vulnerability. Often a snippet of the request or response, this can be used to help verify that the finding is a vulnerability. | +| Identifiers | Identifiers of the vulnerability. | +| Severity | Severity of the vulnerability. | +| Scanner Type | Type of vulnerability report. | +| Links | Links to further details of the detected vulnerability. | +| Solution | Details of a recommended solution to the vulnerability (optional). | + +#### Hide sensitive information in headers + +HTTP request and response headers may contain sensitive information, including cookies and +authorization credentials. By default, content of specific headers are masked in DAST vulnerability +reports. You can specify the list of all headers to be masked. For details, see +[Hide sensitive information](dast/index.md#hide-sensitive-information). + ### Dismissing a vulnerability To dismiss a vulnerability, you must set its status to Dismissed. Follow these steps to do so: @@ -258,14 +296,16 @@ An approval is optional when a security report: > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/13067) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.3. -To enable License Approvals, a [project approval rule](../project/merge_requests/merge_request_approvals.md#multiple-approval-rules-premium) -must be created with the case-sensitive name `License-Check`. This approval group must be set -with the number of approvals required greater than zero. +`License-Check` is an approval rule you can enable to allow an individual or group to approve a +merge request that contains a `denied` license. + +You can enable `License-Check` one of two ways: -Once this group is added to your project, the approval rule is enabled for all Merge Requests. To -configure how this rule behaves, you can choose which licenses to `allow` or `deny` in the -[project policies for License Compliance](../compliance/license_compliance/index.md#project-policies-for-license-compliance) -section. +- Create a [project approval rule](../project/merge_requests/merge_request_approvals.md#multiple-approval-rules-premium) + with the case-sensitive name `License-Check`. +- Create an approval group in the [project policies section for License Compliance](../compliance/license_compliance/index.md#policies). + You must set this approval group's number of approvals required to greater than zero. Once you + enable this group in your project, the approval rule is enabled for all merge requests. Any code changes cause the approvals required to reset. diff --git a/doc/user/application_security/sast/analyzers.md b/doc/user/application_security/sast/analyzers.md index 0aa20bf4373..214044ad783 100644 --- a/doc/user/application_security/sast/analyzers.md +++ b/doc/user/application_security/sast/analyzers.md @@ -32,7 +32,6 @@ SAST supports the following official analyzers: - [`security-code-scan`](https://gitlab.com/gitlab-org/security-products/analyzers/security-code-scan) (Security Code Scan (.NET)) - [`sobelow`](https://gitlab.com/gitlab-org/security-products/analyzers/sobelow) (Sobelow (Elixir Phoenix)) - [`spotbugs`](https://gitlab.com/gitlab-org/security-products/analyzers/spotbugs) (SpotBugs with the Find Sec Bugs plugin (Ant, Gradle and wrapper, Grails, Maven and wrapper, SBT)) -- [`tslint`](https://gitlab.com/gitlab-org/security-products/analyzers/tslint) (TSLint (TypeScript)) The analyzers are published as Docker images that SAST will use to launch dedicated containers for each analysis. @@ -145,24 +144,24 @@ The [Security Scanner Integration](../../../development/integrations/secure.md) ## Analyzers Data -| Property \ Tool | Apex | Bandit | Brakeman | ESLint security | Find Sec Bugs | Flawfinder | Gosec | Kubesec Scanner | NodeJsScan | PHP CS Security Audit | Security code Scan (.NET) | Sobelow | TSLint Security | -| --------------------------------------- | :------------------: | :------------------: | :------------------: | :------------------: | :------------------: | :------------------: | :------------------: | :------------------: | :------------------: | :---------------------: | :-------------------------: | :----------------: | :-------------: | -| Severity | ✓ | ✓ | 𐄂 | 𐄂 | ✓ | 𐄂 | ✓ | ✓ | 𐄂 | ✓ | 𐄂 | 𐄂 | ✓ | -| Title | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | -| Description | ✓ | 𐄂 | 𐄂 | ✓ | ✓ | 𐄂 | 𐄂 | ✓ | ✓ | 𐄂 | 𐄂 | ✓ | ✓ | -| File | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | -| Start line | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | 𐄂 | ✓ | ✓ | ✓ | ✓ | ✓ | -| End line | ✓ | ✓ | 𐄂 | ✓ | ✓ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | ✓ | -| Start column | ✓ | 𐄂 | 𐄂 | ✓ | ✓ | ✓ | ✓ | 𐄂 | 𐄂 | ✓ | ✓ | 𐄂 | ✓ | -| End column | ✓ | 𐄂 | 𐄂 | ✓ | ✓ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | ✓ | -| External ID (e.g. CVE) | 𐄂 | 𐄂 | ⚠ | 𐄂 | ⚠ | ✓ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | -| URLs | ✓ | 𐄂 | ✓ | 𐄂 | ⚠ | 𐄂 | ⚠ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | -| Internal doc/explanation | ✓ | ⚠ | ✓ | 𐄂 | ✓ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | ✓ | 𐄂 | -| Solution | ✓ | 𐄂 | 𐄂 | 𐄂 | ⚠ | ✓ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | -| Affected item (e.g. class or package) | ✓ | 𐄂 | ✓ | 𐄂 | ✓ | ✓ | 𐄂 | ✓ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | -| Confidence | 𐄂 | ✓ | ✓ | 𐄂 | ✓ | ✓ | ✓ | ✓ | 𐄂 | 𐄂 | 𐄂 | ✓ | 𐄂 | -| Source code extract | 𐄂 | ✓ | ✓ | ✓ | 𐄂 | ✓ | ✓ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | -| Internal ID | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | 𐄂 | 𐄂 | ✓ | ✓ | ✓ | ✓ | +| Property / Tool | Apex | Bandit | Brakeman | ESLint security | SpotBugs | Flawfinder | Gosec | Kubesec Scanner | NodeJsScan | PHP CS Security Audit | Security code Scan (.NET) | Sobelow | +| --------------------------------------- | :------------------: | :------------------: | :------------------: | :------------------: | :------------------: | :------------------: | :------------------: | :------------------: | :------------------: | :---------------------: | :-------------------------: | :----------------: | +| Severity | ✓ | ✓ | 𐄂 | 𐄂 | ✓ | 𐄂 | ✓ | ✓ | 𐄂 | ✓ | 𐄂 | 𐄂 | +| Title | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| Description | ✓ | 𐄂 | 𐄂 | ✓ | ✓ | 𐄂 | 𐄂 | ✓ | ✓ | 𐄂 | 𐄂 | ✓ | +| File | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| Start line | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | 𐄂 | ✓ | ✓ | ✓ | ✓ | +| End line | ✓ | ✓ | 𐄂 | ✓ | ✓ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | +| Start column | ✓ | 𐄂 | 𐄂 | ✓ | ✓ | ✓ | ✓ | 𐄂 | 𐄂 | ✓ | ✓ | 𐄂 | +| End column | ✓ | 𐄂 | 𐄂 | ✓ | ✓ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | +| External ID (e.g. CVE) | 𐄂 | 𐄂 | ⚠ | 𐄂 | ⚠ | ✓ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | +| URLs | ✓ | 𐄂 | ✓ | 𐄂 | ⚠ | 𐄂 | ⚠ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | +| Internal doc/explanation | ✓ | ⚠ | ✓ | 𐄂 | ✓ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | ✓ | +| Solution | ✓ | 𐄂 | 𐄂 | 𐄂 | ⚠ | ✓ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | +| Affected item (e.g. class or package) | ✓ | 𐄂 | ✓ | 𐄂 | ✓ | ✓ | 𐄂 | ✓ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | +| Confidence | 𐄂 | ✓ | ✓ | 𐄂 | ✓ | ✓ | ✓ | ✓ | 𐄂 | 𐄂 | 𐄂 | ✓ | +| Source code extract | 𐄂 | ✓ | ✓ | ✓ | 𐄂 | ✓ | ✓ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | +| Internal ID | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | 𐄂 | 𐄂 | ✓ | ✓ | ✓ | - ✓ => we have that data - ⚠ => we have that data but it's partially reliable, or we need to extract it from unstructured content diff --git a/doc/user/application_security/sast/img/sast_v13_0.png b/doc/user/application_security/sast/img/sast_v13_0.png Binary files differdeleted file mode 100644 index b4aea6ea466..00000000000 --- a/doc/user/application_security/sast/img/sast_v13_0.png +++ /dev/null diff --git a/doc/user/application_security/sast/img/sast_v13_2.png b/doc/user/application_security/sast/img/sast_v13_2.png Binary files differnew file mode 100644 index 00000000000..5697ed9beb0 --- /dev/null +++ b/doc/user/application_security/sast/img/sast_v13_2.png diff --git a/doc/user/application_security/sast/index.md b/doc/user/application_security/sast/index.md index a5497e3d38c..70d4b513cf9 100644 --- a/doc/user/application_security/sast/index.md +++ b/doc/user/application_security/sast/index.md @@ -9,9 +9,9 @@ type: reference, howto > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/3775) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 10.3. -NOTE: **4 of the top 6 attacks were application based.** -Download our whitepaper, -["A Seismic Shift in Application Security"](https://about.gitlab.com/resources/whitepaper-seismic-shift-application-security/) +NOTE: **Note:** +The whitepaper ["A Seismic Shift in Application Security"](https://about.gitlab.com/resources/whitepaper-seismic-shift-application-security/) +explains how **4 of the top 6 attacks were application based**. Download it to learn how to protect your organization. ## Overview @@ -28,7 +28,7 @@ You can take advantage of SAST by doing one of the following: GitLab checks the SAST report, compares the found vulnerabilities between the source and target branches, and shows the information right on the merge request. - + The results are sorted by the priority of the vulnerability: @@ -58,7 +58,8 @@ If you're using the shared Runners on GitLab.com, this is enabled by default. Beginning with GitLab 13.0, Docker privileged mode is necessary only if you've [enabled Docker-in-Docker for SAST](#enabling-docker-in-docker). -CAUTION: **Caution:** Our SAST jobs currently expect a Linux container type. Windows containers are not yet supported. +CAUTION: **Caution:** +Our SAST jobs currently expect a Linux container type. Windows containers are not yet supported. CAUTION: **Caution:** If you use your own Runners, make sure the Docker version installed @@ -70,31 +71,54 @@ The following table shows which languages, package managers and frameworks are s | Language (package managers) / framework | Scan tool | Introduced in GitLab Version | |-----------------------------------------------------------------------------|----------------------------------------------------------------------------------------|------------------------------| -| .NET Core | [Security Code Scan](https://security-code-scan.github.io) | 11.0 | -| .NET Framework | [Security Code Scan](https://security-code-scan.github.io) | 13.0 | -| Any | [Gitleaks](https://github.com/zricethezav/gitleaks) and [TruffleHog](https://github.com/dxa4481/truffleHog) | 11.9 | -| Apex (Salesforce) | [PMD](https://pmd.github.io/pmd/index.html) | 12.1 | -| C/C++ | [Flawfinder](https://github.com/david-a-wheeler/flawfinder) | 10.7 | -| Elixir (Phoenix) | [Sobelow](https://github.com/nccgroup/sobelow) | 11.10 | -| Go | [Gosec](https://github.com/securego/gosec) | 10.7 | -| Groovy ([Ant](https://ant.apache.org/), [Gradle](https://gradle.org/), [Maven](https://maven.apache.org/) and [SBT](https://www.scala-sbt.org/)) | [SpotBugs](https://spotbugs.github.io/) with the [find-sec-bugs](https://find-sec-bugs.github.io/) plugin | 11.3 (Gradle) & 11.9 (Ant, Maven, SBT) | -| Helm Charts | [Kubesec](https://github.com/controlplaneio/kubesec) | 13.1 | +| .NET Core | [Security Code Scan](https://security-code-scan.github.io) | 11.0 | +| .NET Framework | [Security Code Scan](https://security-code-scan.github.io) | 13.0 | +| Any | [Gitleaks](https://github.com/zricethezav/gitleaks) and [TruffleHog](https://github.com/dxa4481/truffleHog) | 11.9 | +| Apex (Salesforce) | [PMD](https://pmd.github.io/pmd/index.html) | 12.1 | +| C/C++ | [Flawfinder](https://github.com/david-a-wheeler/flawfinder) | 10.7 | +| Elixir (Phoenix) | [Sobelow](https://github.com/nccgroup/sobelow) | 11.10 | +| Go | [Gosec](https://github.com/securego/gosec) | 10.7 | +| Groovy ([Ant](https://ant.apache.org/), [Gradle](https://gradle.org/), [Maven](https://maven.apache.org/) and [SBT](https://www.scala-sbt.org/)) | [SpotBugs](https://spotbugs.github.io/) with the [find-sec-bugs](https://find-sec-bugs.github.io/) plugin | 11.3 (Gradle) & 11.9 (Ant, Maven, SBT) | +| Helm Charts | [Kubesec](https://github.com/controlplaneio/kubesec) | 13.1 | | Java ([Ant](https://ant.apache.org/), [Gradle](https://gradle.org/), [Maven](https://maven.apache.org/) and [SBT](https://www.scala-sbt.org/)) | [SpotBugs](https://spotbugs.github.io/) with the [find-sec-bugs](https://find-sec-bugs.github.io/) plugin | 10.6 (Maven), 10.8 (Gradle) & 11.9 (Ant, SBT) | -| JavaScript | [ESLint security plugin](https://github.com/nodesecurity/eslint-plugin-security) | 11.8 | +| JavaScript | [ESLint security plugin](https://github.com/nodesecurity/eslint-plugin-security) | 11.8, moved to [GitLab Core](https://about.gitlab.com/pricing/) in 13.2 | | Kubernetes manifests | [Kubesec](https://github.com/controlplaneio/kubesec) | 12.6 | | Node.js | [NodeJsScan](https://github.com/ajinabraham/NodeJsScan) | 11.1 | | PHP | [phpcs-security-audit](https://github.com/FloeDesignTechnologies/phpcs-security-audit) | 10.8 | | Python ([pip](https://pip.pypa.io/en/stable/)) | [bandit](https://github.com/PyCQA/bandit) | 10.3 | | React | [ESLint react plugin](https://github.com/yannickcr/eslint-plugin-react) | 12.5 | -| Ruby on Rails | [brakeman](https://brakemanscanner.org) | 10.3 | +| Ruby on Rails | [brakeman](https://brakemanscanner.org) | 10.3, moved to [GitLab Core](https://about.gitlab.com/pricing/) in 13.1 | | Scala ([Ant](https://ant.apache.org/), [Gradle](https://gradle.org/), [Maven](https://maven.apache.org/) and [SBT](https://www.scala-sbt.org/)) | [SpotBugs](https://spotbugs.github.io/) with the [find-sec-bugs](https://find-sec-bugs.github.io/) plugin | 11.0 (SBT) & 11.9 (Ant, Gradle, Maven) | -| TypeScript | [`tslint-config-security`](https://github.com/webschik/tslint-config-security/) | 11.9 | +| TypeScript | [ESLint security plugin](https://github.com/nodesecurity/eslint-plugin-security) | 11.9, merged with ESLint in 13.2 | NOTE: **Note:** The Java analyzers can also be used for variants like the [Gradle wrapper](https://docs.gradle.org/current/userguide/gradle_wrapper.html), [Grails](https://grails.org/) and the [Maven wrapper](https://github.com/takari/maven-wrapper). +### Making SAST analyzers available to all GitLab tiers + +All open source (OSS) analyzers are in the process of being reviewed and potentially moved to the GitLab Core tier. Progress can be +tracked in the corresponding +[epic](https://gitlab.com/groups/gitlab-org/-/epics/2098). + +Please note that support for [Docker-in-Docker](#enabling-docker-in-docker) +will not be extended to the GitLab Core tier. + +#### Summary of features per tier + +Different features are available in different [GitLab tiers](https://about.gitlab.com/pricing/), +as shown in the following table: + +| Capability | In Core | In Ultimate | +|:--------------------------------------------------------------------------|:--------------------|:-------------------| +| [Configure SAST Scanners](#configuration) | **{check-circle}** | **{check-circle}** | +| [Customize SAST Settings](#customizing-the-sast-settings) | **{check-circle}** | **{check-circle}** | +| View [JSON Report](#reports-json-format) | **{check-circle}** | **{check-circle}** | +| [Presentation of JSON Report in Merge Request](#overview) | **{dotted-circle}** | **{check-circle}** | +| [Interaction with Vulnerabilities](#interacting-with-the-vulnerabilities) | **{dotted-circle}** | **{check-circle}** | +| [Access to Security Dashboard](#security-dashboard) | **{dotted-circle}** | **{check-circle}** | + ## Contribute your scanner The [Security Scanner Integration](../../../development/integrations/secure.md) documentation explains how to integrate other security scanners into GitLab. @@ -222,7 +246,7 @@ a `before_script` execution to prepare your scan job. To pass your project's dependencies as artifacts, the dependencies must be included in the project's working directory and specified using the `artifacts:path` configuration. -If all dependencies are present, the `-compile=false` flag can be provided to the +If all dependencies are present, the `COMPILE=false` variable can be provided to the analyzer and compilation will be skipped: ```yaml @@ -247,10 +271,9 @@ build: spotbugs-sast: dependencies: - build - script: - - /analyzer run -compile=false variables: MAVEN_REPO_PATH: ./.m2/repository + COMPILE: false artifacts: reports: sast: gl-sast-report.json @@ -266,6 +289,16 @@ See [Analyzer settings](#analyzer-settings) for the complete list of available o SAST can be [configured](#customizing-the-sast-settings) using environment variables. +#### Logging Level + +You can control the verbosity of logs by setting the `SECURE_LOG_LEVEL` env var. The default is set to `info`, you can set it to any of the following levels: + +- `fatal` +- `error` +- `warn` +- `info` +- `debug` + #### Custom Certificate Authority To trust a custom Certificate Authority, set the `ADDITIONAL_CA_CERT_BUNDLE` variable to the bundle @@ -278,7 +311,6 @@ The following are Docker image-related variables. | Environment variable | Description | |------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `SECURE_ANALYZERS_PREFIX` | Override the name of the Docker registry providing the default images (proxy). Read more about [customizing analyzers](analyzers.md). | -| `SAST_ANALYZER_IMAGE_PREFIX` | **DEPRECATED**: Use `SECURE_ANALYZERS_PREFIX` instead. | | `SAST_ANALYZER_IMAGE_TAG` | **DEPRECATED:** Override the Docker tag of the default images. Read more about [customizing analyzers](analyzers.md). | | `SAST_DEFAULT_ANALYZERS` | Override the names of default images. Read more about [customizing analyzers](analyzers.md). | | `SAST_DISABLE_DIND` | Disable Docker-in-Docker and run analyzers [individually](#enabling-docker-in-docker). This variable is `true` by default. | @@ -287,17 +319,18 @@ The following are Docker image-related variables. Some analyzers make it possible to filter out vulnerabilities under a given threshold. -| Environment variable | Default value | Description | -|-------------------------|---------------|-------------| +| Environment variable | Default value | Description | +|-------------------------------|--------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `SAST_EXCLUDED_PATHS` | `spec, test, tests, tmp` | Exclude vulnerabilities from output based on the paths. This is a comma-separated list of patterns. Patterns can be globs, or file or folder paths (for example, `doc,spec` ). Parent directories will also match patterns. | -| `SAST_BANDIT_EXCLUDED_PATHS` | - | comma-separated list of paths to exclude from scan. Uses Python's [`fnmatch` syntax](https://docs.python.org/2/library/fnmatch.html); For example: `'*/tests/*'` | -| `SAST_BRAKEMAN_LEVEL` | 1 | Ignore Brakeman vulnerabilities under given confidence level. Integer, 1=Low 3=High. | -| `SAST_FLAWFINDER_LEVEL` | 1 | Ignore Flawfinder vulnerabilities under given risk level. Integer, 0=No risk, 5=High risk. | -| `SAST_GITLEAKS_ENTROPY_LEVEL` | 8.0 | Minimum entropy for secret detection. Float, 0.0 = low, 8.0 = high. | -| `SAST_GOSEC_LEVEL` | 0 | Ignore Gosec vulnerabilities under given confidence level. Integer, 0=Undefined, 1=Low, 2=Medium, 3=High. | -| `SAST_GITLEAKS_COMMIT_FROM` | - | The commit a Gitleaks scan starts at. | -| `SAST_GITLEAKS_COMMIT_TO` | - | The commit a Gitleaks scan ends at. | -| `SAST_GITLEAKS_HISTORIC_SCAN` | false | Flag to enable a historic Gitleaks scan. | +| `SAST_BANDIT_EXCLUDED_PATHS` | | Comma-separated list of paths to exclude from scan. Uses Python's [`fnmatch` syntax](https://docs.python.org/2/library/fnmatch.html); For example: `'*/tests/*, */venv/*'` | +| `SAST_BRAKEMAN_LEVEL` | 1 | Ignore Brakeman vulnerabilities under given confidence level. Integer, 1=Low 3=High. | +| `SAST_DISABLE_BABEL` | `false` | Disable Babel processing for the NodeJsScan scanner. Set to `true` to disable Babel processing. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/33065) in GitLab 13.2. | +| `SAST_FLAWFINDER_LEVEL` | 1 | Ignore Flawfinder vulnerabilities under given risk level. Integer, 0=No risk, 5=High risk. | +| `SAST_GITLEAKS_ENTROPY_LEVEL` | 8.0 | Minimum entropy for secret detection. Float, 0.0 = low, 8.0 = high. | +| `SAST_GOSEC_LEVEL` | 0 | Ignore Gosec vulnerabilities under given confidence level. Integer, 0=Undefined, 1=Low, 2=Medium, 3=High. | +| `SAST_GITLEAKS_COMMIT_FROM` | | The commit a Gitleaks scan starts at. | +| `SAST_GITLEAKS_COMMIT_TO` | | The commit a Gitleaks scan ends at. | +| `SAST_GITLEAKS_HISTORIC_SCAN` | `false` | Flag to enable a historic Gitleaks scan. | #### Docker-in-Docker orchestrator @@ -315,11 +348,12 @@ The following variables configure the Docker-in-Docker orchestrator, and therefo Some analyzers can be customized with environment variables. -| Environment variable | Analyzer | Description | -|-----------------------------|----------|-------------| +| Environment variable | Analyzer | Description | +|---------------------------------------|----------------------|-------------| | `SCAN_KUBERNETES_MANIFESTS` | Kubesec | Set to `"true"` to scan Kubernetes manifests. | | `KUBESEC_HELM_CHARTS_PATH` | Kubesec | Optional path to Helm charts that `helm` will use to generate a Kubernetes manifest that `kubesec` will scan. If dependencies are defined, `helm dependency build` should be ran in a `before_script` to fetch the necessary dependencies. | | `KUBESEC_HELM_OPTIONS` | Kubesec | Additional arguments for the `helm` executable. | +| `COMPILE` | SpotBugs | Set to `false` to disable project compilation and dependency fetching. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/195252) in GitLab 13.1. | | `ANT_HOME` | SpotBugs | The `ANT_HOME` environment variable. | | `ANT_PATH` | SpotBugs | Path to the `ant` executable. | | `GRADLE_PATH` | SpotBugs | Path to the `gradle` executable. | @@ -333,6 +367,7 @@ Some analyzers can be customized with environment variables. | `FAIL_NEVER` | SpotBugs | Set to `1` to ignore compilation failure. | | `SAST_GOSEC_CONFIG` | Gosec | Path to configuration for Gosec (optional). | | `PHPCS_SECURITY_AUDIT_PHP_EXTENSIONS` | phpcs-security-audit | Comma separated list of additional PHP Extensions. | +| `SEARCH_MAX_DEPTH` | any | Maximum number of directories traversed when searching for source code files. Default: `4`. | #### Custom environment variables @@ -494,7 +529,6 @@ registry.gitlab.com/gitlab-org/security-products/analyzers/secrets:2 registry.gitlab.com/gitlab-org/security-products/analyzers/security-code-scan:2 registry.gitlab.com/gitlab-org/security-products/analyzers/sobelow:2 registry.gitlab.com/gitlab-org/security-products/analyzers/spotbugs:2 -registry.gitlab.com/gitlab-org/security-products/analyzers/tslint:2 ``` The process for importing Docker images into a local offline Docker registry depends on @@ -509,7 +543,7 @@ For details on saving and transporting Docker images as a file, see Docker's doc ### Set SAST CI job variables to use local SAST analyzers Add the following configuration to your `.gitlab-ci.yml` file. You must replace -`SAST_ANALYZER_IMAGE_PREFIX` to refer to your local Docker container registry: +`SECURE_ANALYZERS_PREFIX` to refer to your local Docker container registry: ```yaml include: diff --git a/doc/user/application_security/secret_detection/img/secret-detection-merge-request-ui.png b/doc/user/application_security/secret_detection/img/secret-detection-merge-request-ui.png Binary files differdeleted file mode 100644 index 17893610f10..00000000000 --- a/doc/user/application_security/secret_detection/img/secret-detection-merge-request-ui.png +++ /dev/null diff --git a/doc/user/application_security/secret_detection/img/secret_detection_v13_2.png b/doc/user/application_security/secret_detection/img/secret_detection_v13_2.png Binary files differnew file mode 100644 index 00000000000..4aa7dd83c8d --- /dev/null +++ b/doc/user/application_security/secret_detection/img/secret_detection_v13_2.png diff --git a/doc/user/application_security/secret_detection/index.md b/doc/user/application_security/secret_detection/index.md index 85933c31a34..ea635212c5d 100644 --- a/doc/user/application_security/secret_detection/index.md +++ b/doc/user/application_security/secret_detection/index.md @@ -25,7 +25,7 @@ GitLab displays identified secrets as part of the SAST reports visibly in a few - Pipelines' **Security** tab - Report in the merge request widget - + ## Use cases @@ -39,7 +39,8 @@ To run Secret Detection jobs, by default, you need GitLab Runner with the [`kubernetes`](https://docs.gitlab.com/runner/install/kubernetes.html) executor. If you're using the shared Runners on GitLab.com, this is enabled by default. -CAUTION: **Caution:** Our Secret Detection jobs currently expect a Linux container type. Windows containers are not yet supported. +CAUTION: **Caution:** +Our Secret Detection jobs currently expect a Linux container type. Windows containers are not yet supported. CAUTION: **Caution:** If you use your own Runners, make sure the Docker version installed @@ -118,15 +119,15 @@ declare a job with the same name as the SAST job to override. Place this new job inclusion and specify any additional keys under it. In the following example, we include the Secret Detection template and at the same time we -override the `secret-scan` job with the `SECRET_DETECTION_HISTORIC_SCAN` variable to `true`: +override the `secret_detection` job with the `SECRET_DETECTION_HISTORIC_SCAN` variable to `true`: ```yaml include: - template: Secret-Detection.gitlab-ci.yml -secrets-scan: +secret_detection: variables: - SECRET_DETECTION_HISTORIC_SCAN: true + SECRET_DETECTION_HISTORIC_SCAN: "true" ``` Because the template is [evaluated before](../../../ci/yaml/README.md#include) @@ -146,6 +147,16 @@ Secret Detection can be customized by defining available variables: | `SECRET_DETECTION_COMMIT_TO` | - | The commit a Gitleaks scan ends at. | | `SECRET_DETECTION_HISTORIC_SCAN` | false | Flag to enable a historic Gitleaks scan. | +### Logging Level + +You can control the verbosity of logs by setting the `SECURE_LOG_LEVEL` env var. The default is set to `info`, you can set it to any of the following levels: + +- `fatal` +- `error` +- `warn` +- `info` +- `debug` + ## Full History Secret Scan GitLab 12.11 introduced support for scanning the full history of a repository. This new functionality diff --git a/doc/user/application_security/security_dashboard/img/group_security_dashboard_export_csv_v13_1.png b/doc/user/application_security/security_dashboard/img/group_security_dashboard_export_csv_v13_1.png Binary files differindex 0dfe7b637cd..d98fb71ae37 100644 --- a/doc/user/application_security/security_dashboard/img/group_security_dashboard_export_csv_v13_1.png +++ b/doc/user/application_security/security_dashboard/img/group_security_dashboard_export_csv_v13_1.png diff --git a/doc/user/application_security/security_dashboard/img/group_security_dashboard_v13_0.png b/doc/user/application_security/security_dashboard/img/group_security_dashboard_v13_0.png Binary files differdeleted file mode 100644 index 4c7b5cc724f..00000000000 --- a/doc/user/application_security/security_dashboard/img/group_security_dashboard_v13_0.png +++ /dev/null diff --git a/doc/user/application_security/security_dashboard/img/group_security_dashboard_v13_2_noNav.png b/doc/user/application_security/security_dashboard/img/group_security_dashboard_v13_2_noNav.png Binary files differnew file mode 100644 index 00000000000..d6cfc2de980 --- /dev/null +++ b/doc/user/application_security/security_dashboard/img/group_security_dashboard_v13_2_noNav.png diff --git a/doc/user/application_security/security_dashboard/img/instance_security_dashboard_with_projects_v13_0.png b/doc/user/application_security/security_dashboard/img/instance_security_dashboard_with_projects_v13_0.png Binary files differdeleted file mode 100644 index a500f186c2b..00000000000 --- a/doc/user/application_security/security_dashboard/img/instance_security_dashboard_with_projects_v13_0.png +++ /dev/null diff --git a/doc/user/application_security/security_dashboard/img/instance_security_dashboard_with_projects_v13_2_sm.png b/doc/user/application_security/security_dashboard/img/instance_security_dashboard_with_projects_v13_2_sm.png Binary files differnew file mode 100644 index 00000000000..75b5ad1d885 --- /dev/null +++ b/doc/user/application_security/security_dashboard/img/instance_security_dashboard_with_projects_v13_2_sm.png diff --git a/doc/user/application_security/security_dashboard/img/pipeline_security_dashboard_v12_6.png b/doc/user/application_security/security_dashboard/img/pipeline_security_dashboard_v12_6.png Binary files differdeleted file mode 100644 index 670c90d10a3..00000000000 --- a/doc/user/application_security/security_dashboard/img/pipeline_security_dashboard_v12_6.png +++ /dev/null diff --git a/doc/user/application_security/security_dashboard/img/pipeline_security_dashboard_v13_2.png b/doc/user/application_security/security_dashboard/img/pipeline_security_dashboard_v13_2.png Binary files differnew file mode 100644 index 00000000000..591a08f4d7a --- /dev/null +++ b/doc/user/application_security/security_dashboard/img/pipeline_security_dashboard_v13_2.png diff --git a/doc/user/application_security/security_dashboard/img/project_security_dashboard_v13_2.png b/doc/user/application_security/security_dashboard/img/project_security_dashboard_v13_2.png Binary files differnew file mode 100644 index 00000000000..7cab7b0a61f --- /dev/null +++ b/doc/user/application_security/security_dashboard/img/project_security_dashboard_v13_2.png diff --git a/doc/user/application_security/security_dashboard/img/standalone_vulnerability_page_v13_1.png b/doc/user/application_security/security_dashboard/img/standalone_vulnerability_page_v13_1.png Binary files differnew file mode 100644 index 00000000000..9cf95b197fe --- /dev/null +++ b/doc/user/application_security/security_dashboard/img/standalone_vulnerability_page_v13_1.png diff --git a/doc/user/application_security/security_dashboard/img/vulnerability_list_table_v13_1.png b/doc/user/application_security/security_dashboard/img/vulnerability_list_table_v13_1.png Binary files differnew file mode 100644 index 00000000000..2b792727a99 --- /dev/null +++ b/doc/user/application_security/security_dashboard/img/vulnerability_list_table_v13_1.png diff --git a/doc/user/application_security/security_dashboard/index.md b/doc/user/application_security/security_dashboard/index.md index 60798b9c921..9a13d143d1f 100644 --- a/doc/user/application_security/security_dashboard/index.md +++ b/doc/user/application_security/security_dashboard/index.md @@ -1,5 +1,8 @@ --- type: reference, howto +stage: Secure +group: Threat Insights +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # GitLab Security Dashboard **(ULTIMATE)** @@ -9,7 +12,7 @@ vulnerabilities in your groups, projects and pipelines. You can also drill down into a vulnerability and get extra information, see which project it comes from, the file it's in, and various metadata to help you analyze -the risk. You can also action these vulnerabilities by creating an issue for them, +the risk. You can also take actions on vulnerabilities by creating an issue for them, or by dismissing them. To benefit from the Security Dashboard you must first configure one of the @@ -42,7 +45,7 @@ At the pipeline level, the Security section displays the vulnerabilities present Visit the page for any pipeline which has run any of the [supported reports](#supported-reports). Click the **Security** tab to view the Security findings. - + NOTE: **Note:** A pipeline consists of multiple jobs, including SAST and DAST scanning. If any job fails to finish for any reason, the security dashboard will not show SAST scanner output. For example, if the SAST job finishes but the DAST job fails, the security dashboard will not show SAST results. The analyzer will output an [exit code](../../../development/integrations/secure.md#exit-code) on failure. @@ -51,56 +54,52 @@ A pipeline consists of multiple jobs, including SAST and DAST scanning. If any j > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/6165) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.1. -At the project level, the Security Dashboard displays the latest security reports for your project. -Use it to find and fix vulnerabilities. +At the project level, the Security Dashboard displays the vulnerabilities merged into your project's +[default branch](../../project/repository/branches/index.md#default-branch). Access it by navigating +to **Security & Compliance > Security Dashboard**. - +The Security Dashboard first displays the total number of vulnerabilities by severity (for example, +Critical, High, Medium, Low). Below this, a table displays each vulnerability's status, severity, +and description. Clicking a vulnerability takes you to its [Vulnerability Details](../vulnerabilities) +page to view more information about that vulnerability. -### Export vulnerabilities +You can filter the vulnerabilities by: -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/197494) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.10. +- Status +- Severity +- Report type -You can export all your project's vulnerabilities as CSV by clicking on the export button located at top right of the Project Security Dashboard. This will initiate the process, and once complete, the CSV report will be downloaded. The report will contain all vulnerabilities in the project as filters won't apply. +You can also dismiss vulnerabilities in the table: -NOTE: **Note:** -It may take several minutes for the download to start if your project consists -of thousands of vulnerabilities. Do not close the page until the download finishes. +1. Select the checkbox for each vulnerability you want to dismiss. +1. In the menu that appears, select the reason for dismissal and click **Dismiss Selected**. - + ## Group Security Dashboard > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/6709) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.5. -The group Security Dashboard gives an overview of the vulnerabilities of all the -projects in a group and its subgroups. +The group Security Dashboard gives an overview of the vulnerabilities in the default branches of the +projects in a group and its subgroups. Access it by navigating to **Security > Security Dashboard** +for your group. -First, navigate to the Security Dashboard found under your group's -**Security** tab. +NOTE: **Note:** +The Security Dashboard only shows projects with [security reports](#supported-reports) enabled in a +group. + + -Once you're on the dashboard, at the top you should see a series of filters for: +You can filter which vulnerabilities the Security Dashboard displays by: - Status - Severity - Report type +- Project -NOTE: **Note:** -The dashboard only shows projects with [security reports](#supported-reports) enabled in a group. - - - -Selecting one or more filters will filter the results in this page. - -The main section is a list of all the vulnerabilities in the group, sorted by severity. -In that list, you can see the severity of the vulnerability, its name, its -confidence (likelihood of the vulnerability to be a positive one), and the project -it's from. - -If you hover over a row, the following actions appear: - -- More info -- Create issue -- Dismiss vulnerability +A table lists the vulnerabilities, sorted by severity. The table shows each vulnerability's status, +severity, and description. Clicking a vulnerability takes you to its [Vulnerability Details](../vulnerabilities) +page to view more information about that vulnerability. Next to the list is a timeline chart that shows how many open vulnerabilities your projects had at various points in time. You can filter among 30, 60, and @@ -120,28 +119,14 @@ vulnerabilities are not included either. Read more on how to [interact with the vulnerabilities](../index.md#interacting-with-the-vulnerabilities). -### Export vulnerabilities - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/213013) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.1. - -You can export all your vulnerabilities as CSV by clicking the **{upload}** **Export** button -located at the top right of the **Group Security Dashboard**. After the report builds, the CSV -report downloads to your local machine. The report contains all vulnerabilities for the projects -defined in the **Group Security Dashboard**, as filters don't apply to the export function. - -NOTE: **Note:** -It may take several minutes for the download to start if your project contains thousands of -vulnerabilities. Don't close the page until the download finishes. - - - ## Instance Security Dashboard > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/6953) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.8. -At the instance level, the Security Dashboard displays the vulnerabilities -present in all of the projects that you have added to it. It includes all -of the features of the [group security dashboard](#group-security-dashboard). +At the instance level, the Security Dashboard displays the vulnerabilities present in the default +branches of all the projects you configure to display on the dashboard. It includes all the +[group Security Dashboard's](#group-security-dashboard) +features. You can access the Instance Security Dashboard from the menu bar at the top of the page. Under **More**, select **Security**. @@ -156,27 +141,25 @@ To add projects to the dashboard: 1. Search for and add one or more projects using the **Search your projects** field. 1. Click the **Add projects** button. -Once added, the dashboard will display the vulnerabilities found in your chosen -projects. +Once added, the Security Dashboard displays the vulnerabilities found in your chosen projects' +default branches. - + -### Export vulnerabilities +## Export vulnerabilities -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/213014) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.0. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/213014) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.10. -You can export all your vulnerabilities as CSV by clicking the **{upload}** **Export** -button located at top right of the **Instance Security Dashboard**. After the report +You can export all your vulnerabilities in CSV format by clicking the **{upload}** **Export** +button located at top right of the **Security Dashboard**. After the report is built, the CSV report downloads to your local machine. The report contains all -vulnerabilities for the projects defined in the **Instance Security Dashboard**, +vulnerabilities for the projects defined in the **Security Dashboard**, as filters don't apply to the export function. NOTE: **Note:** It may take several minutes for the download to start if your project contains thousands of vulnerabilities. Do not close the page until the download finishes. - - ## Keeping the dashboards up to date The Security Dashboard displays information from the results of the most recent @@ -194,12 +177,34 @@ Dashboard regardless of how often the default branch is updated. That way, reports are created even if no code change happens. +CAUTION: **Warning:** +Running Dependency Scanning from a scheduled pipeline might result in false negatives if your +project doesn't have a lock file and isn't configured for Continuous Delivery. A lock file is a file +that lists all transient dependencies and keeps track of their exact versions. The false negative +can occur because the dependency version resolved during the scan might differ from the ones +resolved when your project was built and released, in a previous pipeline. Java projects can't have +lock files. Python projects can have lock files, but GitLab Secure tools don't support them. + ## Security scans using Auto DevOps When using [Auto DevOps](../../../topics/autodevops/index.md), use [special environment variables](../../../topics/autodevops/customize.md#environment-variables) to configure daily security scans. +## Vulnerability list + +Each dashboard's vulnerability list contains vulnerabilities from the latest scans that were merged +into the default branch. +Click any vulnerability in the table to see more information on that vulnerability. To create an +issue associated with the vulnerability, click the **Create Issue** button. + + + +Once you create the issue, the vulnerability list contains a link to the issue and an icon whose +color indicates the issue's status (green for open issues, blue for closed issues). + + + <!-- ## Troubleshooting Include any troubleshooting steps that you can foresee. If you know beforehand what issues diff --git a/doc/user/application_security/threat_monitoring/index.md b/doc/user/application_security/threat_monitoring/index.md index 434048896fe..a6738677454 100644 --- a/doc/user/application_security/threat_monitoring/index.md +++ b/doc/user/application_security/threat_monitoring/index.md @@ -58,12 +58,15 @@ prerequisites: If you're using custom Helm values for Cilium, you must enable Hubble with flow metrics for each namespace by adding the following lines to -your [Hubble values](../../clusters/applications.md#install-cilium-using-gitlab-cicd): +your [Cilium values](../../clusters/applications.md#install-cilium-using-gitlab-cicd): ```yaml -metrics: - enabled: - - 'flow:sourceContext=namespace;destinationContext=namespace' +global: + hubble: + enabled: true + metrics: + enabled: + - 'flow:sourceContext=namespace;destinationContext=namespace' ``` The **Container Network Policy** section displays the following information diff --git a/doc/user/application_security/vulnerabilities/img/standalone_vulnerability_page_v12_10.png b/doc/user/application_security/vulnerabilities/img/standalone_vulnerability_page_v12_10.png Binary files differdeleted file mode 100644 index 0fdb8d1e201..00000000000 --- a/doc/user/application_security/vulnerabilities/img/standalone_vulnerability_page_v12_10.png +++ /dev/null diff --git a/doc/user/application_security/vulnerabilities/img/standalone_vulnerability_page_v13_1.png b/doc/user/application_security/vulnerabilities/img/standalone_vulnerability_page_v13_1.png Binary files differnew file mode 100644 index 00000000000..e0e0fdb6f6e --- /dev/null +++ b/doc/user/application_security/vulnerabilities/img/standalone_vulnerability_page_v13_1.png diff --git a/doc/user/application_security/vulnerabilities/index.md b/doc/user/application_security/vulnerabilities/index.md index b3128e49980..d5cce6434d8 100644 --- a/doc/user/application_security/vulnerabilities/index.md +++ b/doc/user/application_security/vulnerabilities/index.md @@ -1,7 +1,7 @@ --- type: reference, howto stage: Secure -group: Vulnerability Research +group: Threat Insights info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- @@ -9,10 +9,10 @@ info: To determine the technical writer assigned to the Stage/Group associated w > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/13561) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.0. -Each security vulnerability in the [Vulnerability List](../dependency_list/index.md) has its own standalone +Each security vulnerability in the [Security Dashboard](../security_dashboard/index.md#project-security-dashboard) has its own standalone page. - + On the standalone vulnerability page, you can interact with the vulnerability in several different ways: @@ -30,7 +30,7 @@ several different ways: You can switch the status of a vulnerability using the **Status** dropdown to one of the following values: -| State | Description | +| Status | Description | |-----------|-------------------------------------------------------------------| | Detected | The default state for a newly discovered vulnerability | | Confirmed | A user has seen this vulnerability and confirmed it to be real | diff --git a/doc/user/asciidoc.md b/doc/user/asciidoc.md index 8834deb8d50..512a98d567b 100644 --- a/doc/user/asciidoc.md +++ b/doc/user/asciidoc.md @@ -10,14 +10,14 @@ You can find the full documentation for the AsciiDoc syntax at <https://asciidoc ### Paragraphs -```asciidoc +```plaintext A normal paragraph. Line breaks are not preserved. ``` Line comments, which are lines that start with `//`, are skipped: -```asciidoc +```plaintext // this is a comment ``` @@ -25,7 +25,7 @@ A blank line separates paragraphs. A paragraph with the `[%hardbreaks]` option will preserve line breaks: -```asciidoc +```plaintext [%hardbreaks] This paragraph carries the `hardbreaks` option. Notice how line breaks are now preserved. @@ -35,7 +35,7 @@ An indented (literal) paragraph disables text formatting, preserves spaces and line breaks, and is displayed in a monospaced font: -```asciidoc +```plaintext This literal paragraph is indented with one space. As a consequence, *text formatting*, spaces, and lines breaks will be preserved. @@ -43,7 +43,7 @@ monospaced font: An admonition paragraph grabs the reader's attention: -```asciidoc +```plaintext NOTE: This is a brief reference, please read the full documentation at https://asciidoctor.org/docs/. TIP: Lists can be indented. Leading whitespace is not significant. @@ -53,7 +53,7 @@ TIP: Lists can be indented. Leading whitespace is not significant. **Constrained (applied at word boundaries)** -```asciidoc +```plaintext *strong importance* (aka bold) _stress emphasis_ (aka italic) `monospaced` (aka typewriter text) @@ -64,7 +64,7 @@ _stress emphasis_ (aka italic) **Unconstrained (applied anywhere)** -```asciidoc +```plaintext **C**reate+**R**ead+**U**pdate+**D**elete fan__freakin__tastic ``mono``culture @@ -72,7 +72,7 @@ fan__freakin__tastic **Replacements** -```asciidoc +```plaintext A long time ago in a galaxy far, far away... (C) 1976 Arty Artisan I believe I shall--no, actually I won't. @@ -80,7 +80,7 @@ I believe I shall--no, actually I won't. **Macros** -```asciidoc +```plaintext // where c=specialchars, q=quotes, a=attributes, r=replacements, m=macros, p=post_replacements, etc. The European icon:flag[role=blue] is blue & contains pass:[************] arranged in a icon:circle-o[role=yellow]. The pass:c[->] operator is often referred to as the stabby lambda. @@ -93,12 +93,12 @@ stem:[sqrt(4) = 2] **User-defined attributes** -```asciidoc +```plaintext // define attributes in the document header :name: value ``` -```asciidoc +```plaintext :url-gem: https://rubygems.org/gems/asciidoctor You can download and install Asciidoctor {asciidoctor-version} from {url-gem}. @@ -117,7 +117,7 @@ GitLab sets the following environment attributes: ### Links -```asciidoc +```plaintext https://example.org/page[A webpage] link:../path/to/file.txt[A local file] xref:document.adoc[A sibling document] @@ -126,7 +126,7 @@ mailto:hello@example.org[Email to say hello!] ### Anchors -```asciidoc +```plaintext [[idname,reference text]] // or written using normal block attributes as `[#idname,reftext=reference text]` A paragraph (or any block) with an anchor (aka ID) and reftext. @@ -142,7 +142,7 @@ This paragraph has a footnote.footnote:[This is the text of the footnote.] #### Unordered -```asciidoc +```plaintext * level 1 ** level 2 *** level 3 @@ -161,7 +161,7 @@ Attach a block or paragraph to a list item using a list continuation (which you #### Ordered -```asciidoc +```plaintext . Step 1 . Step 2 .. Step 2a @@ -177,14 +177,14 @@ Attach a block or paragraph to a list item using a list continuation (which you #### Checklist -```asciidoc +```plaintext * [x] checked * [ ] not checked ``` #### Callout -```asciidoc +```plaintext // enable callout bubbles by adding `:icons: font` to the document header [,ruby] ---- @@ -195,7 +195,7 @@ puts 'Hello, World!' # <1> #### Description -```asciidoc +```plaintext first term:: description of first term second term:: description of second term @@ -205,7 +205,7 @@ description of second term #### Header -```asciidoc +```plaintext = Document Title Author Name <author@example.org> v1.0, 2019-01-01 @@ -213,7 +213,7 @@ v1.0, 2019-01-01 #### Sections -```asciidoc +```plaintext = Document Title (Level 0) == Level 1 === Level 2 @@ -225,7 +225,7 @@ v1.0, 2019-01-01 #### Includes -```asciidoc +```plaintext include::basics.adoc[] // define -a allow-uri-read to allow content to be read from URI @@ -239,13 +239,13 @@ included, a number that is inclusive of transitive dependencies. ### Blocks -```asciidoc +```plaintext -- open - a general-purpose content wrapper; useful for enclosing content to attach to a list item -- ``` -```asciidoc +```plaintext // recognized types include CAUTION, IMPORTANT, NOTE, TIP, and WARNING // enable admonition icons by setting `:icons: font` in the document header [NOTE] @@ -254,13 +254,13 @@ admonition - a notice for the reader, ranging in severity from a tip to an alert ==== ``` -```asciidoc +```plaintext ==== example - a demonstration of the concept being documented ==== ``` -```asciidoc +```plaintext .Toggle Me [%collapsible] ==== @@ -268,58 +268,58 @@ collapsible - these details are revealed by clicking the title ==== ``` -```asciidoc +```plaintext **** sidebar - auxiliary content that can be read independently of the main content **** ``` -```asciidoc +```plaintext .... literal - an exhibit that features program output .... ``` -```asciidoc +```plaintext ---- listing - an exhibit that features program input, source code, or the contents of a file ---- ``` -```asciidoc +```plaintext [,language] ---- source - a listing that is embellished with (colorized) syntax highlighting ---- ``` -````asciidoc +````plaintext \```language fenced code - a shorthand syntax for the source block \``` ```` -```asciidoc +```plaintext [,attribution,citetitle] ____ quote - a quotation or excerpt; attribution with title of source are optional ____ ``` -```asciidoc +```plaintext [verse,attribution,citetitle] ____ verse - a literary excerpt, often a poem; attribution with title of source are optional ____ ``` -```asciidoc +```plaintext ++++ pass - content passed directly to the output document; often raw HTML ++++ ``` -```asciidoc +```plaintext // activate stem support by adding `:stem:` to the document header [stem] ++++ @@ -327,7 +327,7 @@ x = y^2 ++++ ``` -```asciidoc +```plaintext //// comment - content which is not included in the output document //// @@ -335,7 +335,7 @@ comment - content which is not included in the output document ### Tables -```asciidoc +```plaintext .Table Attributes [cols=>1h;2d,width=50%,frame=topbot] |=== @@ -366,7 +366,7 @@ comment - content which is not included in the output document ### Multimedia -```asciidoc +```plaintext image::screenshot.png[block image,800,450] Press image:reload.svg[reload,16,opts=interactive] to reload the page. @@ -380,12 +380,12 @@ video::300817511[vimeo] ### Breaks -```asciidoc +```plaintext // thematic break (aka horizontal rule) --- ``` -```asciidoc +```plaintext // page break <<< ``` diff --git a/doc/user/clusters/applications.md b/doc/user/clusters/applications.md index 86624d12bcf..507ba25850d 100644 --- a/doc/user/clusters/applications.md +++ b/doc/user/clusters/applications.md @@ -32,8 +32,6 @@ To see a list of available applications to install. For a: - [Group-level cluster](../group/clusters/index.md), navigate to your group's **{cloud-gear}** **Kubernetes** page. -Install Helm first as it's used to install other applications. - NOTE: **Note:** As of GitLab 11.6, Helm will be upgraded to the latest version supported by GitLab before installing any of the applications. @@ -71,17 +69,47 @@ can lead to confusion during deployments. > - Introduced in GitLab 10.2 for project-level clusters. > - Introduced in GitLab 11.6 for group-level clusters. +> - A local Tiller option was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/209736) in GitLab 13.2 behind a feature flag, enabled by default. +> - The feature flag for local Tiller is enabled on GitLab.com. [Helm](https://helm.sh/docs/) is a package manager for Kubernetes and is -required to install all the other applications. It is installed in its -own pod inside the cluster which can run the `helm` CLI in a safe -environment. +used to install the GitLab-managed apps. GitLab runs each `helm` command +in a pod within the `gitlab-managed-apps` namespace inside the cluster. + +As of GitLab 13.2, the integration uses a local +[Tiller](https://v2.helm.sh/docs/glossary/#tiller) by default. When using a +local Tiller, the Helm application does not need to be installed and will not +be shown in the list of applications. NOTE: **Note:** -Installing Helm as a GitLab-managed App behind a proxy is not supported, -but a [workaround](../../topics/autodevops/index.md#installing-helm-behind-a-proxy) +GitLab's Helm integration does not support installing applications behind a proxy, +but a [workaround](../../topics/autodevops/index.md#install-applications-behind-a-proxy) is available. +### Enable or disable local Tiller **(CORE ONLY)** + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/209736) in GitLab 13.2 +> - The option to disable local Tiller is [planned for removal](https://gitlab.com/gitlab-org/gitlab/-/issues/209736) in GitLab 13.3 + +Local Tiller is under development, but is ready for production use. It is +deployed behind a feature flag that is **enabled by default**. +[GitLab administrators with access to the GitLab Rails console](../../administration/feature_flags.md) +can enable it for your instance. + +To enable it: + +```ruby +# Instance-wide +Feature.enable(:managed_apps_local_tiller) +``` + +To disable it: + +```ruby +# Instance-wide +Feature.disable(:managed_apps_local_tiller) +``` + ### cert-manager > Introduced in GitLab 11.6 for project- and group-level clusters. @@ -609,6 +637,7 @@ Supported applications: - [Sentry](#install-sentry-using-gitlab-cicd) - [GitLab Runner](#install-gitlab-runner-using-gitlab-cicd) - [Cilium](#install-cilium-using-gitlab-cicd) +- [Falco](#install-falco-using-gitlab-cicd) - [Vault](#install-vault-using-gitlab-cicd) - [JupyterHub](#install-jupyterhub-using-gitlab-cicd) - [Elastic Stack](#install-elastic-stack-using-gitlab-cicd) @@ -634,6 +663,11 @@ To install applications using GitLab CI/CD: - template: Managed-Cluster-Applications.gitlab-ci.yml ``` + NOTE: **Note:** + The job provided by this template connects to the cluster using tools provided + in a custom Docker image. It requires that you have a runner registered with the Docker, + Kubernetes, or Docker Machine executor. + 1. Add a `.gitlab/managed-apps/config.yaml` file to define which applications you would like to install. Define the `installed` key as `true` to install the application and `false` to uninstall the @@ -683,6 +717,10 @@ management project. Refer to the [chart](https://github.com/helm/charts/tree/master/stable/nginx-ingress) for the available configuration options. +NOTE: **Note:** +Support for installing the Ingress managed application is provided by the GitLab Configure group. +If you run into unknown issues, please [open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new) and ping at least 2 people from the [Configure group](https://about.gitlab.com/handbook/product/categories/#configure-group). + ### Install cert-manager using GitLab CI/CD cert-manager is installed using GitLab CI/CD by defining configuration in @@ -720,6 +758,10 @@ management project. Refer to the [chart](https://hub.helm.sh/charts/jetstack/cert-manager) for the available configuration options. +NOTE: **Note:** +Support for installing the Cert Manager managed application is provided by the GitLab Configure group. +If you run into unknown issues, please [open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new) and ping at least 2 people from the [Configure group](https://about.gitlab.com/handbook/product/categories/#configure-group). + ### Install Sentry using GitLab CI/CD NOTE: **Note:** @@ -781,6 +823,10 @@ postgresql: postgresqlPassword: example-postgresql-password ``` +NOTE: **Note:** +Support for installing the Sentry managed application is provided by the GitLab Health group. +If you run into unknown issues, please [open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new) and ping at least 2 people from the [Health group](https://about.gitlab.com/handbook/product/product-categories/#health-group). + ### Install PostHog using GitLab CI/CD [PostHog](https://www.posthog.com) 🦔 is a developer-friendly, open-source product analytics platform. @@ -852,6 +898,10 @@ project. Refer to the [Configuration section of the Prometheus chart's README](https://github.com/helm/charts/tree/master/stable/prometheus#configuration) for the available configuration options. +NOTE: **Note:** +Support for installing the Prometheus managed application is provided by the GitLab APM group. +If you run into unknown issues, please [open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new) and ping at least 2 people from the [APM group](https://about.gitlab.com/handbook/product/product-categories/#apm-group). + ### Install GitLab Runner using GitLab CI/CD GitLab Runner is installed using GitLab CI/CD by defining configuration in @@ -883,6 +933,10 @@ management project. Refer to the [chart](https://gitlab.com/gitlab-org/charts/gitlab-runner) for the available configuration options. +NOTE: **Note:** +Support for installing the Runner managed application is provided by the GitLab Runner group. +If you run into unknown issues, please [open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new) and ping at least 2 people from the [Runner group](https://about.gitlab.com/handbook/product/product-categories/#runner-group). + ### Install Cilium using GitLab CI/CD > [Introduced](https://gitlab.com/gitlab-org/cluster-integration/cluster-applications/-/merge_requests/22) in GitLab 12.8. @@ -909,8 +963,8 @@ a corresponding cluster type. The default value is blank. You can check the recommended variables for each cluster type in the official documentation: -- [Google GKE](https://cilium.readthedocs.io/en/stable/gettingstarted/k8s-install-gke/#prepare-deploy-cilium) -- [AWS EKS](https://cilium.readthedocs.io/en/stable/gettingstarted/k8s-install-eks/#prepare-deploy-cilium) +- [Google GKE](https://docs.cilium.io/en/stable/gettingstarted/k8s-install-gke/#deploy-cilium) +- [AWS EKS](https://docs.cilium.io/en/stable/gettingstarted/k8s-install-eks/#deploy-cilium) You can customize Cilium's Helm variables by defining the `.gitlab/managed-apps/cilium/values.yaml` file in your cluster @@ -920,9 +974,9 @@ for the available configuration options. CAUTION: **Caution:** Installation and removal of the Cilium requires a **manual** -[restart](https://cilium.readthedocs.io/en/stable/gettingstarted/k8s-install-gke/#restart-remaining-pods) +[restart](https://docs.cilium.io/en/stable/gettingstarted/k8s-install-gke/#restart-unmanaged-pods) of all affected pods in all namespaces to ensure that they are -[managed](https://cilium.readthedocs.io/en/stable/troubleshooting/#ensure-pod-is-managed-by-cilium) +[managed](https://docs.cilium.io/en/stable/troubleshooting/#ensure-pod-is-managed-by-cilium) by the correct networking plugin. NOTE: **Note:** @@ -930,23 +984,20 @@ Major upgrades might require additional setup steps, please consult the official [upgrade guide](https://docs.cilium.io/en/stable/install/upgrade/) for more information. -By default, Cilium will drop all disallowed packets upon policy -deployment. The audit mode is scheduled for release in -[Cilium 1.8](https://github.com/cilium/cilium/pull/9970). In the audit -mode, disallowed packets will not be dropped, and audit -notifications will be generated instead. GitLab provides alternative Docker -images for Cilium with the audit patch included. You can switch to the -custom build and enable the audit mode by adding the following to +By default, Cilium's [audit +mode](https://docs.cilium.io/en/v1.8/gettingstarted/policy-creation/?highlight=policy-audit#enable-policy-audit-mode) +is enabled. In audit mode, Cilium doesn't drop disallowed packets. You +can use `policy-verdict` log to observe policy-related decisions. You +can disable audit mode by adding the following to `.gitlab/managed-apps/cilium/values.yaml`: ```yaml -global: - registry: registry.gitlab.com/gitlab-org/defend/cilium - policyAuditMode: true +config: + policyAuditMode: false agent: monitor: - eventTypes: ["drop", "audit"] + eventTypes: ["drop"] ``` The Cilium monitor log for traffic is logged out by the @@ -968,24 +1019,121 @@ The [Hubble](https://github.com/cilium/hubble) monitoring daemon is enabled by default and it's set to collect per namespace flow metrics. This metrics are accessible on the [Threat Monitoring](../application_security/threat_monitoring/index.md) dashboard. You can disable Hubble by adding the following to -`.gitlab/managed-apps/config.yaml`: +`.gitlab/managed-apps/cilium/values.yaml`: ```yaml -cilium: - installed: true +global: hubble: - installed: false + enabled: false ``` You can also adjust Helm values for Hubble via -`.gitlab/managed-apps/cilium/hubble-values.yaml`: +`.gitlab/managed-apps/cilium/values.yaml`: ```yaml -metrics: - enabled: - - 'flow:sourceContext=namespace;destinationContext=namespace' +global: + hubble: + enabled: true + metrics: + enabled: + - 'flow:sourceContext=namespace;destinationContext=namespace' +``` + +NOTE: **Note:** +Support for installing the Cilium managed application is provided by the GitLab Container Security group. +If you run into unknown issues, please [open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new) and ping at least 2 people from the [Container Security group](https://about.gitlab.com/handbook/product/product-categories/#container-security-group). + +### Install Falco using GitLab CI/CD + +> [Introduced](https://gitlab.com/gitlab-org/cluster-integration/cluster-applications/-/merge_requests/91) in GitLab 13.1. + +GitLab Container Host Security Monitoring uses [Falco](https://falco.org/) +as a runtime security tool that listens to the Linux kernel using eBPF. Falco parses system calls +and asserts the stream against a configurable rules engine in real-time. For more information, see +[Falco's Documentation](https://falco.org/docs/). + +You can enable Falco in the +`.gitlab/managed-apps/config.yaml` file: + +```yaml +falco: + installed: true +``` + +You can customize Falco's Helm variables by defining the +`.gitlab/managed-apps/falco/values.yaml` file in your cluster +management project. Refer to the +[Falco chart](https://github.com/falcosecurity/charts/tree/master/falco) +for the available configuration options. + +CAUTION: **Caution:** +By default eBPF support is enabled and Falco will use an [eBPF probe](https://falco.org/docs/event-sources/drivers/#using-the-ebpf-probe) to pass system calls to userspace. +If your cluster doesn't support this, you can configure it to use Falco kernel module instead by adding the following to `.gitlab/managed-apps/falco/values.yaml`: + +```yaml +ebpf: + enabled: false ``` +In rare cases where automatic probe installation on your cluster isn't possible and the kernel/probe +isn't precompiled, you may need to manually prepare the kernel module or eBPF probe with +[driverkit](https://github.com/falcosecurity/driverkit#against-a-kubernetes-cluster) +and install it on each cluster node. + +By default, Falco is deployed with a limited set of rules. To add more rules, add the following to +`.gitlab/managed-apps/falco/values.yaml` (you can get examples from +[Cloud Native Security Hub](https://securityhub.dev/)): + +```yaml +customRules: + file-integrity.yaml: |- + - rule: Detect New File + desc: detect new file created + condition: > + evt.type = chmod or evt.type = fchmod + output: > + File below a known directory opened for writing (user=%user.name + command=%proc.cmdline file=%fd.name parent=%proc.pname pcmdline=%proc.pcmdline gparent=%proc.aname[2]) + priority: ERROR + tags: [filesystem] + - rule: Detect New Directory + desc: detect new directory created + condition: > + mkdir + output: > + File below a known directory opened for writing (user=%user.name + command=%proc.cmdline file=%fd.name parent=%proc.pname pcmdline=%proc.pcmdline gparent=%proc.aname[2]) + priority: ERROR + tags: [filesystem] +``` + +By default, Falco only outputs security events to logs as JSON objects. To set it to output to an +[external API](https://falco.org/docs/alerts/#https-output-send-alerts-to-an-https-end-point) +or [application](https://falco.org/docs/alerts/#program-output), +add the following to `.gitlab/managed-apps/falco/values.yaml`: + +```yaml +falco: + programOutput: + enabled: true + keepAlive: false + program: mail -s "Falco Notification" someone@example.com + + httpOutput: + enabled: true + url: http://some.url +``` + +You can check these logs with the following command: + +```shell +kubectl logs -l app=falco -n gitlab-managed-apps +``` + +NOTE: **Note:** +Support for installing the Falco managed application is provided by the GitLab Container Security group. +If you run into unknown issues, please [open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new) and ping at least 2 people from the [Container Security group](https://about.gitlab.com/handbook/product/product-categories/#container-security-group). + ### Install Vault using GitLab CI/CD > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9982) in GitLab 12.9. @@ -1035,7 +1183,7 @@ below are examples and should be replaced with settings specific to your environ ui: enabled: true server: - # Disable the built in data storage volume as it's not safe for Hight Availablity mode + # Disable the built in data storage volume as it's not safe for Hight Availability mode dataStorage: enabled: false # Enable High Availability Mode @@ -1075,6 +1223,10 @@ kubectl -n gitlab-managed-apps exec -it vault-0 sh This should give you your unseal keys and initial root token. Make sure to note these down and keep these safe as you will need them to unseal the Vault throughout its lifecycle. +NOTE: **Note:** +Support for installing the Vault managed application is provided by the GitLab Release Management group. +If you run into unknown issues, please [open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new) and ping at least 2 people from the [Release Management group](https://about.gitlab.com/handbook/product/product-categories/#release-management-group). + ### Install JupyterHub using GitLab CI/CD > [Introduced](https://gitlab.com/gitlab-org/cluster-integration/cluster-applications/-/merge_requests/40) in GitLab 12.8. @@ -1124,6 +1276,10 @@ Refer to the [chart reference](https://zero-to-jupyterhub.readthedocs.io/en/stable/reference/reference.html) for the available configuration options. +NOTE: **Note:** +Support for installing the JupyterHub managed application is provided by the GitLab Configure group. +If you run into unknown issues, please [open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new) and ping at least 2 people from the [Configure group](https://about.gitlab.com/handbook/product/categories/#configure-group). + ### Install Elastic Stack using GitLab CI/CD > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/25138) in GitLab 12.8. @@ -1151,6 +1307,10 @@ available configuration options. NOTE: **Note:** In this alpha implementation of installing Elastic Stack through CI, reading the environment logs through Elasticsearch is unsupported. This is supported if [installed via the UI](#elastic-stack). +NOTE: **Note:** +Support for installing the Elastic Stack managed application is provided by the GitLab APM group. +If you run into unknown issues, please [open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new) and ping at least 2 people from the [APM group](https://about.gitlab.com/handbook/product/product-categories/#apm-group). + ### Install Crossplane using GitLab CI/CD > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/35675) in GitLab 12.9. @@ -1177,6 +1337,10 @@ management project. Refer to the [chart](https://github.com/crossplane/crossplane/tree/master/cluster/charts/crossplane#configuration) for the available configuration options. Note that this link points to the documentation for the current development release, which may differ from the version you have installed. +NOTE: **Note:** +Support for the Crossplane managed application is provided by the Crossplane team. +If you run into issues, please [open a support ticket](https://github.com/crossplane/crossplane/issues/new/choose) directly. + ### Install Fluentd using GitLab CI/CD > [Introduced](https://gitlab.com/gitlab-org/cluster-integration/cluster-applications/-/merge_requests/76) in GitLab 12.10. @@ -1201,6 +1365,10 @@ The configuration chart link points to the current development release, which may differ from the version you have installed. To ensure compatibility, switch to the specific branch or tag you are using. +NOTE: **Note:** +Support for installing the Fluentd managed application is provided by the GitLab Container Security group. +If you run into unknown issues, please [open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new) and ping at least 2 people from the [Container Security group](https://about.gitlab.com/handbook/product/product-categories/#container-security-group). + ### Install Knative using GitLab CI/CD To install Knative, define the `.gitlab/managed-apps/config.yaml` file @@ -1223,6 +1391,10 @@ domain: 'my.wildcard.A.record.dns' If you plan to use GitLab Serverless capabilities, be sure to set an A record wildcard domain on your custom configuration. +NOTE: **Note:** +Support for installing the Knative managed application is provided by the GitLab Configure group. +If you run into unknown issues, please [open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new) and ping at least 2 people from the [Configure group](https://about.gitlab.com/handbook/product/categories/#configure-group). + #### Knative Metrics GitLab provides [Invocation Metrics](../project/clusters/serverless/index.md#invocation-metrics) for your functions. To collect these metrics, you must have: @@ -1243,6 +1415,92 @@ by running the following command: kubectl delete -f https://gitlab.com/gitlab-org/cluster-integration/cluster-applications/-/raw/02c8231e30ef5b6725e6ba368bc63863ceb3c07d/src/default-data/knative/istio-metrics.yaml ``` +### Install AppArmor using GitLab CI/CD + +> [Introduced](https://gitlab.com/gitlab-org/cluster-integration/cluster-applications/-/merge_requests/100) in GitLab 13.1. + +To install AppArmor into the `gitlab-managed-apps` namespace of your cluster using GitLab CI/CD, define the following configuration in `.gitlab/managed-apps/config.yaml`: + +```yaml +apparmor: + installed: true +``` + +You can define one or more AppArmor profiles by adding them into `.gitlab/managed-apps/apparmor/values.yaml` as the following: + +```yaml +profiles: + profile-one: |- + profile profile-one { + file, + } +``` + +Refer to the [AppArmor chart](https://gitlab.com/gitlab-org/charts/apparmor) for more information on this chart. + +#### Using AppArmor profiles in your deployments + +After installing AppAmor, you can use profiles by adding Pod Annotations. If you're using Auto +DevOps, you can [customize `auto-deploy-values.yaml`](../../topics/autodevops/customize.md#customize-values-for-helm-chart) +to annotate your pods. Although it's helpful to be aware of the [list of custom attributes](https://gitlab.com/gitlab-org/charts/auto-deploy-app#gitlabs-auto-deploy-helm-chart), you're only required to set +`podAnnotations` as follows: + +```yaml +podAnnotations: + container.apparmor.security.beta.kubernetes.io/auto-deploy-app: localhost/profile-one +``` + +The only information to be changed here is the profile name which is `profile-one` in this example. Refer to the [AppArmor tutorial](https://kubernetes.io/docs/tutorials/clusters/apparmor/#securing-a-pod) for more information on how AppArmor is integrated in Kubernetes. + +#### Using PodSecurityPolicy in your deployments + +NOTE: **Note:** +To enable AppArmor annotations on a Pod Security Policy you must first +load the correspondingAppArmor profile. + +[Pod Security Policies](https://kubernetes.io/docs/concepts/policy/pod-security-policy/)are +resources at the cluster level that control security-related +properties of deployed pods. You can use such a policy to enable +loaded AppArmor profiles and apply necessary pod restrictions across a +cluster. You can deploy a new policy by adding the following +to`.gitlab/managed-apps/apparmor/values.yaml`: + +```yaml +securityPolicies: + example: + defaultProfile: profile-one + allowedProfiles: + - profile-one + - profile-two + spec: + privileged: false + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + runAsUser: + rule: RunAsAny + fsGroup: + rule: RunAsAny + volumes: + - '*' +``` + +This example creates a single policy named `example` with the provided +specification, and enables [AppArmor +annotations](https://kubernetes.io/docs/tutorials/clusters/apparmor/#podsecuritypolicy-annotations)on +it. + +NOTE: **Note:** +Support for installing the AppArmor managed application is provided by the GitLab Container Security group. +If you run into unknown issues, please [open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new) and ping at least 2 people from the [Container Security group](https://about.gitlab.com/handbook/product/product-categories/#container-security-group). + +## Browse applications logs + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/36769) in GitLab 13.2. + +Logs produced by pods running **GitLab Managed Apps** can be browsed using [**Log Explorer**](../project/clusters/kubernetes_pod_logs.md). + ## Upgrading applications > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/24789) in GitLab 11.8. @@ -1342,3 +1600,16 @@ The number and size of nodes might not have enough IP addresses to run or instal For reference, all the AWS instance IP limits are found [in this AWS repository on GitHub](https://github.com/aws/amazon-vpc-cni-k8s/blob/master/pkg/awsutils/vpc_ip_resource_limit.go) (search for `InstanceENIsAvailable`). + +### Unable to install Prometheus + +Installing Prometheus is failing with the following error: + +```shell +# kubectl -n gitlab-managed-apps logs install-prometheus +... +Error: Could not get apiVersions from Kubernetes: unable to retrieve the complete list of server APIs: admission.certmanager.k8s.io/v1beta1: the server is currently unable to handle the request +``` + +This is a bug that was introduced in Helm `2.15` and fixed in `3.0.2`. As a workaround, you'll need +to make sure that [`cert-manager`](#cert-manager) is installed successfully prior to installing Prometheus. diff --git a/doc/user/clusters/crossplane.md b/doc/user/clusters/crossplane.md index 3a430ad55bd..b30ebc57338 100644 --- a/doc/user/clusters/crossplane.md +++ b/doc/user/clusters/crossplane.md @@ -6,17 +6,15 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Crossplane configuration -Once Crossplane [is installed](applications.md#crossplane), it must be configured for -use. - +After [installing](applications.md#crossplane) Crossplane, you must configure it for use. The process of configuring Crossplane includes: -1. Configuring RBAC permissions. -1. Configuring Crossplane with a cloud provider. -1. Configure managed service access. -1. Setting up Resource classes. -1. Using Auto DevOps configuration options. -1. Connect to the PostgreSQL instance. +1. [Configure RBAC permissions](#configure-rbac-permissions). +1. [Configure Crossplane with a cloud provider](#configure-crossplane-with-a-cloud-provider). +1. [Configure managed service access](#configure-managed-service-access). +1. [Set up Resource classes](#setting-up-resource-classes). +1. Use [Auto DevOps configuration options](#auto-devops-configuration-options). +1. [Connect to the PostgreSQL instance](#connect-to-the-postgresql-instance). To allow Crossplane to provision cloud services such as PostgreSQL, the cloud provider stack must be configured with a user account. For example: @@ -24,14 +22,13 @@ stack must be configured with a user account. For example: - A service account for GCP. - An IAM user for AWS. -Important notes: +Some important notes: -- This guide uses GCP as an example. However, the process for AWS and Azure will be -similar. -- Crossplane requires the Kubernetes cluster to be VPC native with Alias IPs enabled so -that the IP address of the pods are routable within the GCP network. +- This guide uses GCP as an example, but the processes for AWS and Azure are similar. +- Crossplane requires the Kubernetes cluster to be VPC native with Alias IPs enabled, + so the IP addresses of the pods can be routed within the GCP network. -First, we need to declare some environment variables with configuration that will be used throughout this guide: +First, declare some environment variables with configuration for use in this guide: ```shell export PROJECT_ID=crossplane-playground # the GCP project where all resources reside. @@ -41,228 +38,223 @@ export REGION=us-central1 # the GCP region where the GKE cluster is provisioned. ## Configure RBAC permissions -- For GitLab-managed clusters, RBAC is configured automatically. - -- For non-GitLab managed clusters, ensure that the service account for the token provided can manage resources in the `database.crossplane.io` API group: - - 1. Save the following YAML as `crossplane-database-role.yaml`: - - ```yaml - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRole - metadata: - name: crossplane-database-role - labels: - rbac.authorization.k8s.io/aggregate-to-edit: "true" - rules: - - apiGroups: - - database.crossplane.io - resources: - - postgresqlinstances - verbs: - - get - - list - - create - - update - - delete - - patch - - watch - ``` - - 1. Apply the cluster role to the cluster: - - ```shell - kubectl apply -f crossplane-database-role.yaml - ``` +For GitLab-managed clusters, role-based access control (RBAC) is configured automatically. + +For non-GitLab managed clusters, ensure that the service account for the token +provided can manage resources in the `database.crossplane.io` API group: + +1. Save the following YAML as `crossplane-database-role.yaml`: + + ```yaml + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: crossplane-database-role + labels: + rbac.authorization.k8s.io/aggregate-to-edit: "true" + rules: + - apiGroups: + - database.crossplane.io + resources: + - postgresqlinstances + verbs: + - get + - list + - create + - update + - delete + - patch + - watch + ``` + +1. Apply the cluster role to the cluster: + + ```shell + kubectl apply -f crossplane-database-role.yaml + ``` ## Configure Crossplane with a cloud provider See [Configure Your Cloud Provider Account](https://crossplane.github.io/docs/v0.4/cloud-providers.html) to configure the installed cloud provider stack with a user account. -Note that the Secret and the Provider resource referencing the Secret needs to be +NOTE: **Note:** +The Secret, and the Provider resource referencing the Secret, must be applied to the `gitlab-managed-apps` namespace in the guide. Make sure you change that while following the process. -[Configure Providers](https://crossplane.github.io/docs/v0.4/cloud-providers.html) - ## Configure Managed Service Access -We need to configure connectivity between the PostgreSQL database and the GKE cluster. -This can done by either: +Next, configure connectivity between the PostgreSQL database and the GKE cluster +by either: - Using Crossplane as demonstrated below. - Directly in the GCP console by -[configuring private services access](https://cloud.google.com/vpc/docs/configure-private-services-access). -Create a GlobalAddress and Connection resources: - -```shell -cat > network.yaml <<EOF ---- -# gitlab-ad-globaladdress defines the IP range that will be allocated for cloud services connecting to the instances in the given Network. - -apiVersion: compute.gcp.crossplane.io/v1alpha3 -kind: GlobalAddress -metadata: - name: gitlab-ad-globaladdress -spec: - providerRef: - name: gcp-provider - reclaimPolicy: Delete - name: gitlab-ad-globaladdress - purpose: VPC_PEERING - addressType: INTERNAL - prefixLength: 16 - network: projects/$PROJECT_ID/global/networks/$NETWORK_NAME ---- -# gitlab-ad-connection is what allows cloud services to use the allocated GlobalAddress for communication. Behind -# the scenes, it creates a VPC peering to the network that those service instances actually live. - -apiVersion: servicenetworking.gcp.crossplane.io/v1alpha3 -kind: Connection -metadata: - name: gitlab-ad-connection -spec: - providerRef: - name: gcp-provider - reclaimPolicy: Delete - parent: services/servicenetworking.googleapis.com - network: projects/$PROJECT_ID/global/networks/$NETWORK_NAME - reservedPeeringRangeRefs: - - name: gitlab-ad-globaladdress -EOF -``` - -Apply the settings specified in the file with the following command: - -```shell -kubectl apply -f network.yaml -``` - -You can verify creation of the network resources with the following commands. -Verify that the status of both of these resources is ready and is synced. - -```shell -kubectl describe connection.servicenetworking.gcp.crossplane.io gitlab-ad-connection -kubectl describe globaladdress.compute.gcp.crossplane.io gitlab-ad-globaladdress -``` + [configuring private services access](https://cloud.google.com/vpc/docs/configure-private-services-access). + +1. Run the following command, which creates a `network.yaml` file, and configures + `GlobalAddress` and connection resources: + + ```plaintext + cat > network.yaml <<EOF + --- + # gitlab-ad-globaladdress defines the IP range that will be allocated + # for cloud services connecting to the instances in the given Network. + + apiVersion: compute.gcp.crossplane.io/v1alpha3 + kind: GlobalAddress + metadata: + name: gitlab-ad-globaladdress + spec: + providerRef: + name: gcp-provider + reclaimPolicy: Delete + name: gitlab-ad-globaladdress + purpose: VPC_PEERING + addressType: INTERNAL + prefixLength: 16 + network: projects/$PROJECT_ID/global/networks/$NETWORK_NAME + --- + # gitlab-ad-connection is what allows cloud services to use the allocated + # GlobalAddress for communication. Behind the scenes, it creates a VPC peering + # to the network that those service instances actually live. + + apiVersion: servicenetworking.gcp.crossplane.io/v1alpha3 + kind: Connection + metadata: + name: gitlab-ad-connection + spec: + providerRef: + name: gcp-provider + reclaimPolicy: Delete + parent: services/servicenetworking.googleapis.com + network: projects/$PROJECT_ID/global/networks/$NETWORK_NAME + reservedPeeringRangeRefs: + - name: gitlab-ad-globaladdress + EOF + ``` + +1. Apply the settings specified in the file with the following command: + + ```shell + kubectl apply -f network.yaml + ``` + +1. Verify the creation of the network resources, and that both resources are ready and synced. + + ```shell + kubectl describe connection.servicenetworking.gcp.crossplane.io gitlab-ad-connection + kubectl describe globaladdress.compute.gcp.crossplane.io gitlab-ad-globaladdress + ``` ## Setting up Resource classes -Resource classes are a way of defining a configuration for the required managed service. We will define the PostgreSQL Resource class - -- Define a `gcp-postgres-standard.yaml` resource class which contains - -1. A default CloudSQLInstanceClass. -1. A CloudSQLInstanceClass with labels. - -```shell -cat > gcp-postgres-standard.yaml <<EOF -apiVersion: database.gcp.crossplane.io/v1beta1 -kind: CloudSQLInstanceClass -metadata: - name: cloudsqlinstancepostgresql-standard - labels: - gitlab-ad-demo: "true" -specTemplate: - writeConnectionSecretsToNamespace: gitlab-managed-apps - forProvider: - databaseVersion: POSTGRES_11_7 - region: $REGION - settings: - tier: db-custom-1-3840 - dataDiskType: PD_SSD - dataDiskSizeGb: 10 - ipConfiguration: - privateNetwork: projects/$PROJECT_ID/global/networks/$NETWORK_NAME - # this should match the name of the provider created in the above step - providerRef: - name: gcp-provider - reclaimPolicy: Delete ---- -apiVersion: database.gcp.crossplane.io/v1beta1 -kind: CloudSQLInstanceClass -metadata: - name: cloudsqlinstancepostgresql-standard-default - annotations: - resourceclass.crossplane.io/is-default-class: "true" -specTemplate: - writeConnectionSecretsToNamespace: gitlab-managed-apps - forProvider: - databaseVersion: POSTGRES_11_7 - region: $REGION - settings: - tier: db-custom-1-3840 - dataDiskType: PD_SSD - dataDiskSizeGb: 10 - ipConfiguration: - privateNetwork: projects/$PROJECT_ID/global/networks/$NETWORK_NAME - # this should match the name of the provider created in the above step - providerRef: - name: gcp-provider - reclaimPolicy: Delete -EOF -``` - -Apply the resource class configuration with the following command: - -```shell -kubectl apply -f gcp-postgres-standard.yaml -``` - -Verify creation of the Resource class with the following command: - -```shell -kubectl get cloudsqlinstanceclasses -``` - -The Resource Classes allow you to define classes of service for a managed service. We could create another `CloudSQLInstanceClass` which requests for a larger or a faster disk. It could also request for a specific version of the database. +Use resource classes to define a configuration for the required managed service. +This example defines the PostgreSQL Resource class: + +1. Run the following command, which define a `gcp-postgres-standard.yaml` resource + class containing a default `CloudSQLInstanceClass` with labels: + + ```plaintext + cat > gcp-postgres-standard.yaml <<EOF + apiVersion: database.gcp.crossplane.io/v1beta1 + kind: CloudSQLInstanceClass + metadata: + name: cloudsqlinstancepostgresql-standard + labels: + gitlab-ad-demo: "true" + specTemplate: + writeConnectionSecretsToNamespace: gitlab-managed-apps + forProvider: + databaseVersion: POSTGRES_11_7 + region: $REGION + settings: + tier: db-custom-1-3840 + dataDiskType: PD_SSD + dataDiskSizeGb: 10 + ipConfiguration: + privateNetwork: projects/$PROJECT_ID/global/networks/$NETWORK_NAME + # this should match the name of the provider created in the above step + providerRef: + name: gcp-provider + reclaimPolicy: Delete + --- + apiVersion: database.gcp.crossplane.io/v1beta1 + kind: CloudSQLInstanceClass + metadata: + name: cloudsqlinstancepostgresql-standard-default + annotations: + resourceclass.crossplane.io/is-default-class: "true" + specTemplate: + writeConnectionSecretsToNamespace: gitlab-managed-apps + forProvider: + databaseVersion: POSTGRES_11_7 + region: $REGION + settings: + tier: db-custom-1-3840 + dataDiskType: PD_SSD + dataDiskSizeGb: 10 + ipConfiguration: + privateNetwork: projects/$PROJECT_ID/global/networks/$NETWORK_NAME + # this should match the name of the provider created in the above step + providerRef: + name: gcp-provider + reclaimPolicy: Delete + EOF + ``` + +1. Apply the resource class configuration with the following command: + + ```shell + kubectl apply -f gcp-postgres-standard.yaml + ``` + +1. Verify creation of the Resource class with the following command: + + ```shell + kubectl get cloudsqlinstanceclasses + ``` + +The Resource Classes allow you to define classes of service for a managed service. +We could create another `CloudSQLInstanceClass` which requests for a larger or a +faster disk. It could also request for a specific version of the database. ## Auto DevOps Configuration Options -The Auto DevOps pipeline can be run with the following options: - -The Environment variables, `AUTO_DEVOPS_POSTGRES_MANAGED` and `AUTO_DEVOPS_POSTGRES_MANAGED_CLASS_SELECTOR` need to be set to provision PostgreSQL using Crossplane - -Alternatively, the following options can be overridden from the values for the Helm chart. - -- `postgres.managed` set to true which will select a default resource class. - The resource class needs to be marked with the annotation - `resourceclass.crossplane.io/is-default-class: "true"`. The CloudSQLInstanceClass - `cloudsqlinstancepostgresql-standard-default` will be used to satisfy the claim. - -- `postgres.managed` set to `true` with `postgres.managedClassSelector` - providing the resource class to choose based on labels. In this case, the - value of `postgres.managedClassSelector.matchLabels.gitlab-ad-demo="true"` - will select the CloudSQLInstance class `cloudsqlinstancepostgresql-standard` - to satisfy the claim request. +You can run the Auto DevOps pipeline with either of the following options: + +- Setting the Environment variables `AUTO_DEVOPS_POSTGRES_MANAGED` and + `AUTO_DEVOPS_POSTGRES_MANAGED_CLASS_SELECTOR` to provision PostgreSQL using Crossplane. +- Overriding values for the Helm chart: + - Set `postgres.managed` to `true`, which selects a default resource class. + Mark the resource class with the annotation + `resourceclass.crossplane.io/is-default-class: "true"`. The CloudSQLInstanceClass + `cloudsqlinstancepostgresql-standard-default` is used to satisfy the claim. + - Set `postgres.managed` to `true` with `postgres.managedClassSelector` + providing the resource class to choose, based on labels. In this case, the + value of `postgres.managedClassSelector.matchLabels.gitlab-ad-demo="true"` + selects the CloudSQLInstance class `cloudsqlinstancepostgresql-standard` + to satisfy the claim request. The Auto DevOps pipeline should provision a PostgresqlInstance when it runs successfully. -Verify creation of the PostgreSQL Instance. +To verify the PostgreSQL instance was created, run this command. When the `STATUS` +field of the PostgresqlInstance changes to `BOUND`, it's successfully provisioned: ```shell -kubectl get postgresqlinstance -``` +$ kubectl get postgresqlinstance -Sample Output: The `STATUS` field of the PostgresqlInstance transitions to `BOUND` when it is successfully provisioned. - -```plaintext NAME STATUS CLASS-KIND CLASS-NAME RESOURCE-KIND RESOURCE-NAME AGE staging-test8 Bound CloudSQLInstanceClass cloudsqlinstancepostgresql-standard CloudSQLInstance xp-ad-demo-24-staging-staging-test8-jj55c 9m ``` -The endpoint of the PostgreSQL instance, and the user credentials, are present in a secret called `app-postgres` within the same project namespace. - -Verify the secret with the database information is created with the following command: +The endpoint of the PostgreSQL instance, and the user credentials, are present in +a secret called `app-postgres` within the same project namespace. You can verify the +secret with the following command: ```shell -kubectl describe secret app-postgres -``` - -Sample Output: +$ kubectl describe secret app-postgres -```plaintext Name: app-postgres Namespace: xp-ad-demo-24-staging Labels: <none> diff --git a/doc/user/clusters/management_project.md b/doc/user/clusters/management_project.md index c8755af29a3..892d2bce184 100644 --- a/doc/user/clusters/management_project.md +++ b/doc/user/clusters/management_project.md @@ -97,7 +97,7 @@ Development, Staging, and Production cluster respectively. ```yaml stages: -- deploy + - deploy configure development cluster: stage: deploy diff --git a/doc/user/compliance/compliance_dashboard/img/compliance_dashboard_v12_10.png b/doc/user/compliance/compliance_dashboard/img/compliance_dashboard_v12_10.png Binary files differdeleted file mode 100644 index 466552f746e..00000000000 --- a/doc/user/compliance/compliance_dashboard/img/compliance_dashboard_v12_10.png +++ /dev/null diff --git a/doc/user/compliance/compliance_dashboard/img/compliance_dashboard_v13_2.png b/doc/user/compliance/compliance_dashboard/img/compliance_dashboard_v13_2.png Binary files differnew file mode 100644 index 00000000000..e1edfcdd024 --- /dev/null +++ b/doc/user/compliance/compliance_dashboard/img/compliance_dashboard_v13_2.png diff --git a/doc/user/compliance/compliance_dashboard/index.md b/doc/user/compliance/compliance_dashboard/index.md index 08a26a45b17..e7db73e25d9 100644 --- a/doc/user/compliance/compliance_dashboard/index.md +++ b/doc/user/compliance/compliance_dashboard/index.md @@ -17,7 +17,7 @@ for merging into production. To access the Compliance Dashboard for a group, navigate to **{shield}** **Security & Compliance > Compliance** on the group's menu. - + ## Use cases @@ -27,6 +27,7 @@ You can use the dashboard to: - Get an overview of the latest Merge Request for each project. - See if Merge Requests were approved and by whom. +- See Merge Request authors. - See the latest [CI Pipeline](../../../ci/pipelines/index.md) result for each Merge Request. ## Permissions diff --git a/doc/user/compliance/license_compliance/img/policies_maintainer_add_v13_0.png b/doc/user/compliance/license_compliance/img/policies_maintainer_add_v13_0.png Binary files differdeleted file mode 100644 index 8070e2cb1a5..00000000000 --- a/doc/user/compliance/license_compliance/img/policies_maintainer_add_v13_0.png +++ /dev/null diff --git a/doc/user/compliance/license_compliance/img/policies_maintainer_add_v13_2.png b/doc/user/compliance/license_compliance/img/policies_maintainer_add_v13_2.png Binary files differnew file mode 100644 index 00000000000..2d5946503cf --- /dev/null +++ b/doc/user/compliance/license_compliance/img/policies_maintainer_add_v13_2.png diff --git a/doc/user/compliance/license_compliance/img/policies_maintainer_edit_v13_0.png b/doc/user/compliance/license_compliance/img/policies_maintainer_edit_v13_0.png Binary files differdeleted file mode 100644 index 741d1237751..00000000000 --- a/doc/user/compliance/license_compliance/img/policies_maintainer_edit_v13_0.png +++ /dev/null diff --git a/doc/user/compliance/license_compliance/img/policies_maintainer_edit_v13_2.png b/doc/user/compliance/license_compliance/img/policies_maintainer_edit_v13_2.png Binary files differnew file mode 100644 index 00000000000..ee3bb310c1a --- /dev/null +++ b/doc/user/compliance/license_compliance/img/policies_maintainer_edit_v13_2.png diff --git a/doc/user/compliance/license_compliance/index.md b/doc/user/compliance/license_compliance/index.md index 4ceb393af8c..fb287fb2bf6 100644 --- a/doc/user/compliance/license_compliance/index.md +++ b/doc/user/compliance/license_compliance/index.md @@ -23,8 +23,8 @@ GitLab checks the License Compliance report, compares the licenses between the source and target branches, and shows the information right on the merge request. Denied licenses will be clearly visible with an `x` red icon next to them as well as new licenses which need a decision from you. In addition, you can -[manually allow or deny](#project-policies-for-license-compliance) -licenses in your project's settings. +[manually allow or deny](#policies) +licenses in your project's license compliance policy section. NOTE: **Note:** If the license compliance report doesn't have anything to compare to, no information @@ -46,7 +46,7 @@ When GitLab detects a **Denied** license, you can view it in the [license list]( You can view and modify existing policies from the [policies](#policies) tab. - + ## Use cases @@ -64,7 +64,7 @@ The following languages and package managers are supported. | Go | [Godep](https://github.com/tools/godep), [go mod](https://github.com/golang/go/wiki/Modules) |[License Finder](https://github.com/pivotal/LicenseFinder)| | Java | [Gradle](https://gradle.org/), [Maven](https://maven.apache.org/) |[License Finder](https://github.com/pivotal/LicenseFinder)| | .NET | [Nuget](https://www.nuget.org/) (.NET Framework is supported via the [mono project](https://www.mono-project.com/). Windows specific dependencies are not supported at this time.) |[License Finder](https://github.com/pivotal/LicenseFinder)| -| Python | [pip](https://pip.pypa.io/en/stable/) (Python is supported through [requirements.txt](https://pip.pypa.io/en/1.1/requirements/) and [Pipfile.lock](https://github.com/pypa/pipfile#pipfilelock).) |[License Finder](https://github.com/pivotal/LicenseFinder)| +| Python | [pip](https://pip.pypa.io/en/stable/) (Python is supported through [requirements.txt](https://pip.pypa.io/en/stable/user_guide/#requirements-files) and [Pipfile.lock](https://github.com/pypa/pipfile#pipfilelock).) |[License Finder](https://github.com/pivotal/LicenseFinder)| | Ruby | [gem](https://rubygems.org/) |[License Finder](https://github.com/pivotal/LicenseFinder)| | Objective-C, Swift | [Carthage](https://github.com/Carthage/Carthage) |[License Finder](https://github.com/pivotal/LicenseFinder)| @@ -86,7 +86,7 @@ which means that the reported licenses might be incomplete or inaccurate. | Elixir | [mix](https://elixir-lang.org/getting-started/mix-otp/introduction-to-mix.html) |[License Finder](https://github.com/pivotal/LicenseFinder)| | C++/C | [conan](https://conan.io/) |[License Finder](https://github.com/pivotal/LicenseFinder)| | Scala | [sbt](https://www.scala-sbt.org/) |[License Finder](https://github.com/pivotal/LicenseFinder)| -| Rust | [cargo](https://crates.io/) |[License Finder](https://github.com/pivotal/LicenseFinder)| +| Rust | [cargo](https://crates.io) |[License Finder](https://github.com/pivotal/LicenseFinder)| | PHP | [composer](https://getcomposer.org/) |[License Finder](https://github.com/pivotal/LicenseFinder)| ## Requirements @@ -339,7 +339,7 @@ strict-ssl = false ### Configuring Yarn projects -You can configure Yarn projects by using a [`.yarnrc.yml`](https://yarnpkg.com/configuration/yarnrc/) +You can configure Yarn projects by using a [`.yarnrc.yml`](https://yarnpkg.com/configuration/yarnrc) file. #### Using private Yarn registries @@ -385,6 +385,26 @@ You can supply a custom root certificate to complete TLS verification by using t specifying a `ca` setting in a [`.bowerrc`](https://bower.io/docs/config/#bowerrc-specification) file. +#### Using private Bundler registries + +If you have a private Bundler registry you can use the +[`source`](https://bundler.io/man/gemfile.5.html#GLOBAL-SOURCES) +setting to specify its location. + +For example: + +```plaintext +source "https://gems.example.com" +``` + +#### Custom root certificates for Bundler + +You can supply a custom root certificate to complete TLS verification by using the +`ADDITIONAL_CA_CERT_BUNDLE` [environment variable](#available-variables), or by +specifying a [`BUNDLE_SSL_CA_CERT`](https://bundler.io/v2.0/man/bundle-config.1.html) +[environment variable](../../../ci/variables/README.md#custom-environment-variables) +in the job definition. + ### Configuring Conan projects You can configure [Conan](https://conan.io/) projects by adding a `.conan` directory to your @@ -490,6 +510,29 @@ license_scanning: GOFLAGS: '-insecure' ``` +#### Using private NuGet registries + +If you have a private NuGet registry you can add it as a source +by adding it to the [`packageSources`](https://docs.microsoft.com/en-us/nuget/reference/nuget-config-file#package-source-sections) +section of a [`nuget.config`](https://docs.microsoft.com/en-us/nuget/reference/nuget-config-file) file. + +For example: + +```xml +<?xml version="1.0" encoding="utf-8"?> +<configuration> + <packageSources> + <clear /> + <add key="custom" value="https://nuget.example.com/v3/index.json" /> + </packageSources> +</configuration> +``` + +#### Custom root certificates for NuGet + +You can supply a custom root certificate to complete TLS verification by using the +`ADDITIONAL_CA_CERT_BUNDLE` [environment variable](#available-variables). + ### Migration from `license_management` to `license_scanning` In GitLab 12.8 a new name for `license_management` job was introduced. This change was made to improve clarity around the purpose of the scan, which is to scan and collect the types of licenses present in a projects dependencies. @@ -594,6 +637,7 @@ your code and generate security reports, without requiring internet access. Additional configuration may be needed for connecting to [private Bower registries](#using-private-bower-registries), +[private Bundler registries](#using-private-bundler-registries), [private Conan registries](#using-private-bower-registries), [private Go registries](#using-private-go-registries), [private Maven repositories](#using-private-maven-repos), @@ -601,69 +645,9 @@ Additional configuration may be needed for connecting to [private Python repositories](#using-private-python-repos), and [private Yarn registries](#using-private-yarn-registries). -Exact name matches are required for [project policies](#project-policies-for-license-compliance) +Exact name matches are required for [project policies](#policies) when running in an offline environment ([see related issue](https://gitlab.com/gitlab-org/gitlab/-/issues/212388)). -## Project policies for License Compliance - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/5940) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.4. - -From the project's settings: - -- The list of licenses and their status can be managed. -- Licenses can be manually allowed or denied. - -To allow or deny a license: - -1. Either use the **Manage licenses** button in the merge request widget, or - navigate to the project's **Settings > CI/CD** and expand the - **License Compliance** section. -1. Click the **Add a license** button. - -  - -1. In the **License name** dropdown, either: - - Select one of the available licenses. You can search for licenses in the field - at the top of the list. - - Enter arbitrary text in the field at the top of the list. This will cause the text to be - added as a license name to the list. -1. Select the **Allow** or **Deny** radio button to allow or deny respectively - the selected license. - -To modify an existing license: - -1. In the **License Compliance** list, click the **Allow/Deny** dropdown to change it to the desired status. - -  - -Searching for Licenses: - -1. Use the **Search** box to search for a specific license. - -  - -## License Compliance report under pipelines - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/5491) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.2. - -From your project's left sidebar, navigate to **CI/CD > Pipelines** and click on the -pipeline ID that has a `license_scanning` job to see the Licenses tab with the listed -licenses (if any). - - - -<!-- ## Troubleshooting - -Include any troubleshooting steps that you can foresee. If you know beforehand what issues -one might have when setting this up, or when something is changed, or on upgrading, it's -important to describe those, too. Think of things that may go wrong and include them here. -This is important to minimize requests for support, and to avoid doc comments with -questions that you know someone might ask. - -Each scenario can be a third-level heading, e.g. `### Getting error message X`. -If you have none to add when creating a doc, leave this section in place -but commented out to help encourage others to add to it in the future. --> - ## License list > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/13582) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.7. @@ -696,13 +680,40 @@ and the associated classifications for each. Policies can be configured by maintainers of the project. - - + + Developers of the project can view the policies configured in a project.  +### Enabling License Approvals within a project + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/13067) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.3. + +`License-Check` is an approval rule you can enable to allow an approver, individual, or group to +approve a merge request that contains a `denied` license. + +You can enable `License-Check` one of two ways: + +- Create a [project approval rule](../../project/merge_requests/merge_request_approvals.md#multiple-approval-rules-premium) + with the case-sensitive name `License-Check`. +- Create an approval group in the [project policies section for License Compliance](#policies). + You must set this approval group's number of approvals required to greater than zero. Once you + enable this group in your project, the approval rule is enabled for all merge requests. + +Any code changes cause the approvals required to reset. + +An approval is required when a license report: + +- Contains a dependency that includes a software license that is `denied`. +- Is not generated during pipeline execution. + +An approval is optional when a license report: + +- Contains no software license violations. +- Contains only new licenses that are `allowed` or unknown. + ## Troubleshooting ### `ERROR -- : asdf: No preset version installed for command` diff --git a/doc/user/discussions/index.md b/doc/user/discussions/index.md index 44802214d7b..599f46b2c40 100644 --- a/doc/user/discussions/index.md +++ b/doc/user/discussions/index.md @@ -487,14 +487,15 @@ For example, to customize the commit message to output NOTE: **Note:** Custom commit messages for each applied Suggestion (and for batch Suggestions) will be -introduced by [#25381](https://gitlab.com/gitlab-org/gitlab/issues/25381). +introduced by [#25381](https://gitlab.com/gitlab-org/gitlab/-/issues/25381). ### Batch Suggestions -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/25486) in GitLab 13.1 as an [alpha feature](https://about.gitlab.com/handbook/product/#alpha). -> - It's deployed behind a feature flag, disabled by default. -> - It's disabled on GitLab.com. -> - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enable-or-disable-batch-suggestions). +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/25486) in GitLab 13.1 as an [alpha feature](https://about.gitlab.com/handbook/product/#alpha). +> - It was deployed behind a feature flag, disabled by default. +> - [Became enabled by default](https://gitlab.com/gitlab-org/gitlab/-/issues/227799) on GitLab 13.2. +> - It's enabled on GitLab.com. +> - For GitLab self-managed instances, GitLab administrators can opt to [disable it](#enable-or-disable-batch-suggestions-core-only). You can apply multiple suggestions at once to reduce the number of commits added to your branch to address your reviewers' requests. @@ -515,25 +516,25 @@ to your branch to address your reviewers' requests.  -#### Enable or disable Batch Suggestions +#### Enable or disable Batch Suggestions **(CORE ONLY)** Batch Suggestions is -deployed behind a feature flag that is **disabled by default**. +deployed behind a feature flag that is **enabled by default**. [GitLab administrators with access to the GitLab Rails console](../../administration/feature_flags.md) -can enable it for your instance. +can opt to disable it for your instance. To enable it: ```ruby # Instance-wide -Feature.enable(:batched_suggestions) +Feature.enable(:batch_suggestions) ``` To disable it: ```ruby # Instance-wide -Feature.disable(:batched_suggestions) +Feature.disable(:batch_suggestions) ``` ## Start a thread by replying to a standard comment diff --git a/doc/user/gitlab_com/index.md b/doc/user/gitlab_com/index.md index 6522f5c4053..bcaba97cab7 100644 --- a/doc/user/gitlab_com/index.md +++ b/doc/user/gitlab_com/index.md @@ -78,31 +78,34 @@ Below are the current settings regarding [GitLab CI/CD](../../ci/README.md). | Setting | GitLab.com | Default | | ----------- | ----------------- | ------------- | | Artifacts maximum size (uncompressed) | 1G | 100M | -| Artifacts [expiry time](../../ci/yaml/README.md#artifactsexpire_in) | kept forever | deleted after 30 days unless otherwise specified | +| Artifacts [expiry time](../../ci/yaml/README.md#artifactsexpire_in) | From June 22, 2020, deleted after 30 days unless otherwise specified (artifacts created before that date have no expiry). | deleted after 30 days unless otherwise specified | | Scheduled Pipeline Cron | `*/5 * * * *` | `19 * * * *` | | [Max jobs in active pipelines](../../administration/instance_limits.md#number-of-jobs-in-active-pipelines) | `500` for Free tier, unlimited otherwise | Unlimited | [Max pipeline schedules in projects](../../administration/instance_limits.md#number-of-pipeline-schedules) | `10` for Free tier, `50` for all paid tiers | Unlimited | | [Max number of instance level variables](../../administration/instance_limits.md#number-of-instance-level-variables) | `25` | `25` | +| [Scheduled Job Archival](../../user/admin_area/settings/continuous_integration.md#archive-jobs-core-only) | 3 months | Never | ## Repository size limit -The maximum size your Git repository is allowed to be, including LFS. If you are near -or over the size limit, you can [reduce your repository size with Git](../project/repository/reducing_the_repo_size_using_git.md). +GitLab.com has the following [account limits](../admin_area/settings/account_and_limit_settings.md) enabled. If a setting is not listed, it is set to the default value. -| Setting | GitLab.com | Default | -| ----------- | ----------------- | ------------- | -| Repository size including LFS | 10G | Unlimited | +If you are near +or over the repository size limit, you can [reduce your repository size with Git](../project/repository/reducing_the_repo_size_using_git.md). + +| Setting | GitLab.com | Default | +| ----------- | ----------- | ------------- | +| Repository size including LFS | 10 GB | Unlimited | NOTE: **Note:** -`git push` and GitLab project imports are limited to 5GB per request. Git LFS and imports other than a file upload are not affected by this limit. +`git push` and GitLab project imports are limited to 5 GB per request through Cloudflare. Git LFS and imports other than a file upload are not affected by this limit. ## IP range GitLab.com is using the IP range `34.74.90.64/28` for traffic from its Web/API fleet. This whole range is solely allocated to GitLab. You can expect connections from webhooks or repository mirroring to come -from those IPs and whitelist them. +from those IPs and allow them. -GitLab.com is fronted by Cloudflare. For incoming connections to GitLab.com you might need to whitelist CIDR blocks of Cloudflare ([IPv4](https://www.cloudflare.com/ips-v4) and [IPv6](https://www.cloudflare.com/ips-v6)) +GitLab.com is fronted by Cloudflare. For incoming connections to GitLab.com you might need to allow CIDR blocks of Cloudflare ([IPv4](https://www.cloudflare.com/ips-v4) and [IPv6](https://www.cloudflare.com/ips-v6)). For outgoing connections from CI/CD runners we are not providing static IP addresses. All our runners are deployed into Google Cloud Platform (GCP) - any IP based @@ -334,9 +337,9 @@ Windows Shared Runners: ```yaml .shared_windows_runners: tags: - - shared-windows - - windows - - windows-1809 + - shared-windows + - windows + - windows-1809 stages: - build @@ -349,17 +352,17 @@ before_script: build: extends: - - .shared_windows_runners + - .shared_windows_runners stage: build script: - - echo "running scripts in the build job" + - echo "running scripts in the build job" test: extends: - - .shared_windows_runners + - .shared_windows_runners stage: test script: - - echo "running scripts in the test job" + - echo "running scripts in the test job" ``` #### Limitations and known issues @@ -581,9 +584,9 @@ is used to forward logs to an [Elastic cluster](https://gitlab.com/gitlab-com/ru You can view more information in our runbooks such as: -- A [detailed list of what we're logging](https://gitlab.com/gitlab-com/runbooks/tree/master/logging/doc#what-are-we-logging) -- Our [current log retention policies](https://gitlab.com/gitlab-com/runbooks/tree/master/logging/doc#retention) -- A [diagram of our logging infrastructure](https://gitlab.com/gitlab-com/runbooks/tree/master/logging/doc#logging-infrastructure-overview) +- A [detailed list of what we're logging](https://gitlab.com/gitlab-com/runbooks/-/tree/master/docs/logging#what-are-we-logging) +- Our [current log retention policies](https://gitlab.com/gitlab-com/runbooks/-/tree/master/docs/logging#retention) +- A [diagram of our logging infrastructure](https://gitlab.com/gitlab-com/runbooks/-/tree/master/docs/logging#logging-infrastructure-overview) ## GitLab.com at scale diff --git a/doc/user/group/bulk_editing/img/bulk-editing.png b/doc/user/group/bulk_editing/img/bulk-editing.png Binary files differdeleted file mode 100644 index ca1662a781b..00000000000 --- a/doc/user/group/bulk_editing/img/bulk-editing.png +++ /dev/null diff --git a/doc/user/group/bulk_editing/img/bulk-editing_v13_2.png b/doc/user/group/bulk_editing/img/bulk-editing_v13_2.png Binary files differnew file mode 100644 index 00000000000..9f28fabdf0c --- /dev/null +++ b/doc/user/group/bulk_editing/img/bulk-editing_v13_2.png diff --git a/doc/user/group/bulk_editing/index.md b/doc/user/group/bulk_editing/index.md index c4ccc1f7b52..35bdc6696eb 100644 --- a/doc/user/group/bulk_editing/index.md +++ b/doc/user/group/bulk_editing/index.md @@ -8,12 +8,12 @@ info: To determine the technical writer assigned to the Stage/Group associated w NOTE: **Note:** Bulk editing issues and merge requests is also available at the **project level**. -For more details, see [Bulk editing issues, epics, and merge requests](../../project/bulk_editing.md). +For more details, see [Bulk editing issues and merge requests at the project level](../../project/bulk_editing.md). If you want to update attributes across multiple issues, epics, or merge requests in a group, you can do it by bulk editing them, that is, editing them together. - + ## Bulk edit issues at the group level @@ -24,8 +24,12 @@ You need a permission level of [Reporter or higher](../../permissions.md) to man When bulk editing issues in a group, you can edit the following attributes: +- Epic ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/210470) in + [GitLab Premium](https://about.gitlab.com/pricing/) 13.2.) **(PREMIUM)** - Milestone - Labels +- Health status ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/218395) in + [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.2.) **(ULTIMATE)** To update multiple project issues at the same time: diff --git a/doc/user/group/clusters/index.md b/doc/user/group/clusters/index.md index 5cdac7ae892..89e0c4898fb 100644 --- a/doc/user/group/clusters/index.md +++ b/doc/user/group/clusters/index.md @@ -38,10 +38,11 @@ the project. In the case of sub-groups, GitLab uses the cluster of the closest ancestor group to the project, provided the cluster is not disabled. -## Multiple Kubernetes clusters **(PREMIUM)** +## Multiple Kubernetes clusters -With [GitLab Premium](https://about.gitlab.com/pricing/premium/), you can associate -more than one Kubernetes cluster to your group, and maintain different clusters +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/35094) to GitLab Core in 13.2. + +You can associate more than one Kubernetes cluster to your group, and maintain different clusters for different environments, such as development, staging, and production. When adding another cluster, @@ -93,7 +94,7 @@ To clear the cache: > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/24580) in GitLab 11.8. Domains at the cluster level permit support for multiple domains -per [multiple Kubernetes clusters](#multiple-kubernetes-clusters-premium). When specifying a domain, +per [multiple Kubernetes clusters](#multiple-kubernetes-clusters) When specifying a domain, this will be automatically set as an environment variable (`KUBE_INGRESS_BASE_DOMAIN`) during the [Auto DevOps](../../../topics/autodevops/index.md) stages. @@ -127,8 +128,8 @@ And the following environments are set in [`.gitlab-ci.yml`](../../../ci/yaml/RE ```yaml stages: -- test -- deploy + - test + - deploy test: stage: test diff --git a/doc/user/group/epics/img/epic_activity_sort_order_v13_2.png b/doc/user/group/epics/img/epic_activity_sort_order_v13_2.png Binary files differnew file mode 100644 index 00000000000..c7e1448fea9 --- /dev/null +++ b/doc/user/group/epics/img/epic_activity_sort_order_v13_2.png diff --git a/doc/user/group/epics/img/epics_list_view_v12.5.png b/doc/user/group/epics/img/epics_list_view_v12.5.png Binary files differdeleted file mode 100644 index 6e3c39009be..00000000000 --- a/doc/user/group/epics/img/epics_list_view_v12.5.png +++ /dev/null diff --git a/doc/user/group/epics/img/new_epic_form_v13.2.png b/doc/user/group/epics/img/new_epic_form_v13.2.png Binary files differnew file mode 100644 index 00000000000..3d24763d105 --- /dev/null +++ b/doc/user/group/epics/img/new_epic_form_v13.2.png diff --git a/doc/user/group/epics/img/new_epic_from_groups_v13.2.png b/doc/user/group/epics/img/new_epic_from_groups_v13.2.png Binary files differnew file mode 100644 index 00000000000..85bc4255595 --- /dev/null +++ b/doc/user/group/epics/img/new_epic_from_groups_v13.2.png diff --git a/doc/user/group/epics/index.md b/doc/user/group/epics/index.md index 85164292265..a2b04e2d7fe 100644 --- a/doc/user/group/epics/index.md +++ b/doc/user/group/epics/index.md @@ -14,8 +14,15 @@ Epics let you manage your portfolio of projects more efficiently and with less effort by tracking groups of issues that share a theme, across projects and milestones. -<!-- Possibly swap this file with one of a single epic --> - +An epic's page contains the following tabs: + +- **Epics and Issues**: epics and issues added to this epic. Child epics, and their issues, are + shown in a tree view. + - Click the chevron (**>**) next to a parent epic to reveal the child epics and issues. + - Hover over the total counts to see a breakdown of open and closed items. +- **Roadmap**: a roadmap view of child epics which have start and due dates. + + ## Use cases @@ -28,6 +35,7 @@ milestones. To learn what you can do with an epic, see [Manage epics](manage_epics.md). Possible actions include: - [Create an epic](manage_epics.md#create-an-epic) +- [Edit an epic](manage_epics.md#edit-an-epic) - [Bulk-edit epics](../bulk_editing/index.md#bulk-edit-epics) - [Delete an epic](manage_epics.md#delete-an-epic) - [Close an epic](manage_epics.md#close-an-epic) @@ -165,6 +173,19 @@ Once you write your comment, you can either: - Click **Comment**, and your comment will be published. - Click **Start thread**, and you will start a thread within that epic's discussion. +### Activity sort order + +> [Introduced](https://https://gitlab.com/gitlab-org/gitlab/-/issues/214364) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.2. + +You can reverse the default order and interact with the activity feed sorted by most recent items +at the top. Your preference is saved via local storage and automatically applied to every issue +you view. + +To change the activity sort order, click the **Oldest first** dropdown menu and select either oldest +or newest items to be shown first. + + + ## Award emoji You can [award an emoji](../../award_emojis.md) to that epic or its comments. diff --git a/doc/user/group/epics/manage_epics.md b/doc/user/group/epics/manage_epics.md index 26d5cb51e01..4f9bb0e24fb 100644 --- a/doc/user/group/epics/manage_epics.md +++ b/doc/user/group/epics/manage_epics.md @@ -18,12 +18,42 @@ A paginated list of epics is available in each group from where you can create a new epic. The list of epics includes also epics from all subgroups of the selected group. From your group page: -1. Go to **Epics**. +### Create an epic from the epic list + +To create an epic from the epic list, in a group: + +1. Go to **{epic}** **Epics**. 1. Click **New epic**. 1. Enter a descriptive title. 1. Click **Create epic**. -You will be taken to the new epic where can edit the following details: +### Access the New Epic form + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/211533) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.2. + +There are two ways to get to the New Epic form and create an epic in the group you're in: + +- From an epic in your group, click **New Epic**. +- From anywhere, in the top menu, click **plus** (**{plus-square}**) **> New epic**. + +  + +### Elements of the New Epic form + +When you're creating a new epic, these are the fields you can fill in: + +- Title +- Description +- Confidentiality checkbox +- Labels +- Start date +- Due date + + + +## Edit an epic + +After you create an epic, you can edit change the following details: - Title - Description @@ -31,15 +61,16 @@ You will be taken to the new epic where can edit the following details: - Due date - Labels -An epic's page contains the following tabs: +To edit an epic's title or description: -- **Epics and Issues**: epics and issues added to this epic. Child epics, and their issues, are - shown in a tree view. - - Click the <kbd>></kbd> beside a parent epic to reveal the child epics and issues. - - Hover over the total counts to see a breakdown of open and closed items. -- **Roadmap**: a roadmap view of child epics which have start and due dates. +1. Click the **Edit title and description** **{pencil}** button. +1. Make your changes. +1. Click **Save changes**. - +To edit an epics' start date, due date, or labels: + +1. Click **Edit** next to each section in the epic sidebar. +1. Select the dates or labels for your epic. ## Bulk-edit epics @@ -118,27 +149,17 @@ The sort option and order is saved and used wherever you browse epics, including ## Make an epic confidential -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/213068) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.0. -> - It's deployed behind a feature flag, disabled by default. -> - It's disabled on GitLab.com. -> - It's not recommended for production use. -> - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enable-confidential-epics-premium-only). **(PREMIUM ONLY)** +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/213068) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.0 behind a feature flag, disabled by default. +> - [Became enabled by default](https://gitlab.com/gitlab-org/gitlab/-/issues/224513) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.2. When you're creating an epic, you can make it confidential by selecting the **Make this epic confidential** checkbox. -### Enable Confidential Epics **(PREMIUM ONLY)** +### Disable confidential epics **(PREMIUM ONLY)** -The Confidential Epics feature is under development and not ready for production use. -It's deployed behind a feature flag that is **disabled by default**. +The confidential epics feature is deployed behind a feature flag that is **enabled by default**. [GitLab administrators with access to the GitLab Rails console](../../../administration/feature_flags.md) -can enable it for your instance. - -To enable it: - -```ruby -Feature.enable(:confidential_epics) -``` +can disable it for your self-managed instance. To disable it: @@ -233,7 +254,7 @@ To move an issue to another epic: > - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/37081) to [GitLab Premium](https://about.gitlab.com/pricing/) in 12.8. If you have the necessary [permissions](../../permissions.md) to close an issue and create an -epic in the parent group, you can promote an issue to an epic with the `/promote` +epic in the immediate parent group, you can promote an issue to an epic with the `/promote` [quick action](../../project/quick_actions.md#quick-actions-for-issues-merge-requests-and-epics). Only issues from projects that are in groups can be promoted. When attempting to promote a confidential issue, a warning will display. Promoting a confidential issue to an epic will make all information diff --git a/doc/user/group/index.md b/doc/user/group/index.md index 324c912b2be..5ba0680127c 100644 --- a/doc/user/group/index.md +++ b/doc/user/group/index.md @@ -31,7 +31,7 @@ Each group on the **Groups** page is listed with: - How many subgroups it has. - How many projects it contains. -- How many members the group has, not including members inherited from parent groups. +- How many members the group has, not including members inherited from parent group(s). - The group's visibility. - A link to the group's settings, if you have sufficient permissions. - A link to leave the group, if you are a member. @@ -184,6 +184,33 @@ of a group: 1. Give a different member **Owner** permissions. 1. Have the new owner sign in and remove **Owner** permissions from you. +## Remove a member from the group + +Only users with permissions of [Owner](../permissions.md#group-members-permissions) can manage +group members. + +You can remove a member from the group if the given member has a direct membership in the group. If +membership is inherited from a parent group, then the member can be removed only from the parent +group itself. + +When removing a member, you can decide whether to unassign the user from all issues and merge +requests they are currently assigned or leave the assignments as they are. + +- **Unassigning the removed member** from all issues and merge requests might be helpful when a user + is leaving a private group and you wish to revoke their access to any issues and merge requests + they are assigned. +- **Keeping the issues and merge requests assigned** might be helpful for groups that accept public + contributions where a user doesn't have to be a member to be able to contribute to issues and + merge requests. + +To remove a member from a group: + +1. In a group, go to **{users}** **Members**. +1. Click the **Delete** **{remove}** button next to a group member you want to remove. + A **Remove member** modal appears. +1. (Optional) Select the **Also unassign this user from related issues and merge requests** checkbox. +1. Click **Remove member**. + ## Changing the default branch protection of a group > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7583) in GitLab 12.9. @@ -397,7 +424,7 @@ When transferring groups, note: - Changing a group's parent can have unintended side effects. See [Redirects when changing repository paths](../project/index.md#redirects-when-changing-repository-paths). - You can only transfer groups to groups you manage. - You must update your local repositories to point to the new location. -- If the parent group's visibility is lower than the group's current visibility, visibility levels for subgroups and projects will change to match the new parent group's visibility. +- If the immediate parent group's visibility is lower than the group's current visibility, visibility levels for subgroups and projects will change to match the new parent group's visibility. - Only explicit group membership is transferred, not inherited membership. If the group's owners have only inherited membership, this leaves the group without an owner. In this case, the user transferring the group becomes the group's owner. ## Group settings @@ -435,7 +462,7 @@ It is currently not possible to rename a namespace if it contains a project with [Container Registry](../packages/container_registry/index.md) tags, because the project cannot be moved. -TIP: **TIP:** +TIP: **Tip:** If you want to retain ownership over the original namespace and protect the URL redirects, then instead of changing a group's path or renaming a username, you can create a new group and transfer projects to it. @@ -516,7 +543,7 @@ underlying projects, issues, etc, by IP address. This can help ensure that particular content doesn't leave the premises, while not blocking off access to the entire instance. -Add one or more allowed IP subnets using CIDR notation in comma separated format to the group settings and anyone +Add one or more allowed IP subnets using CIDR notation to the group settings and anyone coming from a different IP address won't be able to access the restricted content. @@ -529,6 +556,12 @@ Restriction currently applies to: To avoid accidental lock-out, admins and group owners are able to access the group regardless of the IP restriction. +To enable this feature: + +1. Navigate to the group’s **Settings > General** page. +1. Expand the **Permissions, LFS, 2FA** section, and enter IP address ranges into **Allow access to the following IP addresses** field. +1. Click **Save changes**. + #### Allowed domain restriction **(PREMIUM)** >- [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7297) in [GitLab Premium and Silver](https://about.gitlab.com/pricing/) 12.2. @@ -554,7 +587,7 @@ Some domains cannot be restricted. These are the most popular public email domai To enable this feature: 1. Navigate to the group's **Settings > General** page. -1. Expand the **Permissions, LFS, 2FA** section, and enter the domain names into **Restrict membership by email** field. You can enter multiple domains by separating each domain with a comma (,). +1. Expand the **Permissions, LFS, 2FA** section, and enter the domain names into **Restrict membership by email** field. 1. Click **Save changes**. This will enable the domain-checking for all new users added to the group from this moment on. @@ -571,9 +604,9 @@ You can only choose projects in the group as the template source. This includes projects shared with the group, but it **excludes** projects in subgroups or parent groups of the group being configured. -You can configure this feature for both subgroups and parent groups. A project +You can configure this feature for both subgroups and immediate parent groups. A project in a subgroup will have access to the templates for that subgroup, as well as -any parent groups. +any immediate parent groups.  @@ -617,6 +650,23 @@ To enable this feature: 1. Expand the **Permissions, LFS, 2FA** section, and select **Disable group mentions**. 1. Click **Save changes**. +#### Enabling delayed Project removal **(PREMIUM)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/220382) in GitLab 13.2. + +By default, projects within a group are deleted immediately. +Optionally, on [Premium or Silver](https://about.gitlab.com/pricing/) or higher tiers, +you can configure the projects within a group to be deleted after a delayed interval. + +During this interval period, the projects will be in a read-only state and can be restored, if required. +The interval period defaults to 7 days, and can be modified by an admin in the [instance settings](../admin_area/settings/visibility_and_access_controls.md#default-deletion-adjourned-period-premium-only). + +To enable delayed deletion of projects: + +1. Navigate to the group's **Settings > General** page. +1. Expand the **Permissions, LFS, 2FA** section, and check **Enable delayed project removal**. +1. Click **Save changes**. + ### Advanced settings - **Projects**: View all projects within that group, add members to each project, diff --git a/doc/user/group/iterations/index.md b/doc/user/group/iterations/index.md index 2704147dcdd..bc9d228011a 100644 --- a/doc/user/group/iterations/index.md +++ b/doc/user/group/iterations/index.md @@ -8,11 +8,12 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Iterations **(STARTER)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/214713) in [GitLab Starter](https://about.gitlab.com/pricing/) 13.1. -> - It's deployed behind a feature flag, disabled by default. -> - It's disabled on GitLab.com. -> - It's able to be enabled or disabled per-group -> - It's not recommended for production use. -> - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enable-iterations-core-only). **(CORE ONLY)** +> - It was deployed behind a feature flag, disabled by default. +> - [Became enabled by default](https://gitlab.com/gitlab-org/gitlab/-/issues/221047) on GitLab 13.2. +> - It's enabled on GitLab.com. +> - It's able to be enabled or disabled per-group. +> - It's recommended for production use. +> - For GitLab self-managed instances, GitLab administrators can opt to [disable it](#disable-iterations-core-only). **(CORE ONLY)** Iterations are a way to track issues over a period of time. This allows teams to track velocity and volatility metrics. Iterations can be used with [milestones](../../project/milestones/index.md) @@ -38,7 +39,7 @@ From there you can create a new iteration or click an iteration to get a more de ## Create an iteration NOTE: **Note:** -A permission level of [Developer or higher](../../permissions.md) is required to create iterations. +You need Developer [permissions](../../permissions.md) or higher to create an iteration. To create an iteration: @@ -47,12 +48,20 @@ To create an iteration: 1. Enter the title, a description (optional), a start date, and a due date. 1. Click **Create iteration**. The iteration details page opens. -### Enable Iterations **(CORE ONLY)** +## Edit an iteration -GitLab Iterations feature is under development and not ready for production use. -It is deployed behind a feature flag that is **disabled by default**. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/218277) in [GitLab Starter](https://about.gitlab.com/pricing/) 13.2. + +NOTE: **Note:** +You need Developer [permissions](../../permissions.md) or higher to edit an iteration. + +To edit an iteration, click the three-dot menu (**{ellipsis_v}**) > **Edit iteration**. + +## Disable Iterations **(CORE ONLY)** + +GitLab Iterations feature is deployed with a feature flag that is **enabled by default**. [GitLab administrators with access to the GitLab Rails console](../../../administration/feature_flags.md) -can enable it for your instance. `:group_iterations` can be enabled or disabled per-group. +can disable it for your instance. `:group_iterations` can be enabled or disabled per-group. To enable it: diff --git a/doc/user/group/roadmap/img/roadmap_view_v13_0.png b/doc/user/group/roadmap/img/roadmap_view_v13_0.png Binary files differdeleted file mode 100644 index a5b76b84418..00000000000 --- a/doc/user/group/roadmap/img/roadmap_view_v13_0.png +++ /dev/null diff --git a/doc/user/group/roadmap/img/roadmap_view_v13_2.png b/doc/user/group/roadmap/img/roadmap_view_v13_2.png Binary files differnew file mode 100644 index 00000000000..b4f889afaa4 --- /dev/null +++ b/doc/user/group/roadmap/img/roadmap_view_v13_2.png diff --git a/doc/user/group/roadmap/index.md b/doc/user/group/roadmap/index.md index 614ed700cfc..c185055f6b2 100644 --- a/doc/user/group/roadmap/index.md +++ b/doc/user/group/roadmap/index.md @@ -12,22 +12,24 @@ info: To determine the technical writer assigned to the Stage/Group associated w > - In [GitLab 12.9](https://gitlab.com/gitlab-org/gitlab/-/issues/5164) and later, the epic bars show epics' title, progress, and completed weight percentage. > - Milestones appear in roadmaps in [GitLab 12.10](https://gitlab.com/gitlab-org/gitlab/-/issues/6802), and later. > - Feature flag for milestones visible in roadmaps removed in [GitLab 13.0](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/29641). +> - In [GitLab 13.2](https://gitlab.com/gitlab-org/gitlab/-/issues/214375) and later, the Roadmap also shows milestones in projects in a group. +> - In [GitLab 13.2](https://gitlab.com/gitlab-org/gitlab/-/issues/212494) and later, milestone bars can be collapsed and expanded. -Epics and milestones within a group containing **Start date** and/or **Due date** -can be visualized in a form of a timeline (that is, a Gantt chart). The Roadmap page -shows such a visualization for all the epics and milestones which are under a group or one of its -subgroups. +Epics and milestones within a group containing a start date or due date can be visualized in a form +of a timeline (that is, a Gantt chart). The Roadmap page shows the epics and milestones in a +group, one of its subgroups, or a project in one of the groups. On the epic bars, you can see the each epic's title, progress, and completed weight percentage. When you hover over an epic bar, a popover appears with the epic's title, start date, due date, and weight completed. You can expand epics that contain child epics to show their child epics in the roadmap. -You can click the chevron **{chevron-down}** next to the epic title to expand and collapse the child epics. +You can click the chevron (**{chevron-down}**) next to the epic title to expand and collapse the child epics. On top of the milestone bars, you can see their title. When you hover a milestone bar or title, a popover appears with its title, start date and due date. +You can also click the chevron (**{chevron-down}**) next to the **Milestones** heading to toggle the list of the milestone bars. - + A dropdown menu allows you to show only open or closed epics. By default, all epics are shown. diff --git a/doc/user/group/saml_sso/group_managed_accounts.md b/doc/user/group/saml_sso/group_managed_accounts.md new file mode 100644 index 00000000000..e317573d89d --- /dev/null +++ b/doc/user/group/saml_sso/group_managed_accounts.md @@ -0,0 +1,116 @@ +--- +type: reference, howto +stage: Manage +group: Access +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Group Managed Accounts **(PREMIUM)** + +CAUTION: **Warning:** +This is a [Closed Beta](https://about.gitlab.com/handbook/product/#closed-beta) feature. + +> - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/709) in GitLab 12.1. +> - It's deployed behind a feature flag, disabled by default. + +When [SSO for Groups](index.md) is being enforced, groups can enable an additional level of protection by enforcing the creation of dedicated user accounts to access the group. + +With group-managed accounts enabled, users are required to create a new, dedicated user linked to the group. +The notification email address associated with the user is locked to the email address received from the configured identity provider. +Without group-managed accounts, users can link their SAML identity with any existing user on the instance. + +When this option is enabled: + +- All users in the group will be required to log in via the SSO URL associated with the group. +- After the group-managed account has been created, group activity will require the use of this user account. +- Users can't share a project in the group outside the top-level group (also applies to forked projects). + +Upon successful authentication, GitLab prompts the user with options, based on the email address received from the configured identity provider: + +- To create a unique account with the newly received email address. +- If the received email address matches one of the user's verified GitLab email addresses, the option to convert the existing account to a group-managed account. ([Introduced in GitLab 12.9](https://gitlab.com/gitlab-org/gitlab/-/issues/13481).) + +Since use of the group-managed account requires the use of SSO, users of group-managed accounts will lose access to these accounts when they are no longer able to authenticate with the connected identity provider. In the case of an offboarded employee who has been removed from your identity provider: + +- The user will be unable to access the group (their credentials will no longer work on the identity provider when prompted to SSO). +- Contributions in the group (e.g. issues, merge requests) will remain intact. + +## Assertions + +When using group-managed accounts, the following user details need to be passed to GitLab as SAML +assertions to be able to create a user. + +| Field | Supported keys | +|-----------------|----------------| +| Email (required)| `email`, `mail` | +| Full Name | `name` | +| First Name | `first_name`, `firstname`, `firstName` | +| Last Name | `last_name`, `lastname`, `lastName` | + +## Feature flag **(PREMIUM ONLY)** + +Currently the group-managed accounts feature is behind a feature flag: `group_managed_accounts`. The flag is disabled by default. +To activate the feature, ask a GitLab administrator with Rails console access to run: + +```ruby +Feature.enable(:group_managed_accounts) +``` + +## Project restrictions for Group-managed accounts + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12420) in GitLab 12.9. + +Projects within groups with enabled group-managed accounts are not to be shared with: + +- Groups outside of the parent group. +- Members who are not users managed by this group. + +This restriction also applies to projects forked from or to those groups. + +## Outer forks restriction for Group-managed accounts + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/34648) in GitLab 12.9. + +Groups with group-managed accounts can disallow forking of projects to destinations outside the group. +To do so, enable the "Prohibit outer forks" option in **Settings > SAML SSO**. +When enabled, projects within the group can only be forked to other destinations within the group (including its subgroups). + +## Credentials inventory for Group-managed accounts **(ULTIMATE)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/38133) in GitLab 12.8. + +Owners who manage user accounts in a group can view the following details of personal access tokens and SSH keys: + +- Owners +- Scopes +- Usage patterns + +To access the Credentials inventory of a group, navigate to **{shield}** **Security & Compliance > Credentials** in your group's sidebar. + +This feature is similar to the [Credentials inventory for self-managed instances](../../admin_area/credentials_inventory.md). + +## Limiting lifetime of personal access tokens of users in Group-managed accounts **(ULTIMATE)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/118893) in GitLab 12.10. + +Users in a group managed account can optionally specify an expiration date for +[personal access tokens](../../profile/personal_access_tokens.md). +This expiration date is not a requirement, and can be set to any arbitrary date. + +Since personal access tokens are the only token needed for programmatic access to GitLab, organizations with security requirements may want to enforce more protection to require regular rotation of these tokens. + +### Setting a limit + +Only a GitLab administrator or an owner of a Group-managed account can set a limit. Leaving it empty means that the [instance level restrictions](../../admin_area/settings/account_and_limit_settings.md#limiting-lifetime-of-personal-access-tokens-ultimate-only) on the lifetime of personal access tokens will apply. + +To set a limit on how long personal access tokens are valid for users in a group managed account: + +1. Navigate to the **{settings}** **Settings > General** page in your group's sidebar. +1. Expand the **Permissions, LFS, 2FA** section. +1. Fill in the **Maximum allowable lifetime for personal access tokens (days)** field. +1. Click **Save changes**. + +Once a lifetime for personal access tokens is set, GitLab will: + +- Apply the lifetime for new personal access tokens, and require users managed by the group to set an expiration date that is no later than the allowed lifetime. +- After three hours, revoke old tokens with no expiration date or with a lifetime longer than the allowed lifetime. Three hours is given to allow administrators/group owner to change the allowed lifetime, or remove it, before revocation takes place. diff --git a/doc/user/group/saml_sso/index.md b/doc/user/group/saml_sso/index.md index 81684038dc2..afd676cf897 100644 --- a/doc/user/group/saml_sso/index.md +++ b/doc/user/group/saml_sso/index.md @@ -5,32 +5,25 @@ group: Access info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# SAML SSO for GitLab.com groups **(SILVER ONLY)** +# SAML SSO for GitLab.com groups **(PREMIUM)** -> Introduced in [GitLab.com Silver](https://about.gitlab.com/pricing/) 11.0. +> Introduced in GitLab 11.0. -SAML on GitLab.com allows users to be added to a group. Those users can then sign in to GitLab.com. If such users don't already have an account on the GitLab instance, they can create one when signing in for the first time. +This page describes SAML for Groups. For instance-wide SAML on self-managed GitLab instances, see [SAML OmniAuth Provider](../../../integration/saml.md). -If you follow our guidance to automate user provisioning using [SCIM](scim_setup.md) or [group-managed accounts](#group-managed-accounts), you do not need to create such accounts manually. +SAML on GitLab.com allows users to sign in through their SAML identity provider. If the user is not already a member, the sign-in process automatically adds the user to the appropriate group. -User synchronization for GitLab.com is partially supported using [SCIM](scim_setup.md). +If you follow our guidance to automate user provisioning using [SCIM](scim_setup.md) or [group-managed accounts](group_managed_accounts.md), you do not need to create such accounts manually. -## Important notes - -Note the following: - -- This topic is for SAML on GitLab.com Silver tier and above. For SAML on self-managed GitLab - instances, see [SAML OmniAuth Provider](../../../integration/saml.md). -- SAML SSO for GitLab.com groups requires SCIM to sync users between providers. If a - group is not using SCIM, group Owners will still need to manage user accounts (for example, - removing users when necessary). +User synchronization of SAML SSO groups is supported through [SCIM](scim_setup.md). SCIM supports adding and removing users from the GitLab group. +For example, if you remove a user from the SCIM app, SCIM removes that same user from the GitLab group. ## Configuring your Identity Provider 1. Navigate to the group and click **Settings > SAML SSO**. -1. Configure your SAML server using the **Assertion consumer service URL** and **Identifier**. Alternatively GitLab provides [metadata XML configuration](#metadata-configuration). See [your identity provider's documentation](#providers) for more details. +1. Configure your SAML server using the **Assertion consumer service URL**, **Identifier**, and **GitLab single sign-on URL**. Alternatively GitLab provides [metadata XML configuration](#metadata-configuration). See [specific identity provider documentation](#providers) for more details. 1. Configure the SAML response to include a NameID that uniquely identifies each user. -1. Configure required assertions using the [table below](#assertions). +1. Configure [required assertions](group_managed_accounts.md#assertions) if using [Group Managed Accounts](group_managed_accounts.md). 1. Once the identity provider is set up, move on to [configuring GitLab](#configuring-gitlab).  @@ -55,123 +48,6 @@ Once users have signed into GitLab using the SSO SAML setup, changing the `NameI We recommend setting the NameID format to `Persistent` unless using a field (such as email) that requires a different format. -### SSO enforcement - -- [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/5291) in GitLab 11.8. -- [Improved](https://gitlab.com/gitlab-org/gitlab/-/issues/9255) in GitLab 11.11 with ongoing enforcement in the GitLab UI. - -With this option enabled, users must use your group's GitLab single sign on URL to be added to the group or be added via SCIM. Users cannot be added manually, and may only access project/group resources via the UI by signing in through the SSO URL. - -However, users will not be prompted to log via SSO on each visit. GitLab will check whether a user has authenticated through the SSO link, and will only prompt the user to login via SSO if the session has expired. - -We intend to add a similar SSO requirement for [Git and API activity](https://gitlab.com/gitlab-org/gitlab/-/issues/9152) in the future. - -When SSO enforcement is enabled for a group, users cannot share a project in the group outside the top-level group, even if the project is forked. - -#### Group-managed accounts - -> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/709) in GitLab 12.1. - -When SSO is being enforced, groups can enable an additional level of protection by enforcing the creation of dedicated user accounts to access the group. - -Without group-managed accounts, users can link their SAML identity with any existing user on the instance. With group-managed accounts enabled, users are required to create a new, dedicated user linked to the group. The notification email address associated with the user is locked to the email address received from the configured identity provider. - -When this option is enabled: - -- All existing and new users in the group will be required to log in via the SSO URL associated with the group. -- After the group-managed account has been created, group activity will require the use of this user account. -- Users can't share a project in the group outside the top-level group (also applies to forked projects). - -Upon successful authentication, GitLab prompts the user with options, based on the email address received from the configured identity provider: - -- To create a unique account with the newly received email address. -- If the received email address matches one of the user's verified GitLab email addresses, the option to convert the existing account to a group-managed account. ([Introduced in GitLab 12.9](https://gitlab.com/gitlab-org/gitlab/-/issues/13481).) - -Since use of the group-managed account requires the use of SSO, users of group-managed accounts will lose access to these accounts when they are no longer able to authenticate with the connected identity provider. In the case of an offboarded employee who has been removed from your identity provider: - -- The user will be unable to access the group (their credentials will no longer work on the identity provider when prompted to SSO). -- Contributions in the group (e.g. issues, merge requests) will remain intact. - -##### Feature flag - -Currently the group-managed accounts feature is behind a feature flag: `group_managed_accounts`. The flag is disabled by default. -To activate the feature, ask a GitLab administrator with Rails console access to run: - -```ruby -Feature.enable(:group_managed_accounts) -``` - -##### Credentials inventory for Group-managed accounts **(ULTIMATE)** - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/38133) in GitLab 12.8. - -Owners who manage user accounts in a group can view the following details of personal access tokens and SSH keys: - -- Owners -- Scopes -- Usage patterns - -To access the Credentials inventory of a group, navigate to **{shield}** **Security & Compliance > Credentials** in your group's sidebar. - -This feature is similar to the [Credentials inventory for self-managed instances](../../admin_area/credentials_inventory.md). - -##### Limiting lifetime of personal access tokens of users in Group-managed accounts **(ULTIMATE)** - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/118893) in GitLab 12.10. - -Users in a group managed account can optionally specify an expiration date for -[personal access tokens](../../profile/personal_access_tokens.md). -This expiration date is not a requirement, and can be set to any arbitrary date. - -Since personal access tokens are the only token needed for programmatic access to GitLab, organizations with security requirements may want to enforce more protection to require regular rotation of these tokens. - -###### Setting a limit - -Only a GitLab administrator or an owner of a Group-managed account can set a limit. Leaving it empty means that the [instance level restrictions](../../admin_area/settings/account_and_limit_settings.md#limiting-lifetime-of-personal-access-tokens-ultimate-only) on the lifetime of personal access tokens will apply. - -To set a limit on how long personal access tokens are valid for users in a group managed account: - -1. Navigate to the **{settings}** **Settings > General** page in your group's sidebar. -1. Expand the **Permissions, LFS, 2FA** section. -1. Fill in the **Maximum allowable lifetime for personal access tokens (days)** field. -1. Click **Save changes**. - -Once a lifetime for personal access tokens is set, GitLab will: - -- Apply the lifetime for new personal access tokens, and require users managed by the group to set an expiration date that is no later than the allowed lifetime. -- After three hours, revoke old tokens with no expiration date or with a lifetime longer than the allowed lifetime. Three hours is given to allow administrators/group owner to change the allowed lifetime, or remove it, before revocation takes place. - -##### Outer forks restriction for Group-managed accounts - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/34648) in GitLab 12.9. - -Groups with group-managed accounts can disallow forking of projects to destinations outside the group. -To do so, enable the "Prohibit outer forks" option in **Settings > SAML SSO**. -When enabled, projects within the group can only be forked to other destinations within the group (including its subgroups). - -##### Other restrictions for Group-managed accounts - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12420) in GitLab 12.9. - -Projects within groups with enabled group-managed accounts are not to be shared with: - -- Groups outside of the parent group. -- Members who are not users managed by this group. - -This restriction also applies to projects forked from or to those groups. - -#### Assertions - -When using group-managed accounts, the following user details need to be passed to GitLab as SAML -assertions to be able to create a user. - -| Field | Supported keys | -|-----------------|----------------| -| Email (required)| `email`, `mail` | -| Full Name | `name` | -| First Name | `first_name`, `firstname`, `firstName` | -| Last Name | `last_name`, `lastname`, `lastName` | - ### Metadata configuration GitLab provides metadata XML that can be used to configure your Identity Provider. @@ -185,7 +61,7 @@ GitLab provides metadata XML that can be used to configure your Identity Provide Once you've set up your identity provider to work with GitLab, you'll need to configure GitLab to use it for authentication: 1. Navigate to the group's **Settings > SAML SSO**. -1. Find the SSO URL from your Identity Provider and enter it the **Identity provider single sign on URL** field. +1. Find the SSO URL from your Identity Provider and enter it the **Identity provider single sign-on URL** field. 1. Find and enter the fingerprint for the SAML token signing certificate in the **Certificate** field. 1. Click the **Enable SAML authentication for this group** toggle switch. 1. Click the **Save changes** button. @@ -193,42 +69,27 @@ Once you've set up your identity provider to work with GitLab, you'll need to co  NOTE: **Note:** -Please note that the certificate [fingerprint algorithm](#additional-setup-options) must be in SHA1. When configuring the identity provider, use a secure [signature algorithm](#additional-setup-options). - -## User access and management - -Once Group SSO is configured and enabled, users can access the GitLab.com group through the identity provider's dashboard. If [SCIM](scim_setup.md) is configured, please see the [user access and linking setup section on the SCIM page](scim_setup.md#user-access-and-linking-setup). - -When a user tries to sign in with Group SSO, they'll need an account that's configured with one of the following: +Please note that the certificate [fingerprint algorithm](#additional-providers-and-setup-options) must be in SHA1. When configuring the identity provider, use a secure signature algorithm. -- [SCIM](scim_setup.md). -- [Group-managed accounts](#group-managed-accounts). -- A GitLab.com account. - -1. Click on the GitLab app in the identity provider's dashboard, or visit the Group's GitLab SSO URL. -1. Sign in to GitLab.com. The next time you connect on the same browser, you won't have to sign in again provided the active session has not expired. -1. Click on the **Authorize** button. - -On subsequent visits, users can access the group through the identify provider's dashboard or by visiting links directly. With the **enforce SSO** option turned on, users will be redirected to log in through the identity provider as required. - -### Role +### SSO enforcement -Upon first sign in, a new user is added to the parent group with the Guest role. Existing members with an appropriate role will have to elevate users to a higher role where relevant. +- [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/5291) in GitLab 11.8. +- [Improved](https://gitlab.com/gitlab-org/gitlab/-/issues/9255) in GitLab 11.11 with ongoing enforcement in the GitLab UI. -If a user is already a member of the group, linking the SAML identity does not change their role. +With this option enabled, users must go through your group's GitLab single sign-on URL. They may also be added via SCIM, if configured. Users cannot be added manually, and may only access project/group resources via the UI by signing in through the SSO URL. -### Blocking access +However, users will not be prompted to sign in through SSO on each visit. GitLab will check whether a user has authenticated through SSO, and will only prompt the user to sign in via SSO if the session has expired. -To rescind access to the group: +We intend to add a similar SSO requirement for [Git and API activity](https://gitlab.com/gitlab-org/gitlab/-/issues/9152). -1. Remove the user from the identity provider or users list for the specific app. -1. Remove the user from the GitLab.com group. +When SSO enforcement is enabled for a group, users cannot share a project in the group outside the top-level group, even if the project is forked. -Even when **enforce SSO** is active, we recommend removing the user from the group. Otherwise, the user can sign in through the identity provider if they do not have an active session. +To disallow users to contribute outside of the top-level group, please see [Group Managed Accounts](group_managed_accounts.md). ## Providers -NOTE: **Note:** GitLab is unable to provide support for IdPs that are not listed here. +NOTE: **Note:** +GitLab is unable to provide support for IdPs that are not listed here. | Provider | Documentation | |----------|---------------| @@ -248,7 +109,8 @@ For a demo of the Azure SAML setup including SCIM, see [SCIM Provisioning on Azu |--------------|----------------| | Identifier | Identifier (Entity ID) | | Assertion consumer service URL | Reply URL (Assertion Consumer Service URL) | -| Identity provider single sign on URL | Sign on URL | +| GitLab single sign-on URL | Sign on URL | +| Identity provider single sign-on URL | Login URL | | Certificate fingerprint | Thumbprint | We recommend: @@ -256,8 +118,6 @@ We recommend: - **Unique User Identifier (Name identifier)** set to `user.objectID`. - **nameid-format** set to persistent. -Set other user attributes and claims according to the [assertions table](#assertions). - ### Okta setup notes <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> @@ -266,19 +126,17 @@ For a demo of the Okta SAML setup including SCIM, see [Demo: Okta Group SAML & S | GitLab Setting | Okta Field | |--------------|----------------| | Identifier | Audience URI | -| Assertion consumer service URL | Single sign on URL | - -Under Okta's **Single sign on URL** field, check the option **Use this for Recipient URL and Destination URL**. +| Assertion consumer service URL | Single sign-on URL | +| GitLab single sign-on URL | Login page URL (under **Application Login Page** settings) | +| Identity provider single sign-on URL | Identity Provider Single Sign-On URL | -Please note that Okta's generic SAML app does not have a **Login URL** field, where the **Identity provider single sign on URL** would normally go. The **Identity provider single sign on URL** may be required the first time a user is logging in if they are having any difficulties. +Under Okta's **Single sign-on URL** field, check the option **Use this for Recipient URL and Destination URL**. We recommend: - **Application username** (NameID) set to **Custom** `user.getInternalProperty("id")`. - **Name ID Format** set to **Persistent**. -Set attribute statements according to the [assertions table](#assertions). - ### OneLogin setup notes The GitLab app listed in the OneLogin app catalog is for self-managed GitLab instances. @@ -290,15 +148,24 @@ For GitLab.com, use a generic SAML Test Connector such as the SAML Test Connecto | Assertion consumer service URL | Recipient | | Assertion consumer service URL | ACS (Consumer) URL | | Assertion consumer service URL (escaped version) | ACS (Consumer) URL Validator | -| GitLab single sign on URL | Login URL | +| GitLab single sign-on URL | Login URL | +| Identity provider single sign-on URL | SAML 2.0 Endpoint | Recommended `NameID` value: `OneLogin ID`. -Set parameters according to the [assertions table](#assertions). +### Additional providers and setup options + +The SAML standard means that a wide range of identity providers will work with GitLab. Unfortunately we have not verified connections with all SAML providers. +For more information, see our [discussion on providers](#providers). -### Additional setup options +Your identity provider may have relevant documentation. It may be generic SAML documentation, or specifically targeted for GitLab. Examples: + +- [Auth0](https://auth0.com/docs/protocols/saml/saml-idp-generic) +- [G Suite](https://support.google.com/a/answer/6087519?hl=en) +- [JumpCloud](https://support.jumpcloud.com/support/s/article/single-sign-on-sso-with-gitlab-2019-08-21-10-36-47) +- [PingOne by Ping Identity](https://docs.pingidentity.com/bundle/pingone/page/xsh1564020480660-1.html) -GitLab [isn't limited to the SAML providers listed above](#my-identity-provider-isnt-listed) but your Identity Provider may require additional configuration, such as the following: +Your Identity Provider may require additional configuration, such as the following: | Field | Value | Notes | |-------|-------|-------| @@ -319,24 +186,48 @@ GitLab [isn't limited to the SAML providers listed above](#my-identity-provider- If the information you need isn't listed above you may wish to check our [troubleshooting docs below](#i-need-additional-information-to-configure-my-identity-provider). -## Linking SAML to your existing GitLab.com account +## User access and management + +Once Group SSO is configured and enabled, users can access the GitLab.com group through the identity provider's dashboard. If [SCIM](scim_setup.md) is configured, please see the [user access and linking setup section on the SCIM page](scim_setup.md#user-access-and-linking-setup). + +When a user tries to sign in with Group SSO, they will need an account that's configured with one of the following: + +- [SCIM](scim_setup.md). +- [Group-managed accounts](group_managed_accounts.md). +- A GitLab.com account. + +### Linking SAML to your existing GitLab.com account To link SAML to your existing GitLab.com account: 1. Sign in to your GitLab.com account. -1. Locate the SSO URL for the group you are signing in to. A group Admin can find this on the group's **Settings > SAML SSO** page. -1. Visit the SSO URL and click **Authorize**. +1. Locate and visit the **GitLab single sign-on URL** for the group you are signing in to. A group Admin can find this on the group's **Settings > SAML SSO** page. If the sign-in URL is configured, users can connect to the GitLab app from the Identity Provider. +1. Click **Authorize**. 1. Enter your credentials on the Identity Provider if prompted. 1. You will be redirected back to GitLab.com and should now have access to the group. In the future, you can use SAML to sign in to GitLab.com. -## Signing in to GitLab.com with SAML +On subsequent visits, you should be able to go [sign in to GitLab.com with SAML](#signing-in-to-gitlabcom-with-saml) or by visiting links directly. If the **enforce SSO** option is turned on, you will be redirected to sign in through the identity provider. -1. Locate the SSO URL for the group you are signing in to. A group Admin can find this on a group's **Settings > SAML SSO** page. If configured, it might also be possible to sign in to GitLab starting from your Identity Provider. -1. Visit the SSO URL and click the **Sign in with Single Sign-On** button. -1. Enter your credentials on the Identity Provider if prompted. +### Signing in to GitLab.com with SAML + +1. Sign in to your identity provider. +1. From the list of apps, click on the "GitLab.com" app (The name is set by the administrator of the identity provider). 1. You will be signed in to GitLab.com and redirected to the group. -## Unlinking accounts +### Role + +The first time you sign in, GitLab adds you to the top-level parent group with the Guest role. Existing members with appropriate privileges can promote that new user. + +If a user is already a member of the group, linking the SAML identity does not change their role. + +### Blocking access + +To rescind access to the group, perform the following steps, in order: + +1. Remove the user from the user datastore on the identity provider or the list of users on the specific app. +1. Remove the user from the GitLab.com group. + +### Unlinking accounts Users can unlink SAML for a group from their profile page. This can be helpful if: @@ -359,7 +250,7 @@ For example, to unlink the `MyOrg` account, the following **Disconnect** button | Issuer | How GitLab identifies itself to the identity provider. Also known as a "Relying party trust identifier". | | Certificate fingerprint | Used to confirm that communications over SAML are secure by checking that the server is signing communications with the correct certificate. Also known as a certificate thumbprint. | -## Configuring on a self-managed GitLab instance +## Configuring on a self-managed GitLab instance **(PREMIUM ONLY)** For self-managed GitLab instances we strongly recommend using the [instance-wide SAML OmniAuth Provider](../../../integration/saml.md) instead. @@ -377,7 +268,7 @@ Group SAML on a self-managed instance is limited when compared to the recommende [instance-wide SAML](../../../integration/saml.md). The recommended solution allows you to take advantage of: - [LDAP compatibility](../../../administration/auth/ldap/index.md). -- [LDAP Group Sync](../index.md#manage-group-memberships-via-ldap) +- [LDAP Group Sync](../index.md#manage-group-memberships-via-ldap). - [Required groups](../../../integration/saml.md#required-groups-starter-only). - [Admin groups](../../../integration/saml.md#admin-groups-starter-only). - [Auditor groups](../../../integration/saml.md#auditor-groups-starter-only). @@ -449,7 +340,6 @@ Here are possible causes and solutions: | Cause | Solution | |------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | You've tried to link multiple SAML identities to the same user, for a given Identity Provider. | Change the identity that you sign in with. To do so, [unlink the previous SAML identity](#unlinking-accounts) from this GitLab account before attempting to sign in again. | -| The Identity Provider might be returning an inconsistent [NameID](#nameid). | Ask an admin of your Identity Provider to use the [SCIM API](../../../api/scim.md) to update your `extern_uid` to match the current **NameID**. | ### Message: "SAML authentication failed: Email has already been taken" @@ -463,6 +353,16 @@ Getting both of these errors at the same time suggests the NameID capitalization This can be prevented by configuring the [NameID](#nameid) to return a consistent value. Fixing this for an individual user involves [unlinking SAML in the GitLab account](#unlinking-accounts), although this will cause group membership and Todos to be lost. +### Message: "Request to link SAML account must be authorized" + +Ensure that the user who is trying to link their GitLab account has been added as a user within the identity provider's SAML app. + +### Stuck in a login "loop" + +Ensure that the **GitLab single sign-on URL** has been configured as "Login URL" (or similarly named field) in the identity provider's SAML app. + +Alternatively, when users need to [link SAML to their existing GitLab.com account](#linking-saml-to-your-existing-gitlabcom-account), provide the **GitLab single sign-on URL** and instruct users not to use the SAML app on first sign in. + ### The NameID has changed | Cause | Solution | @@ -473,17 +373,6 @@ This can be prevented by configuring the [NameID](#nameid) to return a consisten Users will need to [unlink the current SAML identity](#unlinking-accounts) and [link their identity](#user-access-and-management) to the new SAML app. -### My identity provider isn't listed - -Not a problem, the SAML standard means that a wide range of identity providers will work with GitLab. Unfortunately we aren't familiar with all of them so can only offer support configuring the [listed providers](#providers). - -Your identity provider may also have relevant documentation. It may be generic SAML documentation, or specifically targeted for GitLab. Examples: - -- [Auth0](https://auth0.com/docs/protocols/saml/saml-idp-generic) -- [G Suite](https://support.google.com/a/answer/6087519?hl=en) -- [JumpCloud](https://support.jumpcloud.com/support/s/article/single-sign-on-sso-with-gitlab-2019-08-21-10-36-47) -- [PingOne by Ping Identity](https://docs.pingidentity.com/bundle/pingone/page/xsh1564020480660-1.html) - ### I need additional information to configure my identity provider Many SAML terms can vary between providers. It is possible that the information you are looking for is listed under another name. diff --git a/doc/user/group/saml_sso/scim_setup.md b/doc/user/group/saml_sso/scim_setup.md index a891962b38e..13e9d623e2c 100644 --- a/doc/user/group/saml_sso/scim_setup.md +++ b/doc/user/group/saml_sso/scim_setup.md @@ -92,7 +92,8 @@ You can then test the connection by clicking on **Test Connection**. If the conn 1. Save your changes. For reference, you can view [an example configuration in the troubleshooting reference](../../../administration/troubleshooting/group_saml_scim.md#azure-active-directory). - NOTE: **Note:** If you used a unique identifier **other than** `objectId`, be sure to map it to `externalId`. + NOTE: **Note:** + If you used a unique identifier **other than** `objectId`, be sure to map it to `externalId`. 1. Below the mapping list click on **Show advanced options > Edit attribute list for AppName**. @@ -127,7 +128,8 @@ Before proceeding, be sure to complete the [GitLab configuration](#gitlab-config 1. If you see an **Admin** button in the top right, click the button. This will ensure you are in the Admin area. - TIP: **Tip:** If you're using the Developer Console, click **Developer Console** in the top + TIP: **Tip:** + If you're using the Developer Console, click **Developer Console** in the top bar and select **Classic UI**. Otherwise, you may not see the buttons described in the following steps: @@ -163,7 +165,7 @@ As long as [Group SAML](index.md) has been configured, prior to turning on sync, - By following these steps: 1. Sign in to GitLab.com if needed. - 1. Click on the GitLab app in the identity provider's dashboard or visit the **GitLab single sign on URL**. + 1. Click on the GitLab app in the identity provider's dashboard or visit the **GitLab single sign-on URL**. 1. Click on the **Authorize** button. New users and existing users on subsequent visits can access the group through the identify provider's dashboard or by visiting links directly. @@ -175,7 +177,7 @@ For role information, please see the [Group SAML page](index.md#user-access-and- To rescind access to the group, we recommend removing the user from the identity provider or users list for the specific app. -Upon the next sync, the user will be deprovisioned, which means that the user will be removed from the group. The user account will not be deleted unless using [group managed accounts](index.md#group-managed-accounts). +Upon the next sync, the user will be deprovisioned, which means that the user will be removed from the group. The user account will not be deleted unless using [group managed accounts](group_managed_accounts.md). ## Troubleshooting diff --git a/doc/user/group/subgroups/index.md b/doc/user/group/subgroups/index.md index 842e2082be4..a370c9cf136 100644 --- a/doc/user/group/subgroups/index.md +++ b/doc/user/group/subgroups/index.md @@ -19,10 +19,13 @@ By using subgroups you can do the following: - **Make it easier to manage people and control visibility.** Give people different [permissions](../../permissions.md#group-members-permissions) depending on their group [membership](#membership). +For more information on allowed permissions in groups and projects, see +[visibility levels](../../../development/permissions.md#general-permissions). + ## Overview A group can have many subgroups inside it, and at the same time a group can have -only 1 parent group. It resembles a directory behavior or a nested items list: +only one immediate parent group. It resembles a directory behavior or a nested items list: - Group 1 - Group 1.1 @@ -86,7 +89,7 @@ of words that are not allowed to be used as group names see the [reserved names](../../reserved_names.md). Users can always create subgroups if they are explicitly added as an Owner (or -Maintainer, if that setting is enabled) to a parent group, even if group +Maintainer, if that setting is enabled) to an immediate parent group, even if group creation is disabled by an administrator in their settings. To create a subgroup: @@ -96,9 +99,9 @@ To create a subgroup:  -1. Create a new group like you would normally do. Notice that the parent group +1. Create a new group like you would normally do. Notice that the immediate parent group namespace is fixed under **Group path**. The visibility level can differ from - the parent group. + the immediate parent group.  @@ -110,12 +113,13 @@ Follow the same process to create any subsequent groups. ## Membership When you add a member to a subgroup, they inherit the membership and permission -level from the parent group. This model allows access to nested groups if you +level from the parent group(s). This model allows access to nested groups if you have membership in one of its parents. -Jobs for pipelines in subgroups can use [Runners](../../../ci/runners/README.md) registered to the parent group. This means secrets configured for the parent group are available to subgroup jobs. +Jobs for pipelines in subgroups can use [Runners](../../../ci/runners/README.md) registered to the parent group(s). +This means secrets configured for the parent group are available to subgroup jobs. -In addition, maintainers of projects that belong to subgroups can see the details of Runners registered to parent groups. +In addition, maintainers of projects that belong to subgroups can see the details of Runners registered to parent group(s). The group permissions for a member can be changed only by Owners, and only on the **Members** page of the group the member was added. diff --git a/doc/user/incident_management/img/pagerduty_incidents_integration_13_2.png b/doc/user/incident_management/img/pagerduty_incidents_integration_13_2.png Binary files differnew file mode 100644 index 00000000000..9277b013676 --- /dev/null +++ b/doc/user/incident_management/img/pagerduty_incidents_integration_13_2.png diff --git a/doc/user/incident_management/index.md b/doc/user/incident_management/index.md index 48805bda909..996f423e770 100644 --- a/doc/user/incident_management/index.md +++ b/doc/user/incident_management/index.md @@ -25,7 +25,7 @@ to create issues when alerts are triggered: checkbox to create an issue based on your own [issue templates](../project/description_templates.md#creating-issue-templates). For more information, see - [Taking Action on Incidents](../project/integrations/prometheus.md#taking-action-on-incidents-ultimate) **(ULTIMATE)**. + [Trigger actions from alerts](../../operations/metrics/alerts.md#trigger-actions-from-alerts-ultimate) **(ULTIMATE)**. 1. To create issues from alerts, select the template in the **Issue Template** select box. 1. To send [separate email notifications](#notify-developers-of-alerts) to users @@ -34,9 +34,9 @@ to create issues when alerts are triggered: 1. Click **Save changes**. Appropriately configured alerts include an -[embedded chart](../project/integrations/prometheus.md#embedding-metrics-based-on-alerts-in-incident-issues) +[embedded chart](../../operations/metrics/embed.md#embedding-metrics-based-on-alerts-in-incident-issues) for the query corresponding to the alert. You can also configure GitLab to -[close issues](../project/integrations/prometheus.md#taking-action-on-incidents-ultimate) +[close issues](../../operations/metrics/alerts.md#trigger-actions-from-alerts-ultimate) when you receive notification that the alert is resolved. ### Notify developers of alerts @@ -49,12 +49,35 @@ These emails contain details of the alert, and a link for more information. To send separate email notifications to users with [Developer permissions](../permissions.md), see [Configure incidents](#configure-incidents-ultimate). +## Configure PagerDuty integration + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/119018) in GitLab 13.2. + +You can set up a webhook with PagerDuty to automatically create a GitLab issue +for each PagerDuty incident. This configuration requires you to make changes +in both PagerDuty and GitLab: + +1. Sign in as a user with Maintainer [permissions](../permissions.md). +1. Navigate to **{settings}** **Settings > Operations > Incidents** and expand **Incidents**. +1. Select the **PagerDuty integration** tab: + +  + +1. Activate the integration, and save the changes in GitLab. +1. Copy the value of **Webhook URL**, as you'll need it in a later step. +1. Follow the steps described in the + [PagerDuty documentation](https://support.pagerduty.com/docs/webhooks) + to add the webhook URL to a PagerDuty webhook integration. + +To confirm the integration is successful, trigger a test incident from PagerDuty to +confirm that a GitLab issue is created from the incident. + ## Configure Prometheus alerts You can set up Prometheus alerts in: -- [GitLab-managed Prometheus](../project/integrations/prometheus.md#setting-up-alerts-for-prometheus-metrics) installations. -- [Self-managed Prometheus](../project/integrations/prometheus.md#external-prometheus-instances) installations. +- [GitLab-managed Prometheus](../../operations/metrics/alerts.md) installations. +- [Self-managed Prometheus](../../operations/metrics/alerts.md#external-prometheus-instances) installations. Prometheus alerts are created by the special Alert Bot user. You can't remove this user, but it does not count toward your license limit. @@ -71,11 +94,11 @@ You can embed metrics anywhere [GitLab Markdown](../markdown.md) is used, such a comments on issues, and merge requests. Embedding metrics helps you share them when discussing incidents or performance issues. You can output the dashboard directly into any issue, merge request, epic, or any other Markdown text field in GitLab -by [copying and pasting the link to the metrics dashboard](../project/integrations/prometheus.md#embedding-gitlab-managed-kubernetes-metrics). +by [copying and pasting the link to the metrics dashboard](../../operations/metrics/embed.md#embedding-gitlab-managed-kubernetes-metrics). You can embed both -[GitLab-hosted metrics](../project/integrations/prometheus.md#embedding-metric-charts-within-gitlab-flavored-markdown) and -[Grafana metrics](../project/integrations/prometheus.md#embedding-grafana-charts) +[GitLab-hosted metrics](../../operations/metrics/embed.md) and +[Grafana metrics](../../operations/metrics/embed_grafana.md) in incidents and issue templates. ### Context menu @@ -86,7 +109,7 @@ above the upper right corner of the panel. The options are: - [View logs](#view-logs-from-metrics-panel). - **Download CSV** - Data from embedded charts can be - [downloaded as CSV](../project/integrations/prometheus.md#downloading-data-as-csv). + [downloaded as CSV](../../operations/metrics/dashboards/index.md#downloading-data-as-csv). #### View logs from metrics panel @@ -94,7 +117,7 @@ above the upper right corner of the panel. The options are: > - [Moved](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/25455) to [GitLab Core](https://about.gitlab.com/pricing/) 12.9. Viewing logs from a metrics panel can be useful if you're triaging an application -incident and need to [explore logs](../project/integrations/prometheus.md#view-logs-ultimate) +incident and need to [explore logs](../../operations/metrics/dashboards/index.md#view-logs-ultimate) from across your application. These logs help you understand what is affecting your application's performance and resolve any problems. diff --git a/doc/user/index.md b/doc/user/index.md index cc521c2a767..f50b712e2c3 100644 --- a/doc/user/index.md +++ b/doc/user/index.md @@ -46,10 +46,10 @@ GitLab is a Git-based platform that integrates a great number of essential tools - Deploying personal and professional static websites with [GitLab Pages](project/pages/index.md). - Integrating with Docker by using [GitLab Container Registry](packages/container_registry/index.md). - Tracking the development lifecycle by using [GitLab Value Stream Analytics](project/cycle_analytics.md). +- Provide support with [Service Desk](project/service_desk.md). With GitLab Enterprise Edition, you can also: -- Provide support with [Service Desk](project/service_desk.md). - Improve collaboration with: - [Merge Request Approvals](project/merge_requests/merge_request_approvals.md). **(STARTER)** - [Multiple Assignees for Issues](project/issues/multiple_assignees_for_issues.md). **(STARTER)** @@ -148,7 +148,7 @@ requests you're assigned to. [Snippets](snippets.md) are code blocks that you want to store in GitLab, from which you have quick access to. You can also gather feedback on them through -[Discussions](#Discussions). +[Discussions](#discussions). ## Keyboard shortcuts diff --git a/doc/user/infrastructure/img/terraform_plan_widget_v13_2.png b/doc/user/infrastructure/img/terraform_plan_widget_v13_2.png Binary files differnew file mode 100644 index 00000000000..564835a5dbe --- /dev/null +++ b/doc/user/infrastructure/img/terraform_plan_widget_v13_2.png diff --git a/doc/user/infrastructure/index.md b/doc/user/infrastructure/index.md index c17d1831b50..e24e669d994 100644 --- a/doc/user/infrastructure/index.md +++ b/doc/user/infrastructure/index.md @@ -6,8 +6,17 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Infrastructure as code with Terraform and GitLab +## Motivation + +The Terraform integration features within GitLab enable your GitOps / Infrastructure-as-Code (IaC) +workflows to tie into GitLab's authentication and authorization. These features focus on +lowering the barrier to entry for teams to adopt Terraform, collaborate effectively within +GitLab, and support Terraform best practices. + ## GitLab managed Terraform State +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/2673) in GitLab 13.0. + [Terraform remote backends](https://www.terraform.io/docs/backends/index.html) enable you to store the state file in a remote, shared store. GitLab uses the [Terraform HTTP backend](https://www.terraform.io/docs/backends/types/http.html) @@ -27,6 +36,14 @@ To get started with a GitLab-managed Terraform State, there are two different op - [Use a local machine](#get-started-using-local-development). - [Use GitLab CI](#get-started-using-gitlab-ci). +## Permissions for using Terraform + +In GitLab version 13.1, [Maintainer access](../permissions.md) was required to use a +GitLab managed Terraform state backend. In GitLab versions 13.2 and greater, +[Maintainer access](../permissions.md) is required to lock, unlock and write to the state +(using `terraform apply`), while [Developer access](../permissions.md) is required to read +the state (using `terraform plan -lock=false`). + ## Get started using local development If you plan to only run `terraform plan` and `terraform apply` commands from your @@ -45,8 +62,7 @@ local machine, this is a simple way to get started: ``` 1. Create a [Personal Access Token](../profile/personal_access_tokens.md) with - the `api` scope. The Terraform backend is restricted to users with - [Maintainer access](../permissions.md) to the repository. + the `api` scope. 1. On your local machine, run `terraform init`, passing in the following options, replacing `<YOUR-PROJECT-NAME>`, `<YOUR-PROJECT-ID>`, `<YOUR-USERNAME>` and @@ -80,10 +96,6 @@ Next, [configure the backend](#configure-the-backend). After executing the `terraform init` command, you must configure the Terraform backend and the CI YAML file: -CAUTION: **Important:** -The Terraform backend is restricted to users with [Maintainer access](../permissions.md) -to the repository. - 1. In your Terraform project, define the [HTTP backend](https://www.terraform.io/docs/backends/types/http.html) by adding the following code block in a `.tf` file (such as `backend.tf`) to define the remote backend: @@ -95,64 +107,75 @@ to the repository. } ``` -1. In the root directory of your project repository, configure a `.gitlab-ci.yaml` file. - This example uses a pre-built image: +1. In the root directory of your project repository, configure a + `.gitlab-ci.yaml` file. This example uses a pre-built image which includes a + `gitlab-terraform` helper. For supported Terraform versions, see the [GitLab + Terraform Images project](https://gitlab.com/gitlab-org/terraform-images). ```yaml - image: - name: hashicorp/terraform:light - entrypoint: - - '/usr/bin/env' - - 'PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' + image: registry.gitlab.com/gitlab-org/terraform-images/stable:latest ``` -1. In the `.gitlab-ci.yaml` file, define some environment variables to ease development. In this - example, `GITLAB_TF_ADDRESS` is the URL of the GitLab instance where this pipeline - runs, and `TF_ROOT` is the directory where the Terraform commands must be executed: +1. In the `.gitlab-ci.yaml` file, define some environment variables to ease + development. In this example, `TF_ROOT` is the directory where the Terraform + commands must be executed, `TF_ADDRESS` is the URL to the state on the GitLab + instance where this pipeline runs, and the final path segment in `TF_ADDRESS` + is the name of the Terraform state. Projects may have multiple states, and + this name is arbitrary, so in this example we will set it to the name of the + project, and we will ensure that the `.terraform` directory is cached between + jobs in the pipeline using a cache key based on the state name: ```yaml variables: - GITLAB_TF_ADDRESS: ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/terraform/state/${CI_PROJECT_NAME} TF_ROOT: ${CI_PROJECT_DIR}/environments/cloudflare/production + TF_ADDRESS: ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/terraform/state/${CI_PROJECT_NAME} cache: + key: ${CI_PROJECT_NAME} paths: - - .terraform + - ${TF_ROOT}/.terraform ``` -1. In a `before_script`, pass a `terraform init` call containing configuration parameters - corresponding to variables required by the - [HTTP backend](https://www.terraform.io/docs/backends/types/http.html): +1. In a `before_script`, change to your `TF_ROOT`: ```yaml before_script: - cd ${TF_ROOT} - - terraform --version - - terraform init -backend-config="address=${GITLAB_TF_ADDRESS}" -backend-config="lock_address=${GITLAB_TF_ADDRESS}/lock" -backend-config="unlock_address=${GITLAB_TF_ADDRESS}/lock" -backend-config="username=gitlab-ci-token" -backend-config="password=${CI_JOB_TOKEN}" -backend-config="lock_method=POST" -backend-config="unlock_method=DELETE" -backend-config="retry_wait_min=5" stages: + - prepare - validate - build - - test - deploy + init: + stage: prepare + script: + - gitlab-terraform init + validate: stage: validate script: - - terraform validate + - gitlab-terraform validate plan: stage: build script: - - terraform plan - - terraform show + - gitlab-terraform plan + - gitlab-terraform plan-json + artifacts: + name: plan + paths: + - ${TF_ROOT}/plan.cache + reports: + terraform: ${TF_ROOT}/plan.json apply: stage: deploy environment: name: production script: - - terraform apply + - gitlab-terraform apply dependencies: - plan when: manual @@ -160,8 +183,9 @@ to the repository. - master ``` -1. Push your project to GitLab, which triggers a CI job pipeline. This pipeline runs - the `terraform init`, `terraform validate`, and `terraform plan` commands. +1. Push your project to GitLab, which triggers a CI job pipeline. This pipeline + runs the `gitlab-terraform init`, `gitlab-terraform validate`, and + `gitlab-terraform plan` commands. The output from the above `terraform` commands should be viewable in the job logs. @@ -176,15 +200,18 @@ you can expose details from `terraform plan` runs directly into a merge request enabling you to see statistics about the resources that Terraform will create, modify, or destroy. -Let's explore how to configure a GitLab Terraform Report artifact: +Let's explore how to configure a GitLab Terraform Report artifact. You can +either use a pre-built image which includes a `gitlab-terraform` helper as +above, where `gitlab-terraform plan-json` outputs the required artifact, or you +can configure this manually as follows: 1. For simplicity, let's define a few reusable variables to allow us to refer to these files multiple times: ```yaml variables: - PLAN: plan.tfplan - PLAN_JSON: tfplan.json + PLAN: plan.cache + PLAN_JSON: plan.json ``` 1. Install `jq`, a @@ -198,6 +225,18 @@ Let's explore how to configure a GitLab Terraform Report artifact: - alias convert_report="jq -r '([.resource_changes[]?.change.actions?]|flatten)|{\"create\":(map(select(.==\"create\"))|length),\"update\":(map(select(.==\"update\"))|length),\"delete\":(map(select(.==\"delete\"))|length)}'" ``` + NOTE: **Note:** + In distributions that use Bash (for example, Ubuntu), `alias` statements are not + expanded in non-interactive mode. If your pipelines fail with the error + `convert_report: command not found`, alias expansion can be activated explicitly + by adding a `shopt` command to your script: + + ```yaml + before_script: + - shopt -s expand_aliases + - alias convert_report="jq -r '([.resource_changes[]?.change.actions?]|flatten)|{\"create\":(map(select(.==\"create\"))|length),\"update\":(map(select(.==\"update\"))|length),\"delete\":(map(select(.==\"delete\"))|length)}'" + ``` + 1. Define a `script` that runs `terraform plan` and `terraform show`. These commands pipe the output and convert the relevant bits into a store variable `PLAN_JSON`. This JSON is used to create a @@ -212,18 +251,18 @@ Let's explore how to configure a GitLab Terraform Report artifact: - terraform plan -out=$PLAN - terraform show --json $PLAN | convert_report > $PLAN_JSON artifacts: - name: plan - paths: - - $PLAN reports: terraform: $PLAN_JSON ``` - For a full example, see [Example `.gitlab-ci.yaml` file](#example-gitlab-ciyaml-file). + For a full example using the pre-built image, see [Example `.gitlab-ci.yaml` + file](#example-gitlab-ciyaml-file). + + For an example displaying multiple reports, see [`.gitlab-ci.yaml` multiple reports file](#multiple-terraform-plan-reports). 1. Running the pipeline displays the widget in the merge request, like this: -  +  1. Clicking the **View Full Log** button in the widget takes you directly to the plan output present in the pipeline logs: @@ -233,64 +272,114 @@ Let's explore how to configure a GitLab Terraform Report artifact: ### Example `.gitlab-ci.yaml` file ```yaml -image: - name: hashicorp/terraform:light - entrypoint: - - '/usr/bin/env' - - 'PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' +image: registry.gitlab.com/gitlab-org/terraform-images/stable:latest -# Default output file for Terraform plan variables: - GITLAB_TF_ADDRESS: ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/terraform/state/${CI_PROJECT_NAME} - PLAN: plan.tfplan - PLAN_JSON: tfplan.json - TF_ROOT: ${CI_PROJECT_DIR} + TF_ROOT: ${CI_PROJECT_DIR}/environments/cloudflare/production + TF_ADDRESS: ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/terraform/state/${CI_PROJECT_NAME} cache: + key: ${CI_PROJECT_NAME} paths: - - .terraform + - ${TF_ROOT}/.terraform before_script: - - apk --no-cache add jq - - alias convert_report="jq -r '([.resource_changes[]?.change.actions?]|flatten)|{\"create\":(map(select(.==\"create\"))|length),\"update\":(map(select(.==\"update\"))|length),\"delete\":(map(select(.==\"delete\"))|length)}'" - cd ${TF_ROOT} - - terraform --version - - terraform init -backend-config="address=${GITLAB_TF_ADDRESS}" -backend-config="lock_address=${GITLAB_TF_ADDRESS}/lock" -backend-config="unlock_address=${GITLAB_TF_ADDRESS}/lock" -backend-config="username=${GITLAB_USER_LOGIN}" -backend-config="password=${GITLAB_TF_PASSWORD}" -backend-config="lock_method=POST" -backend-config="unlock_method=DELETE" -backend-config="retry_wait_min=5" stages: + - prepare - validate - build - deploy +init: + stage: prepare + script: + - gitlab-terraform init + validate: stage: validate script: - - terraform validate + - gitlab-terraform validate plan: stage: build script: - - terraform plan -out=$PLAN - - terraform show --json $PLAN | convert_report > $PLAN_JSON + - gitlab-terraform plan + - gitlab-terraform plan-json artifacts: name: plan paths: - - ${TF_ROOT}/plan.tfplan + - ${TF_ROOT}/plan.cache reports: - terraform: ${TF_ROOT}/tfplan.json + terraform: ${TF_ROOT}/plan.json -# Separate apply job for manual launching Terraform as it can be destructive -# action. apply: stage: deploy environment: name: production script: - - terraform apply -input=false $PLAN + - gitlab-terraform apply dependencies: - plan when: manual only: - master +``` + +### Multiple Terraform Plan reports + +Starting with 13.2, you can display mutiple reports on the Merge Request page. The reports will also display the `artifact: name:`. See example below for a suggested setup. + +```yaml +image: + name: registry.gitlab.com/gitlab-org/gitlab-build-images:terraform + entrypoint: + - '/usr/bin/env' + - 'PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' + +cache: + paths: + - .terraform +stages: + - build + +.terraform-plan-generation: + stage: build + variables: + PLAN: plan.tfplan + JSON_PLAN_FILE: tfplan.json + before_script: + - cd ${TERRAFORM_DIRECTORY} + - terraform --version + - terraform init + - apk --no-cache add jq + script: + - terraform validate + - terraform plan -out=${PLAN} + - terraform show --json ${PLAN} | jq -r '([.resource_changes[]?.change.actions?]|flatten)|{"create":(map(select(.=="create"))|length),"update":(map(select(.=="update"))|length),"delete":(map(select(.=="delete"))|length)}' > ${JSON_PLAN_FILE} + artifacts: + reports: + terraform: ${TERRAFORM_DIRECTORY}/${JSON_PLAN_FILE} + +review_plan: + extends: .terraform-plan-generation + variables: + TERRAFORM_DIRECTORY: "review/" + # Review will not include an artifact name + +staging_plan: + extends: .terraform-plan-generation + variables: + TERRAFORM_DIRECTORY: "staging/" + artifacts: + name: Staging + +production_plan: + extends: .terraform-plan-generation + variables: + TERRAFORM_DIRECTORY: "production/" + artifacts: + name: Production ``` diff --git a/doc/user/markdown.md b/doc/user/markdown.md index 0d028cfe77a..b2b3f0f925c 100644 --- a/doc/user/markdown.md +++ b/doc/user/markdown.md @@ -14,7 +14,8 @@ website uses an extended Kramdown gem, [GitLab Kramdown](https://gitlab.com/gitl Consult the [GitLab Kramdown Guide](https://about.gitlab.com/handbook/markdown-guide/) for a complete Kramdown reference. -NOTE: **Note:** We encourage you to view this document as [rendered by GitLab itself](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/user/markdown.md). +NOTE: **Note:** +We encourage you to view this document as [rendered by GitLab itself](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/user/markdown.md). ## GitLab Flavored Markdown (GFM) @@ -54,7 +55,7 @@ repository that were written using some of the nuances of GitLab's RedCarpet ver of Markdown. Since CommonMark uses slightly stricter syntax, these documents might now appear a little differently since we have transitioned to CommonMark. -It's usually quite easy to fix. For example, numbered lists with nested lists may +For example, numbered lists with nested lists may render incorrectly: ```markdown @@ -63,8 +64,8 @@ render incorrectly: - milk ``` -Simply add a space to each nested item to align the `-` with the first character of -the top list item (`C` in this case): +To correct their rendering, add a space to each nested item to align the `-` with the first +character of the top list item (`C` in this case): ```markdown 1. Chocolate @@ -76,7 +77,8 @@ the top list item (`C` in this case): - dark - milk -NOTE: **Note:** We will flag any significant differences between Redcarpet and CommonMark +NOTE: **Note:** +We will flag any significant differences between Redcarpet and CommonMark Markdown in this document. If you have a large volume of Markdown files, it can be tedious to determine @@ -92,7 +94,7 @@ to change. GitLab makes full use of the standard (CommonMark) formatting, but also includes additional functionality useful for GitLab users. -It makes use of [new Markdown features](#new-GFM-markdown-extensions), +It makes use of [new Markdown features](#new-gfm-markdown-extensions), not found in standard Markdown: - [Color "chips" written in HEX, RGB or HSL](#colors) @@ -241,7 +243,7 @@ Sometimes you want to :monkey: around a bit and add some :star2: to your :speech You can use it to point out a :bug: or warn about :speak_no_evil: patches. And if someone improves your really :snail: code, send them some :birthday:. People will :heart: you for that. -If you're new to this, don't be :fearful:. You can easily join the emoji :family:. All you need to do is to look up one of the supported codes. +If you're new to this, don't be :fearful:. You can join the emoji :family:. All you need to do is to look up one of the supported codes. Consult the [Emoji Cheat Sheet](https://www.emojicopy.com) for a list of all supported emoji codes. :thumbsup: ``` @@ -252,7 +254,7 @@ Sometimes you want to <img src="https://gitlab.com/gitlab-org/gitlab-foss/raw/ma You can use it to point out a <img src="https://gitlab.com/gitlab-org/gitlab-foss/raw/master/app/assets/images/emoji/bug.png" width="20px" height="20px" style="display:inline;margin:0"> or warn about <img src="https://gitlab.com/gitlab-org/gitlab-foss/raw/master/app/assets/images/emoji/speak_no_evil.png" width="20px" height="20px" style="display:inline;margin:0"> patches. And if someone improves your really <img src="https://gitlab.com/gitlab-org/gitlab-foss/raw/master/app/assets/images/emoji/snail.png" width="20px" height="20px" style="display:inline;margin:0"> code, send them some <img src="https://gitlab.com/gitlab-org/gitlab-foss/raw/master/app/assets/images/emoji/birthday.png" width="20px" height="20px" style="display:inline;margin:0">. People will <img src="https://gitlab.com/gitlab-org/gitlab-foss/raw/master/app/assets/images/emoji/heart.png" width="20px" height="20px" style="display:inline;margin:0"> you for that. -If you're new to this, don't be <img src="https://gitlab.com/gitlab-org/gitlab-foss/raw/master/app/assets/images/emoji/fearful.png" width="20px" height="20px" style="display:inline;margin:0">. You can easily join the emoji <img src="https://gitlab.com/gitlab-org/gitlab-foss/raw/master/app/assets/images/emoji/family.png" width="20px" height="20px" style="display:inline;margin:0">. All you need to do is to look up one of the supported codes. +If you're new to this, don't be <img src="https://gitlab.com/gitlab-org/gitlab-foss/raw/master/app/assets/images/emoji/fearful.png" width="20px" height="20px" style="display:inline;margin:0">. You can join the emoji <img src="https://gitlab.com/gitlab-org/gitlab-foss/raw/master/app/assets/images/emoji/family.png" width="20px" height="20px" style="display:inline;margin:0">. All you need to do is to look up one of the supported codes. Consult the [Emoji Cheat Sheet](https://www.webfx.com/tools/emoji-cheat-sheet/) for a list of all supported emoji codes. <img src="https://gitlab.com/gitlab-org/gitlab-foss/raw/master/app/assets/images/emoji/thumbsup.png" width="20px" height="20px" style="display:inline;margin:0"> @@ -261,7 +263,8 @@ when rendered within GitLab, may appear different depending on the OS and browse Most emoji are natively supported on macOS, Windows, iOS, Android, and will fall back on image-based emoji where there is no support. -NOTE: **Note:** On Linux, you can download [Noto Color Emoji](https://www.google.com/get/noto/help/emoji/) +NOTE: **Note:** +On Linux, you can download [Noto Color Emoji](https://www.google.com/get/noto/help/emoji/) to get full native emoji support. Ubuntu 18.04 (like many modern Linux distributions) has this font installed by default. @@ -402,14 +405,15 @@ a^2+b^2=c^2 _Be advised that KaTeX only supports a [subset](https://katex.org/docs/supported.html) of LaTeX._ -NOTE: **Note:** This also works for the Asciidoctor `:stem: latexmath`. For details see +NOTE: **Note:** +This also works for the Asciidoctor `:stem: latexmath`. For details see the [Asciidoctor user manual](https://asciidoctor.org/docs/user-manual/#activating-stem-support). ### Special GitLab references -GFM recognizes special GitLab related references. For example, you can easily reference +GFM recognizes special GitLab related references. For example, you can reference an issue, a commit, a team member, or even the whole team within a project. GFM will turn -that reference into a link so you can navigate between them easily. +that reference into a link so you can navigate between them. Additionally, GFM recognizes certain cross-project references and also has a shorthand version to reference other projects from the same namespace. @@ -502,8 +506,6 @@ Second section content.  ---- - ### Wiki-specific Markdown The following examples show how links inside wikis behave. @@ -581,7 +583,7 @@ This snippet links to `<wiki_root>/miscellaneous.md`: ### Embedding metrics in GitLab Flavored Markdown -Metric charts can be embedded within GitLab Flavored Markdown. See [Embedding Metrics within GitLab flavored Markdown](../user/project/integrations/prometheus.md#embedding-metric-charts-within-gitlab-flavored-markdown) for more details. +Metric charts can be embedded within GitLab Flavored Markdown. See [Embedding Metrics within GitLab flavored Markdown](../operations/metrics/embed.md#embedding-metric-charts-within-gitlab-flavored-markdown) for more details. ## Standard Markdown and extensions in GitLab @@ -591,7 +593,7 @@ If a functionality is extended, the new option will be listed as a sub-section. ### Blockquotes -Blockquotes are an easy way to highlight information, such as a side-note. It's generated +Blockquotes are useful to highlight information, such as a side-note. It's generated by starting the lines of the blockquote with `>`: ```markdown @@ -637,9 +639,9 @@ you can quote that without having to manually prepend `>` to every line! ### Code spans and blocks -You can easily highlight anything that should be viewed as code and not simple text. +You can highlight anything that should be viewed as code and not simple text. -Simple inline code is easily highlighted with single backticks `` ` ``: +Simple inline code is highlighted with single backticks `` ` ``: ```markdown Inline `code` has `back-ticks around` it. @@ -781,7 +783,8 @@ Combined emphasis with **asterisks and _underscores_**. Strikethrough uses two tildes. ~~Scratch this.~~ -NOTE: **Note:** Strikethrough is not part of the core Markdown standard, but is part of GFM. +NOTE: **Note:** +Strikethrough is not part of the core Markdown standard, but is part of GFM. #### Multiple underscores in words and mid-word emphasis @@ -1150,7 +1153,7 @@ A line break will be inserted (a new paragraph will start) if the previous text ended with two newlines, like when you hit <kbd>Enter</kbd> twice in a row. If you only use one newline (hit <kbd>Enter</kbd> once), the next sentence will be part of the same paragraph. This is useful if you want to keep long lines from wrapping, and keep -them easily editable: +them editable: ```markdown Here's a line for us to start with. @@ -1248,7 +1251,8 @@ Do not change to reference style links. Some text to show that the reference links can follow later. -NOTE: **Note:** Relative links do not allow the referencing of project files in a wiki +NOTE: **Note:** +Relative links do not allow the referencing of project files in a wiki page, or a wiki page in a project file. The reason for this is that a wiki is always in a separate Git repository in GitLab. For example, `[I'm a reference-style link](style)` will point the link to `wikis/style` only when the link is inside of a wiki Markdown file. @@ -1275,7 +1279,7 @@ GFM will auto-link almost any URL you put into your text: ### Lists -Ordered and unordered lists can be easily created. +Ordered and unordered lists can be created. For an ordered list, add the number you want the list to start with, like `1.`, followed by a space, at the start of each line for ordered lists. diff --git a/doc/user/packages/composer_repository/index.md b/doc/user/packages/composer_repository/index.md index 8a7c70ec74d..542efba1fae 100644 --- a/doc/user/packages/composer_repository/index.md +++ b/doc/user/packages/composer_repository/index.md @@ -6,7 +6,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w # GitLab Composer Repository **(PREMIUM)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/15886) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.1. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/15886) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.2. With the GitLab Composer Repository, every project can have its own space to store [Composer](https://getcomposer.org/) packages. @@ -19,7 +19,7 @@ This option is available only if your GitLab administrator has After the Composer Repository is enabled, it will be available for all new projects by default. To enable it for existing projects, or if you want to disable it: -1. Navigate to your project's **Settings > General > Permissions**. +1. Navigate to your project's **Settings > General > Visibility, project features, permissions**. 1. Find the Packages feature and enable or disable it. 1. Click on **Save changes** for the changes to take effect. diff --git a/doc/user/packages/conan_repository/index.md b/doc/user/packages/conan_repository/index.md index 020028dfc10..41b420ce7f6 100644 --- a/doc/user/packages/conan_repository/index.md +++ b/doc/user/packages/conan_repository/index.md @@ -22,7 +22,7 @@ This option is available only if your GitLab administrator has After the Conan Repository is enabled, it will be available for all new projects by default. To enable it for existing projects, or if you want to disable it: -1. Navigate to your project's **Settings > General > Permissions**. +1. Navigate to your project's **Settings > General > Visibility, project features, permissions**. 1. Find the Packages feature and enable or disable it. 1. Click on **Save changes** for the changes to take effect. @@ -85,7 +85,7 @@ Next, you will create a package for that recipe by running `conan create` provid conan create . my-org+my-group+my-project/beta ``` -NOTE: **Note** +NOTE: **Note:** Current [naming restrictions](#package-recipe-naming-convention) require you to name the `user` value as the `+` separated path of your project on GitLab. The example above would create a package belonging to this project: `https://gitlab.com/my-org/my-group/my-project` with a channel of `beta`. @@ -129,12 +129,12 @@ Once you have a personal access token and have [set your Conan remote](#adding-t conan user <gitlab_username or deploy_token_username> -r gitlab -p <personal_access_token or deploy_token> ``` -Note: **Note** +NOTE: **Note:** If you named your remote something other than `gitlab`, your remote name should be used in this command instead of `gitlab`. From now on, when you run commands using `--remote=gitlab`, your username and password will automatically be included in the requests. -Note: **Note** +NOTE: **Note:** The personal access token is not stored locally at any moment. Conan uses JWT, so when you run this command, Conan will request an expirable token from GitLab using your token. The JWT does expire on a regular basis, so you will need to re-enter your personal access token when that happens. Alternatively, you could explicitly include your credentials in any given command. @@ -152,7 +152,7 @@ If you'd like Conan to always use GitLab as the registry for your package, you c conan remote add_ref Hello/0.1@my-group+my-project/beta gitlab ``` -NOTE: **Note** +NOTE: **Note:** The package recipe does include the version, so setting the default remote for `Hello/0.1@user/channel` will not work for `Hello/0.2@user/channel`. This functionality is best suited for when you want to consume or install packages from the GitLab registry without having to specify a remote. diff --git a/doc/user/packages/container_registry/img/expiration_policy_app_v13_0.png b/doc/user/packages/container_registry/img/expiration_policy_app_v13_0.png Binary files differdeleted file mode 100644 index 93c9e00a8f5..00000000000 --- a/doc/user/packages/container_registry/img/expiration_policy_app_v13_0.png +++ /dev/null diff --git a/doc/user/packages/container_registry/index.md b/doc/user/packages/container_registry/index.md index eb1933de62a..429d29b7677 100644 --- a/doc/user/packages/container_registry/index.md +++ b/doc/user/packages/container_registry/index.md @@ -71,7 +71,7 @@ This view will: - Allow you to [delete](#delete-images-from-within-gitlab) one or more image repository. - Allow you to navigate to the image repository details page. - Show a **Quick start** dropdown with the most common commands to log in, build and push -- Optionally, a banner will be visible if the [expiration policy](#expiration-policy) is enabled for this project. +- Optionally, a banner will be visible if the [cleanup policy](#cleanup-policy) is enabled for this project. ### Control Container Registry for your group @@ -248,10 +248,10 @@ should look similar to this: ```yaml build: - image: docker:19.03.11 + image: docker:19.03.12 stage: build services: - - docker:19.03.11-dind + - docker:19.03.12-dind script: - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - docker build -t $CI_REGISTRY/group/project/image:latest . @@ -262,10 +262,10 @@ You can also make use of [other variables](../../../ci/variables/README.md) to a ```yaml build: - image: docker:19.03.11 + image: docker:19.03.12 stage: build services: - - docker:19.03.11-dind + - docker:19.03.12-dind variables: IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG script: @@ -288,9 +288,9 @@ when needed. Changes to `master` also get tagged as `latest` and deployed using an application-specific deploy script: ```yaml -image: docker:19.03.11 +image: docker:19.03.12 services: - - docker:19.03.11-dind + - docker:19.03.12-dind stages: - build @@ -363,9 +363,9 @@ Below is an example of what your `.gitlab-ci.yml` should look like: ```yaml build: - image: $CI_REGISTRY/group/project/docker:19.03.11 + image: $CI_REGISTRY/group/project/docker:19.03.12 services: - - name: $CI_REGISTRY/group/project/docker:19.03.11-dind + - name: $CI_REGISTRY/group/project/docker:19.03.12-dind alias: docker stage: build script: @@ -373,7 +373,7 @@ Below is an example of what your `.gitlab-ci.yml` should look like: - docker run my-docker-image /script/to/run/tests ``` -If you forget to set the service alias, the `docker:19.03.11` image won't find the +If you forget to set the service alias, the `docker:19.03.12` image won't find the `dind` service, and an error like the following will be thrown: ```plaintext @@ -443,10 +443,10 @@ stages: - clean build_image: - image: docker:19.03.11 + image: docker:19.03.12 stage: build services: - - docker:19.03.11-dind + - docker:19.03.12-dind variables: IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG script: @@ -459,10 +459,10 @@ build_image: - master delete_image: - image: docker:19.03.11 + image: docker:19.03.12 stage: clean services: - - docker:19.03.11-dind + - docker:19.03.12-dind variables: IMAGE_TAG: $CI_PROJECT_PATH:$CI_COMMIT_REF_SLUG REG_SHA256: ade837fc5224acd8c34732bf54a94f579b47851cc6a7fd5899a98386b782e228 @@ -486,25 +486,33 @@ You can download the latest `reg` release from the code example by changing the `REG_SHA256` and `REG_VERSION` variables defined in the `delete_image` job. -### Delete images using an expiration policy +### Delete images by using a cleanup policy -You can create a per-project [expiration policy](#expiration-policy) to ensure -older tags and images are regularly removed from the Container Registry. +You can create a per-project [cleanup policy](#cleanup-policy) to ensure older tags and images are regularly removed from the +Container Registry. -## Expiration policy +## Cleanup policy -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/15398) in GitLab 12.8. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/15398) in GitLab 12.8. +> - [Renamed](https://gitlab.com/gitlab-org/gitlab/-/issues/218737) from "expiration policy" to "cleanup policy" in GitLab 13.2. + +For a specific project, if you want to remove tags you no longer need, +you can create a cleanup policy. When the policy is applied, tags matching the regex pattern are removed. +The underlying layers and images remain. + +To delete the underlying layers and images no longer associated with any tags, Instance Administrators can use +[garbage collection](../../../administration/packages/container_registry.md#removing-unused-layers-not-referenced-by-manifests) with the `-m` switch. NOTE: **Note:** -For GitLab.com, expiration policies are not available for projects created before GitLab 12.8. -For self-managed instances, expiration policies may be enabled by an admin in the -[CI/CD Package Registry settings](./../../admin_area/settings/index.md#cicd). +For GitLab.com, cleanup policies are not available for projects created +before this feature was deployed to production (February 2020). +Support for pre-existing projects on GitLab.com +[is planned](https://gitlab.com/gitlab-org/gitlab/-/issues/196124). +For self-managed instances, cleanup policies may be enabled by an admin in the +[GitLab application settings](../../../api/settings.md#change-application-settings) by setting `container_expiration_policies_enable_historic_entries` to true. Note the inherent [risks involved](./index.md#use-with-external-container-registries). -It is possible to create a per-project expiration policy, so that you can make sure that -older tags and images are regularly removed from the Container Registry. - -The expiration policy algorithm starts by collecting all the tags for a given repository in a list, +The cleanup policy algorithm starts by collecting all the tags for a given repository in a list, then goes through a process of excluding tags from it until only the ones to be deleted remain: 1. Collect all the tags for a given repository in a list. @@ -513,43 +521,41 @@ then goes through a process of excluding tags from it until only the ones to be 1. Excludes any tags that do not have a manifest (not part of the options). 1. Orders the remaining tags by `created_date`. 1. Excludes from the list the N tags based on the `keep_n` value (Number of tags to retain). -1. Excludes from the list the tags more recent than the `older_than` value (Expiration interval). +1. Excludes from the list the tags more recent than the `older_than` value (Cleanup interval). 1. Excludes from the list any tags matching the `name_regex_keep` value (Images to preserve). 1. Finally, the remaining tags in the list are deleted from the Container Registry. -### Managing project expiration policy through the UI - -To manage project expiration policy, navigate to **{settings}** **Settings > CI/CD > Container Registry tag expiration policy**. +### Managing project cleanup policy through the UI - +To manage project cleanup policy, navigate to **{settings}** **Settings > CI/CD > Container Registry tag cleanup policy**. The UI allows you to configure the following: -- **Expiration policy:** enable or disable the expiration policy. -- **Expiration interval:** how long tags are exempt from being deleted. -- **Expiration schedule:** how often the cron job checking the tags should run. +- **Cleanup policy:** enable or disable the cleanup policy. +- **Cleanup interval:** how long tags are exempt from being deleted. +- **Cleanup schedule:** how often the cron job checking the tags should run. - **Number of tags to retain:** how many tags to _always_ keep for each image. -- **Docker tags with names matching this regex pattern will expire:** the regex used to determine what tags should be expired. To qualify all tags for expiration, use the default value of `.*`. +- **Docker tags with names matching this regex pattern will expire:** the regex used to determine what tags should be cleaned up. To qualify all tags for cleanup, use the default value of `.*`. - **Docker tags with names matching this regex pattern will be preserved:** the regex used to determine what tags should be preserved. To preserve all tags, use the default value of `.*`. -#### Troubleshooting expiration policies +#### Troubleshooting cleanup policies If you see the following message: -"Something went wrong while updating the expiration policy." +"Something went wrong while updating the cleanup policy." Check the regex patterns to ensure they are valid. You can use [Rubular](https://rubular.com/) to check your regex. View some common [regex pattern examples](#regex-pattern-examples). -### Managing project expiration policy through the API +### Managing project cleanup policy through the API -You can set, update, and disable the expiration policies using the GitLab API. +You can set, update, and disable the cleanup policies using the GitLab API. Examples: -- Select all tags, keep at least 1 tag per image, expire any tag older than 14 days, run once a month, preserve any images with the name `master` and the policy is enabled: +- Select all tags, keep at least 1 tag per image, clean up any tag older than 14 days, run once a month, preserve any images with the name `master` and the policy is enabled: ```shell curl --request PUT --header 'Content-Type: application/json;charset=UTF-8' --header "PRIVATE-TOKEN: <your_access_token>" --data-binary '{"container_expiration_policy_attributes":{"cadence":"1month","enabled":true,"keep_n":1,"older_than":"14d","name_regex":"","name_regex_delete":".*","name_regex_keep":".*-master"}}' 'https://gitlab.example.com/api/v4/projects/2' @@ -560,15 +566,15 @@ See the API documentation for further details: [Edit project](../../../api/proje ### Use with external container registries When using an [external container registry](./../../../administration/packages/container_registry.md#use-an-external-container-registry-with-gitlab-as-an-auth-endpoint), -running an expiration policy on a project may have some performance risks. If a project is going to run +running a cleanup policy on a project may have some performance risks. If a project is going to run a policy that will remove large quantities of tags (in the thousands), the GitLab background jobs that -run the policy may get backed up or fail completely. It is recommended you only enable container expiration +run the policy may get backed up or fail completely. It is recommended you only enable container cleanup policies for projects that were created before GitLab 12.8 if you are confident the amount of tags being cleaned up will be minimal. ### Regex pattern examples -Expiration policies use regex patterns to determine which tags should be preserved or removed, both in the UI and the API. +Cleanup policies use regex patterns to determine which tags should be preserved or removed, both in the UI and the API. Here are examples of regex patterns you may want to use: @@ -596,6 +602,15 @@ Here are examples of regex patterns you may want to use: (?:v.+|master|release) ``` +## Use the Container Registry to store Helm Charts + +With the launch of [Helm v3](https://helm.sh/docs/topics/registries/), +you can use the Container Registry to store Helm Charts. However, due to the way metadata is passed +and stored by Docker, it is not possible for GitLab to parse this data and meet performance standards. +[This epic](https://gitlab.com/groups/gitlab-org/-/epics/2313) updates the architecture of the Container Registry to support Helm Charts. + +You can read more about the above challenges [here](https://gitlab.com/gitlab-org/gitlab/-/issues/38047#note_298842890). + ## Limitations - Moving or renaming existing Container Registry repositories is not supported @@ -603,7 +618,7 @@ once you have pushed images, because the images are signed, and the signature includes the repository name. To move or rename a repository with a Container Registry, you will have to delete all existing images. - Prior to GitLab 12.10, any tags that use the same image ID as the `latest` tag -will not be deleted by the expiration policy. +will not be deleted by the cleanup policy. ## Troubleshooting the GitLab Container Registry diff --git a/doc/user/packages/img/group_packages_list_v13_0.png b/doc/user/packages/img/group_packages_list_v13_0.png Binary files differdeleted file mode 100644 index 8cf3fd1a131..00000000000 --- a/doc/user/packages/img/group_packages_list_v13_0.png +++ /dev/null diff --git a/doc/user/packages/img/package_detail_v13_0.png b/doc/user/packages/img/package_detail_v13_0.png Binary files differdeleted file mode 100644 index dcfbc0a4787..00000000000 --- a/doc/user/packages/img/package_detail_v13_0.png +++ /dev/null diff --git a/doc/user/packages/img/project_packages_list_v13_0.png b/doc/user/packages/img/project_packages_list_v13_0.png Binary files differdeleted file mode 100644 index 468a6fe6467..00000000000 --- a/doc/user/packages/img/project_packages_list_v13_0.png +++ /dev/null diff --git a/doc/user/packages/index.md b/doc/user/packages/index.md index 6e59a87ae36..ab9cdc204f8 100644 --- a/doc/user/packages/index.md +++ b/doc/user/packages/index.md @@ -6,11 +6,11 @@ info: To determine the technical writer assigned to the Stage/Group associated w # GitLab Package Registry -GitLab Packages allows organizations to utilize GitLab as a private repository -for a variety of common package managers. Users are able to build and publish +With the GitLab Package Registry, you can use GitLab as a private or public repository +for a variety of common package managers. You can build and publish packages, which can be easily consumed as a dependency in downstream projects. -The Packages feature allows GitLab to act as a repository for the following: +GitLab acts as a repository for the following: | Software repository | Description | Available in GitLab version | | ------------------- | ----------- | --------------------------- | @@ -22,87 +22,88 @@ The Packages feature allows GitLab to act as a repository for the following: | [NuGet Repository](nuget_repository/index.md) **(PREMIUM)** | The GitLab NuGet Repository will enable every project in GitLab to have its own space to store [NuGet](https://www.nuget.org/) packages. | 12.8+ | | [PyPi Repository](pypi_repository/index.md) **(PREMIUM)** | The GitLab PyPi Repository will enable every project in GitLab to have its own space to store [PyPi](https://pypi.org/) packages. | 12.10+ | | [Go Proxy](go_proxy/index.md) **(PREMIUM)** | The Go proxy for GitLab enables every project in GitLab to be fetched with the [Go proxy protocol](https://proxy.golang.org/). | 13.1+ | -| [Composer Repository](composer_repository/index.md) **(PREMIUM)** | The GitLab Composer Repository will enable every project in GitLab to have its own space to store [Composer](https://getcomposer.org/) packages. | 13.1+ | +| [Composer Repository](composer_repository/index.md) **(PREMIUM)** | The GitLab Composer Repository will enable every project in GitLab to have its own space to store [Composer](https://getcomposer.org/) packages. | 13.2+ | -## Enable the Package Registry for your project +## View packages -If you cannot find the **{package}** **Packages & Registries > Package -Registry** entry under your project's sidebar, ensure that: +You can view packages for your project or group. -1. The GitLab Package Registry has been enabled by your administrator (following - [this documentation](../../administration/packages/index.md)); and -1. The Package Registry has been enabled for your project. +1. Go to the project or group. +1. Go to **{package}** **Packages & Registries > Package Registry**. -Once an administrator has enabled the GitLab Package Registry for your GitLab -instance, to enable Package Registry for your project: +You can search, sort, and filter packages on this page. -1. Go to your project's **Settings > General** page. -1. Expand the **Visibility, project features, permissions** section and enable the -**Packages** feature on your project. -1. Press **Save changes** for the changes to take effect. You should now be able to -see the **Packages & Registries > Package Registry** link in the sidebar. +For information on how to create and upload a package, view the GitLab documentation for your package type. -### View Packages for your project +## Use GitLab CI/CD to build packages -Navigating to your project's **{package}** **Packages & Registries > Package Registry** will show a list -of all packages that have been added to your project. +You can use [GitLab CI/CD](./../../ci/README.md) to build packages. +For Maven and NPM packages, and Composer dependencies, you can +authenticate with GitLab by using the `CI_JOB_TOKEN`. - +CI/CD templates, which you can use to get started, are in [this repo](https://gitlab.com/gitlab-org/gitlab/-/tree/master/lib/gitlab/ci/templates). -On this page, you can: +Learn more about [using CI/CD to build Maven packages](maven_repository/index.md#creating-maven-packages-with-gitlab-cicd) +and [NPM packages](npm_registry/index.md#publishing-a-package-with-cicd). -- View all the packages that have been uploaded to the project. -- Sort the packages list by created date, version or name. -- Filter the list by package name. -- Change tabs to display packages of a certain type. -- Remove a package (if you have suitable [permissions](../permissions.md)). -- Navigate to specific package detail page. +If you use CI/CD to build a package, extended activity +information is displayed when you view the package details: -### View Packages for your group + -You can view all packages belonging to a group by navigating to **{package}** -**Packages & Registries > Package Registry** from the group sidebar. +You can view which pipeline published the package, as well as the commit and +user who triggered it. - +## Download a package -On this page, you can: +To download a package: -- View all the packages that have been uploaded to each of the groups projects. -- Sort the packages list by created date, version, name or project. -- Filter the list by package name. -- Change tabs to display packages of a certain type. -- Navigate to specific package detail page. +1. Go to **{package}** **Packages & Registries > Package Registry**. +1. Click the name of the package you want to download. +1. In the **Activity** section, click the name of the package you want to download. -### View additional package information +## Delete a package -Additional package information can be viewed by browsing to the package details -page from the either the project or group list. +You cannot edit a package after you publish it in the Package Registry. Instead, you +must delete and recreate it. - +- You cannot delete packages from the group view. You must delete them from the project view instead. + See [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/227714) for details. +- You must have suitable [permissions](../permissions.md). -On this page you can: +You can delete packages by using [the API](../../api/packages.md#delete-a-project-package) or the UI. -- See the extended package information, including metadata. This is unique to -each package type and will display different information for different types. -- View quick installation and registry setup instructions. These are a shortcut -for users who have already set up the Package Registry and just want quick -installation instructions. -- View the package activity, including when and how a package was published. -- View and download the contents of the package. Outside of installing a -package via a manager, you can also download the files individually. -- Delete the package (if you have suitable [permissions](../permissions.md)). +To delete a package in the UI: -### Build packages via GitLab CI/CD +1. Go to **{package}** **Packages & Registries > Package Registry**. +1. Find the name of the package you want to delete. +1. Click **Delete**. -Some of the supported packages can be built via [GitLab CI/CD](./../../ci/README.md) -using the `CI_JOB_TOKEN`. If a package is built this way, then extended activity -information is displayed on the package details page: +The package is permanently deleted. - +## Disable the Package Registry -You can view which pipeline published the package, as well as the commit and -user who triggered it. To see if a package type supports being built via CI/CD, -check the specific documentation for your package type. +The Package Registry is automatically enabled. + +If you are using a self-managed instance of GitLab, your administrator can remove +the menu item, **{package}** **Packages & Registries**, from the GitLab sidebar. For more information, +see the [administration documentation](../../administration/packages/index.md). + +You can also remove the Package Registry for your project specifically: + +1. In your project, go to **{settings}** **Settings > General**. +1. Expand the **Visibility, project features, permissions** section and disable the + **Packages** feature. +1. Click **Save changes**. + +The **{package}** **Packages & Registries > Package Registry** entry is removed from the sidebar. + +## Package workflows + +Learn how to use the GitLab Package Registry to build your own custom package workflow. + +- [Use a project as a package registry](./workflows/project_registry.md) to publish all of your packages to one project. +- Publish multiple different packages from one [monorepo project](./workflows/monorepo.md). ## Suggested contributions @@ -125,10 +126,3 @@ are adding support for [PHP](https://gitlab.com/gitlab-org/gitlab/-/merge_reques | [RubyGems](https://gitlab.com/gitlab-org/gitlab/-/issues/803) | Use GitLab to host your own gems. | | [SBT](https://gitlab.com/gitlab-org/gitlab/-/issues/36898) | Resolve dependencies from and deploy build output to SBT repositories when running SBT builds. | | [Vagrant](https://gitlab.com/gitlab-org/gitlab/-/issues/36899) | Securely host your Vagrant boxes in local repositories. | - -## Package workflows - -Learning how to use the GitLab Package Registry will help you build your own custom package workflow. - -- [Use a project as a package registry](./workflows/project_registry.md) to publish all of your packages to one project. -- [Working with a monorepo](./workflows/monorepo.md): Learn how to publish multiple different packages from one monorepo project. diff --git a/doc/user/packages/maven_repository/index.md b/doc/user/packages/maven_repository/index.md index 2d40a623fb8..b194b7d837a 100644 --- a/doc/user/packages/maven_repository/index.md +++ b/doc/user/packages/maven_repository/index.md @@ -23,7 +23,7 @@ After the Packages feature is enabled, the Maven Repository will be available fo all new projects by default. To enable it for existing projects, or if you want to disable it: -1. Navigate to your project's **Settings > General > Permissions**. +1. Navigate to your project's **Settings > General > Visibility, project features, permissions**. 1. Find the Packages feature and enable or disable it. 1. Click on **Save changes** for the changes to take effect. @@ -821,6 +821,16 @@ user's home location (in this case the user is `root` since it runs in a Docker container), and Maven will use the configured CI [environment variables](../../../ci/variables/README.md#predefined-environment-variables). +### Version validation + +The version string is validated using the following regex. + +```ruby +\A(\.?[\w\+-]+\.?)+\z +``` + +You can play around with the regex and try your version strings on [this regular expression editor](https://rubular.com/r/rrLQqUXjfKEoL6). + ## Troubleshooting ### Useful Maven command line options diff --git a/doc/user/packages/npm_registry/index.md b/doc/user/packages/npm_registry/index.md index 4d60c227ccd..390b2c28e10 100644 --- a/doc/user/packages/npm_registry/index.md +++ b/doc/user/packages/npm_registry/index.md @@ -25,7 +25,7 @@ This option is available only if your GitLab administrator has After the NPM registry is enabled, it will be available for all new projects by default. To enable it for existing projects, or if you want to disable it: -1. Navigate to your project's **Settings > General > Permissions**. +1. Navigate to your project's **Settings > General > Visibility, project features, permissions**. 1. Find the Packages feature and enable or disable it. 1. Click on **Save changes** for the changes to take effect. @@ -238,7 +238,8 @@ The regex that is used for naming is validating all package names from all packa It allows for capital letters, while NPM does not, and allows for almost all of the characters NPM allows with a few exceptions (`~` is not allowed). -NOTE: **Note:** Capital letters are needed because the scope is required to be +NOTE: **Note:** +Capital letters are needed because the scope is required to be identical to the top level namespace of the project. So, for example, if your project path is `My-Group/project-foo`, your package must be named `@My-Group/any-package-name`. `@my-group/any-package-name` will not work. @@ -306,10 +307,12 @@ stages: deploy: stage: deploy script: - - echo '//gitlab.com/api/v4/projects/<your_project_id>/packages/npm/:_authToken=${CI_JOB_TOKEN}'>.npmrc + - echo '//gitlab.com/api/v4/projects/${CI_PROJECT_ID}/packages/npm/:_authToken=${CI_JOB_TOKEN}'>.npmrc - npm publish ``` +Learn more about [using `CI_JOB_TOKEN` to authenticate to the GitLab NPM registry](#authenticating-with-a-ci-job-token). + ## Troubleshooting ### Error running yarn with NPM registry diff --git a/doc/user/packages/nuget_repository/index.md b/doc/user/packages/nuget_repository/index.md index 6ada68dcceb..4ee5e5c4627 100644 --- a/doc/user/packages/nuget_repository/index.md +++ b/doc/user/packages/nuget_repository/index.md @@ -63,7 +63,7 @@ This option is available only if your GitLab administrator has After the NuGet Repository is enabled, it will be available for all new projects by default. To enable it for existing projects, or if you want to disable it: -1. Navigate to your project's **Settings > General > Permissions**. +1. Navigate to your project's **Settings > General > Visibility, project features, permissions**. 1. Find the Packages feature and enable or disable it. 1. Click on **Save changes** for the changes to take effect. diff --git a/doc/user/packages/pypi_repository/index.md b/doc/user/packages/pypi_repository/index.md index 6f6609d82b1..615741cc303 100644 --- a/doc/user/packages/pypi_repository/index.md +++ b/doc/user/packages/pypi_repository/index.md @@ -28,7 +28,7 @@ This option is available only if your GitLab administrator has After the PyPi Repository is enabled, it will be available for all new projects by default. To enable it for existing projects, or if you want to disable it: -1. Navigate to your project's **Settings > General > Permissions**. +1. Navigate to your project's **Settings > General > Visibility, project features, permissions**. 1. Find the Packages feature and enable or disable it. 1. Click on **Save changes** for the changes to take effect. diff --git a/doc/user/permissions.md b/doc/user/permissions.md index 21435c41d68..ba62cf81847 100644 --- a/doc/user/permissions.md +++ b/doc/user/permissions.md @@ -25,7 +25,7 @@ To add or import a user, you can follow the ## Principles behind permissions -See our [product handbook on permissions](https://about.gitlab.com/handbook/product/#permissions-in-gitlab) +See our [product handbook on permissions](https://about.gitlab.com/handbook/product/gitlab-the-product/#permissions-in-gitlab). ## Instance-wide user permissions @@ -47,7 +47,7 @@ The following table depicts the various user permission levels in a project. |---------------------------------------------------|---------|------------|-------------|----------|--------| | Download project | ✓ (*1*) | ✓ | ✓ | ✓ | ✓ | | Leave comments | ✓ (*1*) | ✓ | ✓ | ✓ | ✓ | -| View allowed and denied licenses **(ULTIMATE)** | ✓ (*1*) | ✓ | ✓ | ✓ | ✓ | +| View allowed and denied licenses **(ULTIMATE)** | ✓ (*1*) | ✓ | ✓ | ✓ | ✓ | | View License Compliance reports **(ULTIMATE)** | ✓ (*1*) | ✓ | ✓ | ✓ | ✓ | | View Security reports **(ULTIMATE)** | ✓ (*3*) | ✓ | ✓ | ✓ | ✓ | | View Dependency list **(ULTIMATE)** | ✓ (*1*) | ✓ | ✓ | ✓ | ✓ | @@ -86,7 +86,7 @@ The following table depicts the various user permission levels in a project. | View metrics dashboard annotations | | ✓ | ✓ | ✓ | ✓ | | Create/edit requirements **(ULTIMATE)** | | ✓ | ✓ | ✓ | ✓ | | Pull [packages](packages/index.md) **(PREMIUM)** | | ✓ | ✓ | ✓ | ✓ | -| Publish [packages](packages/index.md) **(PREMIUM)**| | | ✓ | ✓ | ✓ | +| Publish [packages](packages/index.md) **(PREMIUM)**| | | ✓ | ✓ | ✓ | | Upload [Design Management](project/issues/design_management.md) files | | | ✓ | ✓ | ✓ | | Create/edit/delete [Releases](project/releases/index.md)| | | ✓ | ✓ | ✓ | | Create new branches | | | ✓ | ✓ | ✓ | @@ -120,6 +120,7 @@ The following table depicts the various user permission levels in a project. | Rewrite/remove Git tags | | | ✓ | ✓ | ✓ | | Manage Feature Flags **(PREMIUM)** | | | ✓ | ✓ | ✓ | | Create/edit/delete metrics dashboard annotations | | | ✓ | ✓ | ✓ | +| Run CI/CD pipeline against a protected branch | | | ✓ (*5*) | ✓ | ✓ | | Use environment terminals | | | | ✓ | ✓ | | Run Web IDE's Interactive Web Terminals **(ULTIMATE ONLY)** | | | | ✓ | ✓ | | Add new team members | | | | ✓ | ✓ | @@ -139,7 +140,10 @@ The following table depicts the various user permission levels in a project. | Manage GitLab Pages domains and certificates | | | | ✓ | ✓ | | Remove GitLab Pages | | | | ✓ | ✓ | | Manage clusters | | | | ✓ | ✓ | +| Manage Project Operations | | | | ✓ | ✓ | | View Pods logs | | | | ✓ | ✓ | +| Read Terraform state | | | ✓ | ✓ | ✓ | +| Manage Terraform state | | | | ✓ | ✓ | | Manage license policy **(ULTIMATE)** | | | | ✓ | ✓ | | Edit comments (posted by any user) | | | | ✓ | ✓ | | Manage Error Tracking | | | | ✓ | ✓ | @@ -154,6 +158,7 @@ The following table depicts the various user permission levels in a project. | Remove project | | | | | ✓ | | Archive project | | | | | ✓ | | Delete issues | | | | | ✓ | +| Delete pipelines | | | | | ✓ | | Delete merge request | | | | | ✓ | | Disable notification emails | | | | | ✓ | | Force push to protected branches (*4*) | | | | | | @@ -246,6 +251,7 @@ group. | Create project in group | | | ✓ (3) | ✓ (3) | ✓ (3) | | Share (invite) groups with groups | | | | | ✓ | | Create/edit/delete group milestones | | | ✓ | ✓ | ✓ | +| Create/edit/delete iterations | | | ✓ | ✓ | ✓ | | Enable/disable a dependency proxy **(PREMIUM)** | | | ✓ | ✓ | ✓ | | Use security dashboard **(ULTIMATE)** | | | ✓ | ✓ | ✓ | | Create/edit/delete metrics dashboard annotations | | | ✓ | ✓ | ✓ | @@ -267,8 +273,8 @@ group. | View Issues analytics **(PREMIUM)** | ✓ | ✓ | ✓ | ✓ | ✓ | | View Productivity analytics **(PREMIUM)** | | ✓ | ✓ | ✓ | ✓ | | View Value Stream analytics | ✓ | ✓ | ✓ | ✓ | ✓ | -| View Billing **(FREE ONLY)** | ✓ | ✓ | ✓ | ✓ | ✓ (4) | -| View Usage Quotas **(FREE ONLY)** | ✓ | ✓ | ✓ | ✓ | ✓ (4) | +| View Billing **(FREE ONLY)** | | | | | ✓ (4) | +| View Usage Quotas **(FREE ONLY)** | | | | | ✓ (4) | 1. Groups can be set to [allow either Owners or Owners and Maintainers to create subgroups](group/subgroups/index.md#creating-a-subgroup) @@ -281,7 +287,7 @@ group. ### Subgroup permissions When you add a member to a subgroup, they inherit the membership and -permission level from the parent group. This model allows access to +permission level from the parent group(s). This model allows access to nested groups if you have membership in one of its parents. To learn more, read through the documentation on diff --git a/doc/user/profile/account/delete_account.md b/doc/user/profile/account/delete_account.md index 3c6f2989091..a70d11438f4 100644 --- a/doc/user/profile/account/delete_account.md +++ b/doc/user/profile/account/delete_account.md @@ -35,7 +35,8 @@ As an administrator, you can delete a user account by: - **Delete user and contributions** to delete the user and their associated records. -DANGER: **Danger:** Using the **Delete user and contributions** option may result +DANGER: **Danger:** +Using the **Delete user and contributions** option may result in removing more data than intended. Please see [associated records](#associated-records) below for additional details. diff --git a/doc/user/profile/account/two_factor_authentication.md b/doc/user/profile/account/two_factor_authentication.md index bfcaeaf6a15..4f769f9a671 100644 --- a/doc/user/profile/account/two_factor_authentication.md +++ b/doc/user/profile/account/two_factor_authentication.md @@ -5,9 +5,9 @@ group: Access info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# Two-Factor Authentication +# Two-factor authentication -Two-factor Authentication (2FA) provides an additional level of security to your +Two-factor authentication (2FA) provides an additional level of security to your GitLab account. Once enabled, in addition to supplying your username and password to login, you'll be prompted for a code generated by your one time password authenticator. For example, a password manager on one of your devices. @@ -62,7 +62,7 @@ To enable 2FA: 1. Click **Submit**. If the pin you entered was correct, you'll see a message indicating that -Two-Factor Authentication has been enabled, and you'll be presented with a list +two-factor authentication has been enabled, and you'll be presented with a list of [recovery codes](#recovery-codes). Make sure you download them and keep them in a safe place. diff --git a/doc/user/profile/index.md b/doc/user/profile/index.md index 663a2888ee7..7a871afd861 100644 --- a/doc/user/profile/index.md +++ b/doc/user/profile/index.md @@ -22,7 +22,7 @@ See the [authentication topic](../../topics/authentication/index.md) for more de ### Unknown sign-in -GitLab will notify you if a sign-in occurs that is from an unknown IP address. +GitLab will notify you if a sign-in occurs that is from an unknown IP address or device. See [Unknown Sign-In Notification](unknown_sign_in_notification.md) for more details. ## User profile diff --git a/doc/user/profile/notifications.md b/doc/user/profile/notifications.md index ee228050945..dbf486e399e 100644 --- a/doc/user/profile/notifications.md +++ b/doc/user/profile/notifications.md @@ -187,7 +187,7 @@ To minimize the number of notifications that do not require any action, from [Gi | Remove milestone merge request | Subscribers, participants mentioned, and Custom notification level with this event selected | | New comment | The above, plus anyone mentioned by `@username` in the comment, with notification level "Mention" or higher | | Failed pipeline | The author of the pipeline | -| Fixed pipeline | The author of the pipeline. Disabled by default. To activate it you must [enable the `ci_pipeline_fixed_notifications` feature flag](../../development/feature_flags/development.md#enabling-a-feature-flag-in-development). | +| Fixed pipeline | The author of the pipeline. Enabled by default. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/24309) in GitLab 13.1. Administrators can disable this notification option using the `ci_pipeline_fixed_notifications` [feature flag](../../administration/feature_flags.md). | | Successful pipeline | The author of the pipeline, if they have the custom notification setting for successful pipelines set. If the pipeline failed previously, a `Fixed pipeline` message will be sent for the first successful pipeline after the failure, then a `Successful pipeline` message for any further successful pipelines. | | New epic **(ULTIMATE)** | | | Close epic **(ULTIMATE)** | | diff --git a/doc/user/profile/personal_access_tokens.md b/doc/user/profile/personal_access_tokens.md index e2c3dc74cf1..59ca124f566 100644 --- a/doc/user/profile/personal_access_tokens.md +++ b/doc/user/profile/personal_access_tokens.md @@ -19,6 +19,7 @@ Personal access tokens expire on the date you define, at midnight UTC. - GitLab runs a check at 01:00 AM UTC every day to identify personal access tokens that will expire in under seven days. The owners of these tokens are notified by email. - In GitLab Ultimate, administrators may [limit the lifetime of personal access tokens](../admin_area/settings/account_and_limit_settings.md#limiting-lifetime-of-personal-access-tokens-ultimate-only). +- In GitLab Ultimate, administrators may [toggle enforcement of personal access token expiry](../admin_area/settings/account_and_limit_settings.md#optional-enforcement-of-personal-access-token-expiry-ultimate-only). For examples of how you can use a personal access token to authenticate with the API, see the following section from our [API Docs](../../api/README.md#personalproject-access-tokens). @@ -43,6 +44,10 @@ profile. At any time, you can revoke any personal access token by clicking the respective **Revoke** button under the **Active Personal Access Token** area. +### Token activity + +You can see when a token was last used from the **Personal Access Tokens** page. Updates to the token usage is fixed at once per 24 hours. Requests to [API resources](../../api/api_resources.md) and the [GraphQL API](../../api/graphql/index.md) will update a token's usage. + ## Limiting scopes of a personal access token Personal access tokens can be created with one or more scopes that allow various diff --git a/doc/user/profile/preferences.md b/doc/user/profile/preferences.md index a5fa3cf373f..b94ae958d3b 100644 --- a/doc/user/profile/preferences.md +++ b/doc/user/profile/preferences.md @@ -63,7 +63,10 @@ Dark theme currently only works with the 'Dark' syntax highlighting. NOTE: **Note:** GitLab uses the [rouge Ruby library](http://rouge.jneen.net/ "Rouge website") -for syntax highlighting. For a list of supported languages visit the rouge website. +for syntax highlighting outside of any Editor context. The WebIDE (like Snippets) +uses [Monaco Editor](https://microsoft.github.io/monaco-editor/) and it's provided [Monarch](https://microsoft.github.io/monaco-editor/monarch.html) library for +syntax highlighting. For a list of supported languages, visit the documentation of +the respective libraries. Changing this setting allows you to customize the color theme when viewing any syntax highlighted code on GitLab. diff --git a/doc/user/profile/unknown_sign_in_notification.md b/doc/user/profile/unknown_sign_in_notification.md index 200358bb050..6a6820bb2d4 100644 --- a/doc/user/profile/unknown_sign_in_notification.md +++ b/doc/user/profile/unknown_sign_in_notification.md @@ -9,16 +9,24 @@ info: To determine the technical writer assigned to the Stage/Group associated w > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/27211) in GitLab 13.0. -When a user successfully signs in from a previously unknown IP address, +NOTE: **Note:** +This feature is enabled by default for self-managed instances. Administrators may disable this feature +through the [Sign-in restrictions](../admin_area/settings/sign_in_restrictions.md#email-notification-for-unknown-sign-ins) section of the UI. +The feature is always enabled on GitLab.com. + +When a user successfully signs in from a previously unknown IP address or device, GitLab notifies the user by email. In this way, GitLab proactively alerts users of potentially malicious or unauthorized sign-ins. -There are two methods used to identify a known sign-in: +There are several methods used to identify a known sign-in. All methods must fail +for a notification email to be sent. - Last sign-in IP: The current sign-in IP address is checked against the last sign-in IP address. - Current active sessions: If the user has an existing active session from the same IP address. See [Active Sessions](active_sessions.md). +- Cookie: After successful sign in, an encrypted cookie is stored in the browser. + This cookie is set to expire 14 days after the last successful sign in. ## Example email diff --git a/doc/user/project/bulk_editing.md b/doc/user/project/bulk_editing.md index 6d091c431a2..c4a6aea807c 100644 --- a/doc/user/project/bulk_editing.md +++ b/doc/user/project/bulk_editing.md @@ -14,7 +14,7 @@ For more details, see If you want to update attributes across multiple issues or merge requests, you can do it by bulk editing them, that is, editing them together. - + ## Bulk edit issues at the project level @@ -25,8 +25,12 @@ When bulk editing issues in a project, you can edit the following attributes: - Status (open/closed) - Assignee +- Epic ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/210470) in + [GitLab Premium](https://about.gitlab.com/pricing/) 13.2.) **(PREMIUM)** - Milestone - Labels +- Health status ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/218395) in + [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.2.) **(ULTIMATE)** - Subscriptions To update multiple project issues at the same time: diff --git a/doc/user/project/clusters/add_remove_clusters.md b/doc/user/project/clusters/add_remove_clusters.md index d0cba729e35..65f1c59f4ca 100644 --- a/doc/user/project/clusters/add_remove_clusters.md +++ b/doc/user/project/clusters/add_remove_clusters.md @@ -13,6 +13,11 @@ GitLab offers integrated cluster creation for the following Kubernetes providers GitLab can also integrate with any standard Kubernetes provider, either on-premise or hosted. +NOTE: **Note:** +Watch the webcast [Scalable app deployment with GitLab and Google Cloud Platform](https://about.gitlab.com/webcast/scalable-app-deploy/) +and learn how to spin up a Kubernetes cluster managed by Google Cloud Platform (GCP) +in a few clicks. + TIP: **Tip:** Every new Google Cloud Platform (GCP) account receives [$300 in credit upon sign up](https://console.cloud.google.com/freetrial), and in partnership with Google, GitLab is able to offer an additional $200 for new GCP accounts to get started with GitLab's @@ -23,7 +28,7 @@ Google Kubernetes Engine Integration. All you have to do is [follow this link](h Before [adding a Kubernetes cluster](#create-new-cluster) using GitLab, you need: - GitLab itself. Either: - - A GitLab.com [account](https://about.gitlab.com/pricing/#gitlab-com). + - A [GitLab.com account](https://about.gitlab.com/pricing/#gitlab-com). - A [self-managed installation](https://about.gitlab.com/pricing/#self-managed) with GitLab version 12.5 or later. This will ensure the GitLab UI can be used for cluster creation. - The following GitLab access: @@ -52,14 +57,10 @@ to manage the newly created cluster. NOTE: **Note:** Restricted service account for deployment was [introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/51716) in GitLab 11.5. -When you install Helm into your cluster, the `tiller` service account -is created with `cluster-admin` privileges in the `gitlab-managed-apps` -namespace. - -This service account will be: - -- Added to the installed Helm Tiller. -- Used by Helm to install and run [GitLab managed applications](index.md#installing-applications). +The first time you install an application into your cluster, the `tiller` service +account is created with `cluster-admin` privileges in the +`gitlab-managed-apps` namespace. This service account will be used by Helm to +install and run [GitLab managed applications](index.md#installing-applications). Helm will also create additional service accounts and other resources for each installed application. Consult the documentation of the Helm charts for each application @@ -88,8 +89,8 @@ GitLab creates the following resources for RBAC clusters. | `gitlab` | `ServiceAccount` | `default` namespace | Creating a new cluster | | `gitlab-admin` | `ClusterRoleBinding` | [`cluster-admin`](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles) roleRef | Creating a new cluster | | `gitlab-token` | `Secret` | Token for `gitlab` ServiceAccount | Creating a new cluster | -| `tiller` | `ServiceAccount` | `gitlab-managed-apps` namespace | Installing Helm Tiller | -| `tiller-admin` | `ClusterRoleBinding` | `cluster-admin` roleRef | Installing Helm Tiller | +| `tiller` | `ServiceAccount` | `gitlab-managed-apps` namespace | Installing Helm charts | +| `tiller-admin` | `ClusterRoleBinding` | `cluster-admin` roleRef | Installing Helm charts | | Environment namespace | `Namespace` | Contains all environment-specific resources | Deploying to a cluster | | Environment namespace | `ServiceAccount` | Uses namespace of environment | Deploying to a cluster | | Environment namespace | `Secret` | Token for environment ServiceAccount | Deploying to a cluster | @@ -103,8 +104,8 @@ GitLab creates the following resources for ABAC clusters. |:----------------------|:---------------------|:-------------------------------------|:---------------------------| | `gitlab` | `ServiceAccount` | `default` namespace | Creating a new cluster | | `gitlab-token` | `Secret` | Token for `gitlab` ServiceAccount | Creating a new cluster | -| `tiller` | `ServiceAccount` | `gitlab-managed-apps` namespace | Installing Helm Tiller | -| `tiller-admin` | `ClusterRoleBinding` | `cluster-admin` roleRef | Installing Helm Tiller | +| `tiller` | `ServiceAccount` | `gitlab-managed-apps` namespace | Installing Helm charts | +| `tiller-admin` | `ClusterRoleBinding` | `cluster-admin` roleRef | Installing Helm charts | | Environment namespace | `Namespace` | Contains all environment-specific resources | Deploying to a cluster | | Environment namespace | `ServiceAccount` | Uses namespace of environment | Deploying to a cluster | | Environment namespace | `Secret` | Token for environment ServiceAccount | Deploying to a cluster | @@ -126,7 +127,7 @@ arbitrary images as they effectively have root access. If you don't want to use GitLab Runner in privileged mode, either: - Use shared Runners on GitLab.com. They don't have this security issue. -- Set up your own Runners using configuration described at +- Set up your own Runners using the configuration described at [Shared Runners](../../gitlab_com/index.md#shared-runners). This involves: 1. Making sure that you don't have it installed via [the applications](index.md#installing-applications). @@ -135,23 +136,26 @@ If you don't want to use GitLab Runner in privileged mode, either: ## Create new cluster -New clusters can be created using GitLab for: +New clusters can be created using GitLab on Google Kubernetes Engine (GKE) or +Amazon Elastic Kubernetes Service (EKS) at the project, group, or instance level: -- [Google Kubernetes Engine (GKE)](add_gke_clusters.md). -- [Amazon Elastic Kubernetes Service (EKS)](add_eks_clusters.md). +1. Navigate to your: + - Project's **{cloud-gear}** **Operations > Kubernetes** page, for a project-level cluster. + - Group's **{cloud-gear}** **Kubernetes** page, for a group-level cluster. + - **{admin}** **Admin Area >** **{cloud-gear}** **Kubernetes** page, for an instance-level cluster. +1. Click **Add Kubernetes cluster**. +1. Click the **Create new cluster** tab. +1. Click either **Amazon EKS** or **Google GKE**, and follow the instructions for your desired service: + - [Amazon EKS](add_eks_clusters.md#new-eks-cluster). + - [Google GKE](add_gke_clusters.md#creating-the-cluster-on-gke). ## Add existing cluster If you have an existing Kubernetes cluster, you can add it to a project, group, or instance. -For more information, see information for adding an: - -- [Existing Kubernetes cluster](#existing-kubernetes-cluster), including GKE clusters. -- [Existing EKS cluster](add_eks_clusters.md#existing-eks-cluster). - NOTE: **Note:** Kubernetes integration is not supported for arm64 clusters. See the issue -[Helm Tiller fails to install on arm64 cluster](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/64044) for details. +[Helm Tiller fails to install on arm64 cluster](https://gitlab.com/gitlab-org/gitlab/-/issues/29838) for details. ### Existing Kubernetes cluster @@ -214,9 +218,9 @@ To add a Kubernetes cluster to your project, group, or instance: kind: ClusterRole name: cluster-admin subjects: - - kind: ServiceAccount - name: gitlab-admin - namespace: kube-system + - kind: ServiceAccount + name: gitlab-admin + namespace: kube-system ``` 1. Apply the service account and cluster role binding to your cluster: @@ -297,14 +301,15 @@ to install some [pre-defined applications](index.md#installing-applications). When connecting a cluster via GitLab integration, you may specify whether the cluster is RBAC-enabled or not. This will affect how GitLab interacts with the -cluster for certain operations. If you **did not** check the "RBAC-enabled cluster" +cluster for certain operations. If you did *not* check the **RBAC-enabled cluster** checkbox at creation time, GitLab will assume RBAC is disabled for your cluster when interacting with it. If so, you must disable RBAC on your cluster for the integration to work properly. - + -NOTE: **Note**: Disabling RBAC means that any application running in the cluster, +NOTE: **Note:** +Disabling RBAC means that any application running in the cluster, or user who can authenticate to the cluster, has full API access. This is a [security concern](index.md#security-implications), and may not be desirable. @@ -320,17 +325,20 @@ kubectl create clusterrolebinding permissive-binding \ ## Enabling or disabling integration -After you have successfully added your cluster information, you can enable the -Kubernetes cluster integration: - -1. Click the **Enabled/Disabled** switch -1. Hit **Save** for the changes to take effect +The Kubernetes cluster integration enables after you have successfully either created +a new cluster or added an existing one. To disable Kubernetes cluster integration: -To disable the Kubernetes cluster integration, follow the same procedure. +1. Navigate to your: + - Project's **{cloud-gear}** **Operations > Kubernetes** page, for a project-level cluster. + - Group's **{cloud-gear}** **Kubernetes** page, for a group-level cluster. + - **{admin}** **Admin Area >** **{cloud-gear}** **Kubernetes** page, for an instance-level cluster. +1. Click on the name of the cluster. +1. Click the **GitLab Integration** toggle. +1. Click **Save changes**. ## Removing integration -To remove the Kubernetes cluster integration from your project, either: +To remove the Kubernetes cluster integration from your project, first navigate to the **Advanced Settings** tab of the cluster details page and either: - Select **Remove integration**, to remove only the Kubernetes integration. - [From GitLab 12.6](https://gitlab.com/gitlab-org/gitlab/-/issues/26815), select diff --git a/doc/user/project/clusters/img/rbac.png b/doc/user/project/clusters/img/rbac.png Binary files differdeleted file mode 100644 index 517e4f7ca44..00000000000 --- a/doc/user/project/clusters/img/rbac.png +++ /dev/null diff --git a/doc/user/project/clusters/img/rbac_v13_1.png b/doc/user/project/clusters/img/rbac_v13_1.png Binary files differnew file mode 100644 index 00000000000..2772af9ff89 --- /dev/null +++ b/doc/user/project/clusters/img/rbac_v13_1.png diff --git a/doc/user/project/clusters/index.md b/doc/user/project/clusters/index.md index 961a9fda5ff..ddcfd376d89 100644 --- a/doc/user/project/clusters/index.md +++ b/doc/user/project/clusters/index.md @@ -12,15 +12,6 @@ info: To determine the technical writer assigned to the Stage/Group associated w > - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/39840) in > GitLab 11.11 for [instances](../../instance/clusters/index.md). -GitLab provides many features with a Kubernetes integration. Kubernetes can be -integrated with projects, but also: - -- [Groups](../../group/clusters/index.md). -- [Instances](../../instance/clusters/index.md). - -NOTE: **Scalable app deployment with GitLab and Google Cloud Platform** -[Watch the webcast](https://about.gitlab.com/webcast/scalable-app-deploy/) and learn how to spin up a Kubernetes cluster managed by Google Cloud Platform (GCP) in a few clicks. - ## Overview Using the GitLab project Kubernetes integration, you can: @@ -28,17 +19,26 @@ Using the GitLab project Kubernetes integration, you can: - Use [Review Apps](../../../ci/review_apps/index.md). - Run [pipelines](../../../ci/pipelines/index.md). - [Deploy](#deploying-to-a-kubernetes-cluster) your applications. -- Detect and [monitor Kubernetes](#kubernetes-monitoring). +- Detect and [monitor Kubernetes](#monitoring-your-kubernetes-cluster). - Use it with [Auto DevOps](#auto-devops). - Use [Web terminals](#web-terminals). - Use [Deploy Boards](#deploy-boards-premium). **(PREMIUM)** - Use [Canary Deployments](#canary-deployments-premium). **(PREMIUM)** -- View [Logs](#logs). +- View [Logs](#viewing-pod-logs). - Run serverless workloads on [Kubernetes with Knative](serverless/index.md). +Besides integration at the project level, Kubernetes clusters can also be +integrated at the [group level](../../group/clusters/index.md) or +[GitLab instance level](../../instance/clusters/index.md). + +## Setting up + ### Supported cluster versions -GitLab is committed to support at least two production-ready Kubernetes minor versions at any given time. We regularly review the versions we support, and provide a four-month deprecation period before we remove support of a specific version. The range of supported versions is based on the evaluation of: +GitLab is committed to support at least two production-ready Kubernetes minor +versions at any given time. We regularly review the versions we support, and +provide a four-month deprecation period before we remove support of a specific +version. The range of supported versions is based on the evaluation of: - Our own needs. - The versions supported by major managed Kubernetes providers. @@ -55,80 +55,83 @@ Currently, GitLab supports the following Kubernetes versions: NOTE: **Note:** Some GitLab features may support versions outside the range provided here. -### Deploy Boards **(PREMIUM)** - -GitLab's Deploy Boards offer a consolidated view of the current health and -status of each CI [environment](../../../ci/environments/index.md) running on Kubernetes, -displaying the status of the pods in the deployment. Developers and other -teammates can view the progress and status of a rollout, pod by pod, in the -workflow they already use without any need to access Kubernetes. - -[Read more about Deploy Boards](../deploy_boards.md) - -### Canary Deployments **(PREMIUM)** - -Leverage [Kubernetes' Canary deployments](https://kubernetes.io/docs/concepts/cluster-administration/manage-deployment/#canary-deployments) -and visualize your canary deployments right inside the Deploy Board, without -the need to leave GitLab. - -[Read more about Canary Deployments](../canary_deployments.md) +### Adding and removing clusters -### Logs - -GitLab makes it easy to view the logs of running pods in connected Kubernetes clusters. By displaying the logs directly in GitLab, developers can avoid having to manage console tools or jump to a different interface. - -[Read more about Kubernetes logs](kubernetes_pod_logs.md) +See [Adding and removing Kubernetes clusters](add_remove_clusters.md) for details on how +to: -### Kubernetes monitoring +- Create a cluster in Google Cloud Platform (GCP) or Amazon Elastic Kubernetes Service + (EKS) using GitLab's UI. +- Add an integration to an existing cluster from any Kubernetes platform. -Automatically detect and monitor Kubernetes metrics. Automatic monitoring of -[NGINX Ingress](../integrations/prometheus_library/nginx.md) is also supported. +### Multiple Kubernetes clusters -[Read more about Kubernetes monitoring](../integrations/prometheus_library/kubernetes.md) +> - Introduced in [GitLab Premium](https://about.gitlab.com/pricing/) 10.3 +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/35094) to GitLab core in 13.2. -### Auto DevOps +You can associate more than one Kubernetes cluster to your +project. That way you can have different clusters for different environments, +like dev, staging, production, and so on. -Auto DevOps automatically detects, builds, tests, deploys, and monitors your -applications. +Simply add another cluster, like you did the first time, and make sure to +[set an environment scope](#setting-the-environment-scope-premium) that will +differentiate the new cluster with the rest. -To make full use of Auto DevOps(Auto Deploy, Auto Review Apps, and Auto Monitoring) -you will need the Kubernetes project integration enabled. +#### Setting the environment scope **(PREMIUM)** -[Read more about Auto DevOps](../../../topics/autodevops/index.md) +When adding more than one Kubernetes cluster to your project, you need to differentiate +them with an environment scope. The environment scope associates clusters with [environments](../../../ci/environments/index.md) similar to how the +[environment-specific variables](../../../ci/variables/README.md#limit-the-environment-scopes-of-environment-variables) work. -NOTE: **Note** -Kubernetes clusters can be used without Auto DevOps. +The default environment scope is `*`, which means all jobs, regardless of their +environment, will use that cluster. Each scope can only be used by a single +cluster in a project, and a validation error will occur if otherwise. +Also, jobs that don't have an environment keyword set will not be able to access any cluster. -### Web terminals +For example, let's say the following Kubernetes clusters exist in a project: -> Introduced in GitLab 8.15. +| Cluster | Environment scope | +| ----------- | ----------------- | +| Development | `*` | +| Production | `production` | -When enabled, the Kubernetes integration adds [web terminal](../../../ci/environments/index.md#web-terminals) -support to your [environments](../../../ci/environments/index.md). This is based on the `exec` functionality found in -Docker and Kubernetes, so you get a new shell session within your existing -containers. To use this integration, you should deploy to Kubernetes using -the deployment variables above, ensuring any deployments, replica sets, and -pods are annotated with: +And the following environments are set in +[`.gitlab-ci.yml`](../../../ci/yaml/README.md): -- `app.gitlab.com/env: $CI_ENVIRONMENT_SLUG` -- `app.gitlab.com/app: $CI_PROJECT_PATH_SLUG` +```yaml +stages: + - test + - deploy -`$CI_ENVIRONMENT_SLUG` and `$CI_PROJECT_PATH_SLUG` are the values of -the CI variables. +test: + stage: test + script: sh test -You must be the project owner or have `maintainer` permissions to use terminals. Support is limited -to the first container in the first pod of your environment. +deploy to staging: + stage: deploy + script: make deploy + environment: + name: staging + url: https://staging.example.com/ -## Adding and removing clusters +deploy to production: + stage: deploy + script: make deploy + environment: + name: production + url: https://example.com/ +``` -See [Adding and removing Kubernetes clusters](add_remove_clusters.md) for details on how -to: +The result will then be: -- Create a cluster in Google Cloud Platform (GCP) or Amazon Elastic Kubernetes Service - (EKS) using GitLab's UI. -- Add an integration to an existing cluster from any Kubernetes platform. +- The Development cluster details will be available in the `deploy to staging` + job. +- The production cluster details will be available in the `deploy to production` + job. +- No cluster details will be available in the `test` job because it doesn't + define any environment. -## Cluster configuration +## Configuring your Kubernetes cluster After [adding a Kubernetes cluster](add_remove_clusters.md) to GitLab, read this section that covers important considerations for configuring Kubernetes clusters with GitLab. @@ -203,72 +206,6 @@ you can either: - Create an `A` record that points to the Ingress IP address with your domain provider. - Enter a wildcard DNS address using a service such as nip.io or xip.io. For example, `192.168.1.1.xip.io`. -### Setting the environment scope **(PREMIUM)** - -When adding more than one Kubernetes cluster to your project, you need to differentiate -them with an environment scope. The environment scope associates clusters with [environments](../../../ci/environments/index.md) similar to how the -[environment-specific variables](../../../ci/variables/README.md#limit-the-environment-scopes-of-environment-variables) work. - -The default environment scope is `*`, which means all jobs, regardless of their -environment, will use that cluster. Each scope can only be used by a single -cluster in a project, and a validation error will occur if otherwise. -Also, jobs that don't have an environment keyword set will not be able to access any cluster. - -For example, let's say the following Kubernetes clusters exist in a project: - -| Cluster | Environment scope | -| ----------- | ----------------- | -| Development | `*` | -| Production | `production` | - -And the following environments are set in -[`.gitlab-ci.yml`](../../../ci/yaml/README.md): - -```yaml -stages: -- test -- deploy - -test: - stage: test - script: sh test - -deploy to staging: - stage: deploy - script: make deploy - environment: - name: staging - url: https://staging.example.com/ - -deploy to production: - stage: deploy - script: make deploy - environment: - name: production - url: https://example.com/ -``` - -The result will then be: - -- The Development cluster details will be available in the `deploy to staging` - job. -- The production cluster details will be available in the `deploy to production` - job. -- No cluster details will be available in the `test` job because it doesn't - define any environment. - -### Multiple Kubernetes clusters **(PREMIUM)** - -> Introduced in [GitLab Premium](https://about.gitlab.com/pricing/) 10.3. - -With GitLab Premium, you can associate more than one Kubernetes cluster to your -project. That way you can have different clusters for different environments, -like dev, staging, production, and so on. - -Simply add another cluster, like you did the first time, and make sure to -[set an environment scope](#setting-the-environment-scope-premium) that will -differentiate the new cluster with the rest. - ## Installing applications GitLab can install and manage some applications like Helm, GitLab Runner, Ingress, @@ -277,6 +214,19 @@ installing, upgrading, uninstalling, and troubleshooting applications for your project cluster, see [GitLab Managed Apps](../../clusters/applications.md). +## Auto DevOps + +Auto DevOps automatically detects, builds, tests, deploys, and monitors your +applications. + +To make full use of Auto DevOps (Auto Deploy, Auto Review Apps, and +Auto Monitoring) you will need the Kubernetes project integration enabled. + +[Read more about Auto DevOps](../../../topics/autodevops/index.md) + +NOTE: **Note:** +Kubernetes clusters can be used without Auto DevOps. + ## Deploying to a Kubernetes cluster A Kubernetes cluster can be the destination for a deployment job. If @@ -325,10 +275,59 @@ For **non**-GitLab-managed clusters, the namespace can be customized using [`environment:kubernetes:namespace`](../../../ci/environments/index.md#configuring-kubernetes-deployments) in `.gitlab-ci.yml`. -NOTE: **Note:** When using a [GitLab-managed cluster](#gitlab-managed-clusters), the +NOTE: **Note:** +When using a [GitLab-managed cluster](#gitlab-managed-clusters), the namespaces are created automatically prior to deployment and [can not be customized](https://gitlab.com/gitlab-org/gitlab/-/issues/38054). +### Integrations + +#### Canary Deployments **(PREMIUM)** + +Leverage [Kubernetes' Canary deployments](https://kubernetes.io/docs/concepts/cluster-administration/manage-deployment/#canary-deployments) +and visualize your canary deployments right inside the Deploy Board, without +the need to leave GitLab. + +[Read more about Canary Deployments](../canary_deployments.md) + +#### Deploy Boards **(PREMIUM)** + +GitLab's Deploy Boards offer a consolidated view of the current health and +status of each CI [environment](../../../ci/environments/index.md) running on Kubernetes, +displaying the status of the pods in the deployment. Developers and other +teammates can view the progress and status of a rollout, pod by pod, in the +workflow they already use without any need to access Kubernetes. + +[Read more about Deploy Boards](../deploy_boards.md) + +#### Viewing pod logs + +GitLab makes it easy to view the logs of running pods in connected Kubernetes +clusters. By displaying the logs directly in GitLab, developers can avoid having +to manage console tools or jump to a different interface. + +[Read more about Kubernetes logs](kubernetes_pod_logs.md) + +#### Web terminals + +> Introduced in GitLab 8.15. + +When enabled, the Kubernetes integration adds [web terminal](../../../ci/environments/index.md#web-terminals) +support to your [environments](../../../ci/environments/index.md). This is based +on the `exec` functionality found in Docker and Kubernetes, so you get a new +shell session within your existing containers. To use this integration, you +should deploy to Kubernetes using the deployment variables above, ensuring any +deployments, replica sets, and pods are annotated with: + +- `app.gitlab.com/env: $CI_ENVIRONMENT_SLUG` +- `app.gitlab.com/app: $CI_PROJECT_PATH_SLUG` + +`$CI_ENVIRONMENT_SLUG` and `$CI_PROJECT_PATH_SLUG` are the values of +the CI variables. + +You must be the project owner or have `maintainer` permissions to use terminals. +Support is limited to the first container in the first pod of your environment. + ### Troubleshooting Before the deployment jobs starts, GitLab creates the following specifically for @@ -353,15 +352,23 @@ Reasons for failure include: [`environment:name`](../../../ci/environments/index.md#defining-environments). If your job has no `environment:name` set, it will not be passed the Kubernetes credentials. -NOTE: **NOTE:** +NOTE: **Note:** Project-level clusters upgraded from GitLab 12.0 or older may be configured in a way that causes this error. Ensure you deselect the [GitLab-managed cluster](#gitlab-managed-clusters) option if you want to manage namespaces and service accounts yourself. -## Monitoring your Kubernetes cluster **(ULTIMATE)** +## Monitoring your Kubernetes cluster + +Automatically detect and monitor Kubernetes metrics. Automatic monitoring of +[NGINX Ingress](../integrations/prometheus_library/nginx.md) is also supported. + +[Read more about Kubernetes monitoring](../integrations/prometheus_library/kubernetes.md) + +### Visualizing cluster health -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/4701) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 10.6. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/4701) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 10.6. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/208224) to GitLab core in 13.2. When [Prometheus is deployed](#installing-applications), GitLab will automatically monitor the cluster's health. At the top of the cluster settings page, CPU and Memory utilization is displayed, along with the total amount available. Keeping an eye on cluster resources can be important, if the cluster runs out of memory pods may be shutdown or fail to start. diff --git a/doc/user/project/clusters/kubernetes_pod_logs.md b/doc/user/project/clusters/kubernetes_pod_logs.md index 509be4ed0a8..ee642dc18cf 100644 --- a/doc/user/project/clusters/kubernetes_pod_logs.md +++ b/doc/user/project/clusters/kubernetes_pod_logs.md @@ -13,9 +13,9 @@ GitLab makes it easy to view the logs of running pods in [connected Kubernetes c By displaying the logs directly in GitLab in the **Log Explorer**, developers can avoid managing console tools or jumping to a different interface. -NOTE: **Kubernetes + GitLab** +NOTE: **Note:** +[Learn more about Kubernetes + GitLab](https://about.gitlab.com/solutions/kubernetes/). Everything you need to build, test, deploy, and run your application at scale. -[Learn more](https://about.gitlab.com/solutions/kubernetes/). ## Overview diff --git a/doc/user/project/clusters/runbooks/img/helm-install.png b/doc/user/project/clusters/runbooks/img/helm-install.png Binary files differdeleted file mode 100644 index e67cf317ec1..00000000000 --- a/doc/user/project/clusters/runbooks/img/helm-install.png +++ /dev/null diff --git a/doc/user/project/clusters/runbooks/index.md b/doc/user/project/clusters/runbooks/index.md index 92ef35ad93f..a592d59f964 100644 --- a/doc/user/project/clusters/runbooks/index.md +++ b/doc/user/project/clusters/runbooks/index.md @@ -42,9 +42,6 @@ To create an executable runbook, you will need: - **Kubernetes** - A Kubernetes cluster is required to deploy the rest of the applications. The simplest way to get started is to add a cluster using one of [GitLab's integrations](../add_remove_clusters.md#create-new-cluster). -- **Helm Tiller** - Helm is a package manager for Kubernetes and is required to - install all the other applications. It's installed in its own pod inside the - cluster which can run the Helm CLI in a safe environment. - **Ingress** - Ingress can provide load balancing, SSL termination, and name-based virtual hosting. It acts as a web proxy for your applications. - **JupyterHub** - [JupyterHub](https://jupyterhub.readthedocs.io/) is a multi-user @@ -68,13 +65,8 @@ the components outlined above and the pre-loaded demo runbook. 1. Add a Kubernetes cluster to your project by following the steps outlined in [Create new cluster](../add_remove_clusters.md#create-new-cluster). -1. After the cluster has been provisioned in GKE, click the **Install** button - next to the **Helm Tiller** application to install Helm Tiller. -  - -1. After Helm Tiller has been installed successfully, click the **Install** button next - to the **Ingress** application. +1. Click the **Install** button next to the **Ingress** application to install Ingress.  diff --git a/doc/user/project/clusters/securing.md b/doc/user/project/clusters/securing.md new file mode 100644 index 00000000000..b4c20cb8dbc --- /dev/null +++ b/doc/user/project/clusters/securing.md @@ -0,0 +1,154 @@ +--- +stage: Defend +group: Container Security +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Securing your deployed applications + +GitLab makes it easy to secure applications deployed in [connected Kubernetes clusters](index.md). +You can benefit from the protection of a [Web Application Firewall](../../../topics/web_application_firewall/quick_start_guide.md), +[Network Policies](../../../topics/autodevops/stages.md#network-policy), +or even [Container Host Security](../../clusters/applications.md#install-falco-using-gitlab-cicd). + +This page contains full end-to-end steps and instructions to connect your cluster to GitLab and +install these features, whether or not your applications are deployed through GitLab CI/CD. If you +use [Auto DevOps](../../../topics/autodevops/index.md) +to build and deploy your application with GitLab, see the documentation for the respective +[GitLab Managed Applications](../../clusters/applications.md) +above. + +## Overview + +At a high level, the required steps include the following: + +- Connect the cluster to GitLab. +- Set up one or more runners. +- Set up a cluster management project. +- Install a Web Application Firewall, Network Policies, and/or Container Host + Security. +- Install Prometheus to get statistics and metrics in the + [threat monitoring](../../application_security/threat_monitoring/) + dashboard. + +### Requirements + +Minimum requirements (depending on the GitLab Manage Application you want to install): + +- Your cluster is connected to GitLab (ModSecurity, Cilium, and Falco). +- At least one GitLab Runner is installed (Cilium and Falco only). + +### Understanding how GitLab Managed Apps are installed + +You install GitLab Managed Apps from the GitLab web interface with a one-click setup process. GitLab +uses Sidekiq (a background processing service) to facilitate this. + +```mermaid + sequenceDiagram + autonumber + GitLab->>+Sidekiq: Install a GitLab Managed App + Sidekiq->>+Kubernetes: Helm install + Kubernetes-->>-Sidekiq: Installation complete + Sidekiq-->>-GitLab: Refresh UI +``` + +NOTE: **Note:** +This diagram uses the term _Kubernetes_ for simplicity. In practice, Sidekiq connects to a Helm +Tiller daemon running in a pod in the cluster. + +Although this installation method is easier because it's a point-and-click action in the user +interface, it's inflexible and hard to debug. When something goes wrong, you can't see the +deployment logs. The Web Application Firewall feature uses this installation method. + +However, the next generation of GitLab Managed Apps V2 ([CI/CD-based GitLab Managed Apps](https://gitlab.com/groups/gitlab-org/-/epics/2103)) +don't use Sidekiq to deploy. All the applications are deployed using a GitLab CI/CD pipeline and +therefore GitLab Runners. + +```mermaid +sequenceDiagram + autonumber + GitLab->>+GitLab: Trigger pipeline + GitLab->>+Runner: Run deployment job + Runner->>+Kubernetes: Helm install + Kubernetes-->>-Runner: Installation is complete + Runner-->>-GitLab: Report job status and update pipeline +``` + +Debugging is easier because you have access to the raw logs of these jobs (the Helm Tiller output is +available as an artifact in case of failure) and the flexibility is much better. Since these +deployments are only triggered when a pipeline is running (most likely when there's a new commit in +the cluster management repository), every action has a paper trail and follows the classic merge +request workflow (approvals, merge, deploy). The Network Policy (Cilium) Managed App and Container +Host Security (Falco) are deployed with this model. + +## Connect the cluster to GitLab + +To deploy GitLab Managed Apps to your cluster, you must first +[add your cluster](add_remove_clusters.md) +to GitLab. Then [install](../../clusters/applications.md#installing-applications) +the Web Application Firewall from the project or group Kubernetes page. + +Note that your project doesn't have to be hosted or deployed through GitLab. You can manage a +cluster independent of the applications that use the cluster. + +## Set up a GitLab Runner + +To install CI/CD-based GitLab Managed Apps, a pipeline using a GitLab Runner must be running in +GitLab. You can [install a GitLab Runner](../../clusters/applications.md#gitlab-runner) +in the Kubernetes cluster added in the previous step, or use one of the shared runners provided by +GitLab if you're using GitLab.com. + +With your cluster connected to GitLab and a GitLab Runner in place, you can proceed to the next +steps and start installing the Cilium and Falco GitLab Managed Apps to secure your applications +hosted on this cluster. + +## Create a Cluster Management Project + +A [Cluster Management Project](../../clusters/management_project.md) +is a GitLab project that contains a `.gitlab-ci.yml` file to deploy GitLab Managed Apps to your +cluster. This project runs the required charts with the Kubernetes +[`cluster-admin`](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles) +privileges. + +The creation of this project starts like any other GitLab project. Use an empty +project and add a `gitlab-ci.yml` file at the root, containing this template: + +```yaml +include: + - template: Managed-Cluster-Applications.gitlab-ci.yml +``` + +To make this project a Cluster Management Project, follow these +[instructions](../../clusters/management_project.md#selecting-a-cluster-management-project). +This project can be designated as such even if your application isn't hosted on GitLab. In this +case, create a new empty project where you can select your newly created Cluster Management Project. + +## Install GitLab Container Network Policy + +GitLab Container Network Policy is based on [Cilium](https://cilium.io/). To +install the Cilium GitLab Managed App, add a +`.gitlab/managed-apps/config.yaml` file to your Cluster Management project: + +```yaml +# possible values are gke, eks or you can leave it blank +clusterType: gke + +cilium: + installed: true +``` + +Your application doesn't have to be managed or deployed by GitLab to leverage this feature. +[Read more](../../clusters/applications.md#install-cilium-using-gitlab-cicd) +about configuring Container Network Policy. + +## Install GitLab Container Host Security + +Similarly, you can install Container Host Security, based on +[Falco](https://falco.org/), in your `.gitlab/managed-apps/config.yaml`: + +```yaml +falco: + installed: true +``` + +[Read more] about configuring Container Host Security. diff --git a/doc/user/project/clusters/serverless/aws.md b/doc/user/project/clusters/serverless/aws.md index 15f7e14fda9..595d8fb3895 100644 --- a/doc/user/project/clusters/serverless/aws.md +++ b/doc/user/project/clusters/serverless/aws.md @@ -392,29 +392,19 @@ want to store your package: image: python:latest stages: - - deploy production: - stage: deploy - before_script: - - pip3 install awscli --upgrade - - pip3 install aws-sam-cli --upgrade - script: - - sam build - - sam package --output-template-file packaged.yaml --s3-bucket <S3_bucket_name> - - sam deploy --template-file packaged.yaml --stack-name gitlabpoc --s3-bucket <S3_bucket_name> --capabilities CAPABILITY_IAM --region us-east-1 - environment: production - ``` +``` Let’s examine the configuration file more closely: diff --git a/doc/user/project/clusters/serverless/index.md b/doc/user/project/clusters/serverless/index.md index 45fb313d177..6af08b06294 100644 --- a/doc/user/project/clusters/serverless/index.md +++ b/doc/user/project/clusters/serverless/index.md @@ -51,8 +51,6 @@ To run Knative on GitLab, you will need: 1. **Kubernetes Cluster:** An RBAC-enabled Kubernetes cluster is required to deploy Knative. The simplest way to get started is to add a cluster using GitLab's [GKE integration](../add_remove_clusters.md). The set of minimum recommended cluster specifications to run Knative is 3 nodes, 6 vCPUs, and 22.50 GB memory. -1. **Helm Tiller:** Helm is a package manager for Kubernetes and is required to install - Knative. 1. **GitLab Runner:** A runner is required to run the CI jobs that will deploy serverless applications or functions onto your cluster. You can install the GitLab Runner onto the existing Kubernetes cluster. See [Installing Applications](../index.md#installing-applications) for more information. @@ -80,8 +78,8 @@ To run Knative on GitLab, you will need: NOTE: **Note:** The minimum recommended cluster size to run Knative is 3-nodes, 6 vCPUs, and 22.50 GB memory. **RBAC must be enabled.** -1. [Add a Kubernetes cluster](../add_remove_clusters.md) and [install Helm](../index.md#installing-applications). -1. Once Helm has been successfully installed, scroll down to the Knative app section. Enter the domain to be used with +1. [Add a Kubernetes cluster](../add_remove_clusters.md). +1. Select the **Applications** tab and scroll down to the Knative app section. Enter the domain to be used with your application/functions (e.g. `example.com`) and click **Install**.  @@ -143,24 +141,24 @@ You must do the following: labels: rbac.authorization.k8s.io/aggregate-to-edit: "true" rules: - - apiGroups: - - serving.knative.dev - resources: - - configurations - - configurationgenerations - - routes - - revisions - - revisionuids - - autoscalers - - services - verbs: - - get - - list - - create - - update - - delete - - patch - - watch + - apiGroups: + - serving.knative.dev + resources: + - configurations + - configurationgenerations + - routes + - revisions + - revisionuids + - autoscalers + - services + verbs: + - get + - list + - create + - update + - delete + - patch + - watch ``` Then run the following command: @@ -570,7 +568,7 @@ The simplest way to accomplish this is to use [Certbot to manually obtain Let's Encrypt certificates](https://knative.dev/docs/serving/using-a-tls-cert/#using-certbot-to-manually-obtain-let-s-encrypt-certificates). Certbot is a free, open source software tool for automatically using Let’s Encrypt certificates on manually-administrated websites to enable HTTPS. NOTE: **Note:** -The instructions below relate to installing and running Certbot on a Linux server and may not work on other operating systems. +The instructions below relate to installing and running Certbot on a Linux server that has Python 3 installed and may not work on other operating systems or with other versions of Python. 1. Install Certbot by running the [`certbot-auto` wrapper script](https://certbot.eff.org/docs/install.html#certbot-auto). @@ -580,7 +578,7 @@ The instructions below relate to installing and running Certbot on a Linux serve wget https://dl.eff.org/certbot-auto sudo mv certbot-auto /usr/local/bin/certbot-auto sudo chown root /usr/local/bin/certbot-auto - chmod 0755 /usr/local/bin/certbot-auto + sudo chmod 0755 /usr/local/bin/certbot-auto /usr/local/bin/certbot-auto --help ``` @@ -609,7 +607,7 @@ The instructions below relate to installing and running Certbot on a Linux serve using DNS challenge during authorization: ```shell - ./certbot-auto certonly --manual --preferred-challenges dns -d '*.<namespace>.example.com' + /usr/local/bin/certbot-auto certonly --manual --preferred-challenges dns -d '*.<namespace>.example.com' ``` Where `<namespace>` is the namespace created by GitLab for your serverless project (composed of `<project_name>-<project_id>-<environment>`) and @@ -835,7 +833,7 @@ The instructions below relate to installing and running Certbot on a Linux serve ## Using an older version of `gitlabktl` There may be situations where you want to run an older version of `gitlabktl`. This -requires setting an older version of the `gitlabktl` image in the `.gitlab-ci.yml file.` +requires setting an older version of the `gitlabktl` image in the `.gitlab-ci.yml` file. To set an older version, add `image:` to the `functions:deploy` block. For example: diff --git a/doc/user/project/code_intelligence.md b/doc/user/project/code_intelligence.md new file mode 100644 index 00000000000..e2c2cae3158 --- /dev/null +++ b/doc/user/project/code_intelligence.md @@ -0,0 +1,54 @@ +--- +type: reference +--- + +# Code Intelligence + +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/1576) in GitLab 13.1. + +Code Intelligence adds code navigation features common to interactive +development environments (IDE), including: + +- Type signatures and symbol documentation. +- Go-to definition + +Code Intelligence is built into GitLab and powered by [LSIF](https://lsif.dev/) +(Language Server Index Format), a file format for precomputed code +intelligence data. + +## Configuration + +Enable code intelligence for a project by adding a GitLab CI/CD job to the project's +`.gitlab-ci.yml` which will generate the LSIF artifact: + +```yaml +code_navigation: + image: golang:1.14.0 + allow_failure: true # recommended + script: + - go get github.com/sourcegraph/lsif-go/cmd/lsif-go + - lsif-go + artifacts: + reports: + lsif: dump.lsif +``` + +The generated LSIF file must be less than 170MiB. + +After the job succeeds, code intelligence data can be viewed while browsing the code: + + + +## Language support + +Generating an LSIF file requires a language server indexer implementation for the +relevant language. + +| Language | Implementation | +|---|---| +| Go | [sourcegraph/lsif-go](https://github.com/sourcegraph/lsif-go) | +| JavaScript | [sourcegraph/lsif-node](https://github.com/sourcegraph/lsif-node) | +| TypeScript | [sourcegraph/lsif-node](https://github.com/sourcegraph/lsif-node) | + +View a complete list of [available LSIF indexers](https://lsif.dev/#implementations-server) on their website and +refer to their documentation to see how to generate an LSIF file for your specific language. diff --git a/doc/user/project/code_owners.md b/doc/user/project/code_owners.md index 6b81aea4b87..b35d04dfdfc 100644 --- a/doc/user/project/code_owners.md +++ b/doc/user/project/code_owners.md @@ -22,7 +22,7 @@ who is responsible for each file or path. ## Why is this useful? -Code Owners allows for a version controlled single source of +Code Owners allows for a version controlled, single source of truth file outlining the exact GitLab users or groups that own certain files or paths in a repository. Code Owners can be utilized in the merge request approval process which can streamline @@ -38,18 +38,18 @@ approval. You can use a `CODEOWNERS` file to specify users or [shared groups](members/share_project_with_groups.md) -that are responsible for certain files in a repository. +that are responsible for specific files and directories in a repository. -You can choose and add the `CODEOWNERS` file in three places: +You can choose to add the `CODEOWNERS` file in three places: - To the root directory of the repository - Inside the `.gitlab/` directory - Inside the `docs/` directory -The `CODEOWNERS` file is scoped to a branch, which means that with the -introduction of new files, the person adding the new content can +The `CODEOWNERS` file is scoped to a branch, which means that as +new files are introduced, the user adding the new content can specify themselves as a code owner, all before the new changes -get merged to the default branch. +get merged to the target branch. When a file matches multiple entries in the `CODEOWNERS` file, the users from last pattern matching the file are displayed on the @@ -67,13 +67,13 @@ The user that would show for `README.md` would be `@user2`. ## Approvals by Code Owners -Once you've set Code Owners to a project, you can configure it to +Once you've added Code Owners to a project, you can configure it to be used for merge request approvals: - As [merge request eligible approvers](merge_requests/merge_request_approvals.md#code-owners-as-eligible-approvers). - As required approvers for [protected branches](protected_branches.md#protected-branches-approval-by-code-owners-premium). **(PREMIUM)** -NOTE: **Note**: +NOTE: **Note:** Developer or higher [permissions](../permissions.md) are required in order to approve a merge request. @@ -81,7 +81,7 @@ Once set, Code Owners are displayed in merge requests widgets:  -While the `CODEOWNERS` file can be used in addition to Merge Request [Approval Rules](merge_requests/merge_request_approvals.md#approval-rules) +While the `CODEOWNERS` file can be used in addition to Merge Request [Approval Rules](merge_requests/merge_request_approvals.md#approval-rules), it can also be used as the sole driver of merge request approvals (without using [Approval Rules](merge_requests/merge_request_approvals.md#approval-rules)). To do so, create the file in one of the three locations specified above and @@ -89,28 +89,38 @@ set the code owners as required approvers for [protected branches](protected_bra Use [the syntax of Code Owners files](code_owners.md#the-syntax-of-code-owners-files) to specify the actual owners and granular permissions. -Using Code Owners in conjunction with [Protected Branches Approvals](protected_branches.md#protected-branches-approval-by-code-owners-premium) -will prevent any user who is not specified in the `CODEOWNERS` file from pushing changes -for the specified files/paths, even if their role is included in the **Allowed to push** column. -This allows for a more inclusive push strategy, as administrators don't have to restrict developers -from pushing directly to the protected branch, but can restrict pushing to certain -files where a review by Code Owners is required. +Using Code Owners in conjunction with [Protected Branches](protected_branches.md#protected-branches-approval-by-code-owners-premium) +will prevent any user who is not specified in the `CODEOWNERS` file from pushing +changes for the specified files/paths, even if their role is included in the +**Allowed to push** column. This allows for a more inclusive push strategy, as +administrators don't have to restrict developers from pushing directly to the +protected branch, but can restrict pushing to certain files where a review by +Code Owners is required. ## The syntax of Code Owners files Files can be specified using the same kind of patterns you would use -in the `.gitignore` file followed by the `@username` or email of one -or more users or by the `@name` of one or more groups that should -be owners of the file. Groups must be added as [members of the project](members/index.md), +in the `.gitignore` file followed by one or more of: + +- A user's `@username`. +- A user's email address. +- The `@name` of one or more groups that should be owners of the file. + +Groups must be added as [members of the project](members/index.md), or they will be ignored. -Starting in [GitLab 13.0](https://gitlab.com/gitlab-org/gitlab/-/issues/32432), you can now specify -groups or subgroups from the project's group hierarchy as potential code owners. +Starting in [GitLab 13.0](https://gitlab.com/gitlab-org/gitlab/-/issues/32432), +you can additionally specify groups or subgroups from the project's upper group +hierarchy as potential code owners, without having to invite them specifically +to the project. Groups outside the project's hierarchy or children beneath the +hierarchy must still be explicitly invited to the project in order to show as +Code Owners. -For example, consider the following hierarchy for a given project: +For example, consider the following hierarchy for the example project +`example_project`: ```plaintext -group >> sub-group >> sub-subgroup >> myproject >> file.md +group >> sub-group >> sub-subgroup >> example_project >> file.md ``` Any of the following groups would be eligible to be specified as code owners: @@ -119,7 +129,8 @@ Any of the following groups would be eligible to be specified as code owners: - `@group/sub-group` - `@group/sub-group/sub-subgroup` -In addition, any groups that have been invited to the project using the **Members** tool will also be recognized as eligible code owners. +In addition, any groups that have been invited to the project using the +**Members** tool will also be recognized as eligible code owners. The order in which the paths are defined is significant: the last pattern that matches a given path will be used to find the code @@ -129,7 +140,98 @@ Starting a line with a `#` indicates a comment. This needs to be escaped using `\#` to address files for which the name starts with a `#`. -Example `CODEOWNERS` file: +### Code Owners Sections **(PREMIUM)** + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12137) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.2. +> - It's deployed behind a feature flag, enabled by default. +> - It's enabled on GitLab.com. +> - It can be enabled or disabled per-project. +> - It's recommended for production use. +> - For GitLab self-managed instances, GitLab administrators can opt to [disable it](#enable-or-disable-code-owner-sections-core-only). **(CORE ONLY)** + +Code Owner rules can be grouped into named sections. This allows for better +organization of broader categories of Code Owner rules to be applied. +Additionally, the usual guidance that only the last pattern matching the file is +applied is expanded such that the last pattern matching _for each section_ is +applied. + +For example, in a large organization, independent teams may have a common interest +in parts of the application, for instance, a payment processing company may have +"development", "security", and "compliance" teams looking after common parts of +the codebase. All three teams may need to approve changes. Although approval rules +make this possible, they apply to every merge request. Also, while Code Owners are +applied based on which files are changed, only one CODEOWNERS pattern can match per +file path. + +Using `CODEOWNERS` sections allows multiple teams that only need to approve certain +changes, to set their own independent patterns by specifying discrete sections in the +`CODEOWNERS` file. The section rules may be used for shared paths so that multiple +teams can be added as reviewers. + +Sections can be added to `CODEOWNERS` files as a new line with the name of the +section inside square brackets. Every entry following it is assigned to that +section. The following example would create 2 Code Owner rules for the "README +Owners" section: + +```plaintext +[README Owners] +README.md @user1 @user 2 +internal/README.md @user2 +``` + +Multiple sections can be used, even with matching file or directory patterns. +Reusing the same section name will group the results together under the same +section, with the most specific rule or last matching pattern being used. For +example, consider the following entries in a `CODEOWNERS` file: + +```plaintext +[Documentation] +ee/docs @gl-docs +docs @gl-docs + +[Database] +README.md @gl-database +model/db @gl-database + +[DOCUMENTATION] +README.md @gl-docs +``` + +This will result in 3 entries under the "Documentation" section header, and 2 +entries under "Database". Case is not considered when combining sections, so in +this example, entries defined under the sections "Documentation" and +"DOCUMENTATION" would be combined into one, using the case of the first instance +of the section encountered in the file. + +When assigned to a section, each code owner rule displayed in merge requests +widget is sorted under a "section" label. In the screenshot below, we can see +the rules for "Groups" and "Documentation" sections: + + + +#### Enable or disable Code Owner Sections **(CORE ONLY)** + +Sections is under development but ready for production use. +It is deployed behind a feature flag that is **enabled by default**. +[GitLab administrators with access to the GitLab Rails console](../../administration/feature_flags.md) +can opt to disable it for your instance. + +To disable it: + +```ruby +Feature.disable(:sectional_codeowners) +``` + +To enable it: + +```ruby +Feature.enable(:sectional_codeowners) +``` + +CAUTION: **Caution:** +Disabling Sections will **not** refresh Code Owner Approval Rules on existing merge requests. + +## Example `CODEOWNERS` file ```plaintext # This is an example of a code owners file @@ -185,4 +287,17 @@ lib/ @lib-owner # If the path contains spaces, these need to be escaped like this: path\ with\ spaces/ @space-owner + +# Code Owners section: +[Documentation] +ee/docs @gl-docs +docs @gl-docs + +[Database] +README.md @gl-database +model/db @gl-database + +# This section will be joined with the [Documentation] section previously defined: +[DOCUMENTATION] +README.md @gl-docs ``` diff --git a/doc/user/project/deploy_tokens/index.md b/doc/user/project/deploy_tokens/index.md index 10f0281d0eb..b7daca567f4 100644 --- a/doc/user/project/deploy_tokens/index.md +++ b/doc/user/project/deploy_tokens/index.md @@ -11,7 +11,7 @@ type: howto > - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/199370) from **Settings > Repository** in GitLab 12.9. > - [Added `write_registry` scope](https://gitlab.com/gitlab-org/gitlab/-/issues/22743) in GitLab 12.10. > - [Moved](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/29280) from **Settings > CI / CD** in GitLab 12.10.1. -> - [Added package registry scopes](https://gitlab.com/gitlab-org/gitlab/-/issues/213566) from **Settings > CI / CD** in GitLab 13.0. +> - [Added package registry scopes](https://gitlab.com/gitlab-org/gitlab/-/issues/213566) in GitLab 13.0. Deploy tokens allow you to download (`git clone`) or push and pull packages and container registry images of a project without having a user and a password. @@ -46,15 +46,17 @@ respective **Revoke** button under the 'Active deploy tokens' area. ## Limiting scopes of a deploy token -Deploy tokens can be created with two different scopes that allow various +Deploy tokens can be created with different scopes that allow various actions that a given token can perform. The available scopes are depicted in -the following table. +the following table along with GitLab version it was introduced in. -| Scope | Description | -| ----- | ----------- | -| `read_repository` | Allows read-access to the repository through `git clone` | -| `read_registry` | Allows read-access to [container registry](../../packages/container_registry/index.md) images if a project is private and authorization is required. | -| `write_registry` | Allows write-access (push) to [container registry](../../packages/container_registry/index.md). | +| Scope | Description | Introduced in GitLab Version | +| ----- | ----------- | ------ | +| `read_repository` | Allows read-access to the repository through `git clone` | 10.7 | +| `read_registry` | Allows read-access to [container registry](../../packages/container_registry/index.md) images if a project is private and authorization is required. | 10.7 | +| `write_registry` | Allows write-access (push) to [container registry](../../packages/container_registry/index.md). | 12.10 | +| `read_package_registry` | Allows read access to the package registry. | 13.0 | +| `write_package_registry` | Allows write access to the package registry. | 13.0 | ## Deploy token custom username @@ -96,6 +98,8 @@ pull images from your Container Registry. ### Push Container Registry images +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/22743) in GitLab 12.10. + To push the container registry images, you'll need to: 1. Create a Deploy Token with `write_registry` as a scope. @@ -111,6 +115,8 @@ push images to your Container Registry. ### Read or pull packages +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/213566) in GitLab 13.0. + To pull packages in the GitLab package registry, you'll need to: 1. Create a Deploy Token with `read_package_registry` as a scope. @@ -119,6 +125,8 @@ To pull packages in the GitLab package registry, you'll need to: ### Push or upload packages +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/213566) in GitLab 13.0. + To upload packages in the GitLab package registry, you'll need to: 1. Create a Deploy Token with `write_package_registry` as a scope. @@ -151,8 +159,7 @@ apply consistently when cloning the repository of related projects. There's a special case when it comes to Deploy Tokens. If a user creates one named `gitlab-deploy-token`, the username and token of the Deploy Token will be automatically exposed to the CI/CD jobs as environment variables: `CI_DEPLOY_USER` and -`CI_DEPLOY_PASSWORD`, respectively. With the GitLab Deploy Token, the -`read_registry` and `write_registry` scopes are implied. +`CI_DEPLOY_PASSWORD`, respectively. After you create the token, you can login to the Container Registry using those variables: diff --git a/doc/user/project/description_templates.md b/doc/user/project/description_templates.md index 0f90c321a14..aa5987bf5f9 100644 --- a/doc/user/project/description_templates.md +++ b/doc/user/project/description_templates.md @@ -81,7 +81,7 @@ changes you made after picking the template and return it to its initial status.  -## Setting a default template for merge requests and issues **(STARTER)** +## Setting a default template for merge requests and issues **(STARTER)** > - This feature was introduced before [description templates](#overview) and is available in [GitLab Starter](https://about.gitlab.com/pricing/). It can be enabled in the project's settings. > - Templates for issues were [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/28) in GitLab EE 8.1. diff --git a/doc/user/project/highlighting.md b/doc/user/project/highlighting.md index 2bdb0ae2706..a2740294e62 100644 --- a/doc/user/project/highlighting.md +++ b/doc/user/project/highlighting.md @@ -1,6 +1,11 @@ # Syntax Highlighting -GitLab provides syntax highlighting on all files and snippets through the [Rouge](https://rubygems.org/gems/rouge) rubygem. It will try to guess what language to use based on the file extension, which most of the time is sufficient. +GitLab provides syntax highlighting on all files through the [Rouge](https://rubygems.org/gems/rouge) Ruby gem. It will try to guess what language to use based on the file extension, which most of the time is sufficient. + +NOTE: **Note:** +The [Web IDE](web_ide/index.md) and [Snippets](../snippets.md) use [Monaco Editor](https://microsoft.github.io/monaco-editor/) +for text editing, which internally uses the [Monarch](https://microsoft.github.io/monaco-editor/monarch.html) +library for syntax highlighting. If GitLab is guessing wrong, you can override its choice of language using the `gitlab-language` attribute in `.gitattributes`. For example, if you are working in a Prolog project and using the `.pl` file extension (which would normally be highlighted as Perl), you can add the following to your `.gitattributes` file: @@ -27,3 +32,6 @@ To disable highlighting entirely, use `gitlab-language=text`. Lots more fun shen ``` Please note that these configurations will only take effect when the `.gitattributes` file is in your default branch (usually `master`). + +NOTE: **Note:** +The Web IDE does not support `.gitattribute` files, but it's [planned for a future release](https://gitlab.com/gitlab-org/gitlab/-/issues/22014). diff --git a/doc/user/project/img/bulk-editing.png b/doc/user/project/img/bulk-editing.png Binary files differdeleted file mode 100644 index 8ae649e5020..00000000000 --- a/doc/user/project/img/bulk-editing.png +++ /dev/null diff --git a/doc/user/project/img/bulk-editing_v13_2.png b/doc/user/project/img/bulk-editing_v13_2.png Binary files differnew file mode 100644 index 00000000000..871cb02c01f --- /dev/null +++ b/doc/user/project/img/bulk-editing_v13_2.png diff --git a/doc/user/project/img/code_intelligence_v13_1.png b/doc/user/project/img/code_intelligence_v13_1.png Binary files differnew file mode 100644 index 00000000000..0dff27bab43 --- /dev/null +++ b/doc/user/project/img/code_intelligence_v13_1.png diff --git a/doc/user/project/img/sectional_code_owners_v13.2.png b/doc/user/project/img/sectional_code_owners_v13.2.png Binary files differnew file mode 100644 index 00000000000..894771c26af --- /dev/null +++ b/doc/user/project/img/sectional_code_owners_v13.2.png diff --git a/doc/user/project/import/gemnasium.md b/doc/user/project/import/gemnasium.md index 0d6e059f1cf..3838289aec4 100644 --- a/doc/user/project/import/gemnasium.md +++ b/doc/user/project/import/gemnasium.md @@ -105,7 +105,7 @@ back to both GitLab and GitHub when completed. 1. The result of the job will be visible directly from the pipeline view: -  +  NOTE: **Note:** If you don't commit very often to your project, you may want to use diff --git a/doc/user/project/import/img/jira/import_issues_from_jira_button_v12_10.png b/doc/user/project/import/img/jira/import_issues_from_jira_button_v12_10.png Binary files differindex 4ab42485d0b..3c1dc44df93 100644 --- a/doc/user/project/import/img/jira/import_issues_from_jira_button_v12_10.png +++ b/doc/user/project/import/img/jira/import_issues_from_jira_button_v12_10.png diff --git a/doc/user/project/import/img/jira/import_issues_from_jira_form_v12_10.png b/doc/user/project/import/img/jira/import_issues_from_jira_form_v12_10.png Binary files differindex 6278cb5f970..d98ad2aaa6e 100644 --- a/doc/user/project/import/img/jira/import_issues_from_jira_form_v12_10.png +++ b/doc/user/project/import/img/jira/import_issues_from_jira_form_v12_10.png diff --git a/doc/user/project/import/img/jira/import_issues_from_jira_form_v13_2.png b/doc/user/project/import/img/jira/import_issues_from_jira_form_v13_2.png Binary files differnew file mode 100644 index 00000000000..9cbffe2bb36 --- /dev/null +++ b/doc/user/project/import/img/jira/import_issues_from_jira_form_v13_2.png diff --git a/doc/user/project/import/img/jira/import_issues_from_jira_projects_v12_10.png b/doc/user/project/import/img/jira/import_issues_from_jira_projects_v12_10.png Binary files differdeleted file mode 100644 index bf9728e0311..00000000000 --- a/doc/user/project/import/img/jira/import_issues_from_jira_projects_v12_10.png +++ /dev/null diff --git a/doc/user/project/import/jira.md b/doc/user/project/import/jira.md index 0b8807bb9b3..395cca4726d 100644 --- a/doc/user/project/import/jira.md +++ b/doc/user/project/import/jira.md @@ -40,6 +40,8 @@ Make sure you have the integration set up before trying to import Jira issues. ## Import Jira issues to GitLab +> New import form [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/216145) in GitLab 13.2. + To import Jira issues to a GitLab project, follow the steps below. NOTE: **Note:** @@ -47,27 +49,34 @@ Importing Jira issues is done as an asynchronous background job, which may result in delays based on import queues load, system load, or other factors. Importing large projects may take several minutes depending on the size of the import. -1. On the **{issues}** **Issues** page, click the **Import Issues** (**{import}**) button. -1. Select **Import from Jira**. - This option is only visible if you have the [correct permissions](#permissions). +1. On the **{issues}** **Issues** page, click **Import Issues** (**{import}**) **> Import from Jira**.  + The **Import from Jira** option is only visible if you have the [correct permissions](#permissions). + The following form appears. + If you've previously set up the [Jira integration](../integrations/jira.md), you can now see + the Jira projects that you have access to in the dropdown. + +  -  +1. Click the **Import from** dropdown and select the Jira project that you wish to import issues from. - If you've previously set up the [Jira integration](../integrations/jira.md), you now see the Jira - projects that you have access to in the dropdown. + In the **Jira-GitLab user mapping template** section, the table shows to which GitLab users your Jira + users will be mapped. + If it wasn't possible to match a Jira user with a GitLab user, the dropdown defaults to the user + conducting the import. -1. Select the Jira project that you wish to import issues from. +1. To change any of the suggested mappings, click the dropdown in the **GitLab username** column and + select the user you want to map to each Jira user. -  + The dropdown may not show all the users, so use the search bar to find a specific + user in this GitLab project. + +1. Click **Continue**. You're presented with a confirmation that import has started. -1. Click **Import Issues**. You're presented with a confirmation that import has started. While the import is running in the background, you can navigate away from the import status page to the issues page, and you'll see the new issues appearing in the issues list. -1. To check the status of your import, go back to the Jira import page. - -  +1. To check the status of your import, go to the Jira import page again. diff --git a/doc/user/project/index.md b/doc/user/project/index.md index 3a4e240fb6c..1e71674c16f 100644 --- a/doc/user/project/index.md +++ b/doc/user/project/index.md @@ -78,7 +78,7 @@ When you create a project in GitLab, you'll have access to a large number of timeout (defines the maximum amount of time in minutes that a job is able run), custom path for `.gitlab-ci.yml`, test coverage parsing, pipeline's visibility, and much more - [Kubernetes cluster integration](clusters/index.md): Connecting your GitLab project with a Kubernetes cluster - - [Feature Flags](operations/feature_flags.md): Feature flags allow you to ship a project in + - [Feature Flags](../../operations/feature_flags.md): Feature flags allow you to ship a project in different flavors by dynamically toggling certain functionality **(PREMIUM)** - [GitLab Pages](pages/index.md): Build, test, and deploy your static website with GitLab Pages @@ -104,6 +104,7 @@ When you create a project in GitLab, you'll have access to a large number of - [Dependency List](../application_security/dependency_list/index.md): view project dependencies. **(ULTIMATE)** - [Requirements](requirements/index.md): Requirements allow you to create criteria to check your products against. **(ULTIMATE)** - [Static Site Editor](static_site_editor/index.md): quickly edit content on static websites without prior knowledge of the codebase or Git commands. +- [Code Intelligence](code_intelligence.md): code navigation features. ### Project integrations @@ -172,6 +173,24 @@ Read through the documentation on [project settings](settings/index.md). - [Export a project from GitLab](settings/import_export.md#exporting-a-project-and-its-data) - [Importing and exporting projects between GitLab instances](settings/import_export.md) +## Remove a project + +To remove a project, first navigate to the home page for that project. + +1. Navigate to **Settings > General**. +1. Expand the **Advanced** section. +1. Scroll down to the **Remove project** section. +1. Click **Remove project** +1. Confirm this action by typing in the expected text. + +### Delayed removal **(PREMIUM)** + +By default, clicking to remove a project is followed by a seven day delay. Admins can restore the project during this period of time. +This delay [may be changed by an admin](../admin_area/settings/visibility_and_access_controls.md#default-deletion-adjourned-period-premium-only). + +Admins can view all projects pending deletion. If you're an administrator, go to the top navigation bar, click **Projects > Your projects**, and then select the **Removed projects** tab. +From this tab an admin can restore any project. + ## CI/CD for external repositories **(PREMIUM)** Instead of importing a repository directly to GitLab, you can connect your repository @@ -242,14 +261,52 @@ When [renaming a user](../profile/index.md#changing-your-username), ## Use your project as a Go package -Any project can be used as a Go package including private projects in subgroups. -GitLab responds correctly to `go get` and `godoc.org` discovery requests, -including the [`go-import`](https://golang.org/cmd/go/#hdr-Remote_import_paths) -and [`go-source`](https://github.com/golang/gddo/wiki/Source-Code-Links) meta -tags, respectively. To use packages hosted in private projects with the `go get` -command, use a [`.netrc` file](https://ec.haxx.se/usingcurl-netrc.html) and a -[personal access token](../profile/personal_access_tokens.md) in the password -field. +Any project can be used as a Go package. GitLab responds correctly to `go get` +and `godoc.org` discovery requests, including the +[`go-import`](https://golang.org/cmd/go/#hdr-Remote_import_paths) and +[`go-source`](https://github.com/golang/gddo/wiki/Source-Code-Links) meta tags. + +Private projects, including projects in subgroups, can be used as a Go package, +but may require configuration to work correctly. GitLab will respond correctly +to `go get` discovery requests for projects that *are not* in subgroups, +regardless of authentication or authorization. +[Authentication](#authenticate-go-requests) is required to use a private project +in a subgroup as a Go package. Otherwise, GitLab will truncate the path for +private projects in subgroups to the first two segments, causing `go get` to +fail. + +GitLab implements its own Go proxy. This feature must be enabled by an +administrator and requires additional configuration. See [GitLab Go +Proxy](../packages/go_proxy/index.md). + +### Disable Go module features for private projects + +In Go 1.12 and later, Go queries module proxies and checksum databases in the +process of [fetching a +module](../../development/go_guide/dependencies.md#fetching). This can be +selectively disabled with `GOPRIVATE` (disable both), +[`GONOPROXY`](../../development/go_guide/dependencies.md#proxies) (disable proxy +queries), and [`GONOSUMDB`](../../development/go_guide/dependencies.md#fetching) +(disable checksum queries). + +`GOPRIVATE`, `GONOPROXY`, and `GONOSUMDB` are comma-separated lists of Go +modules and Go module prefixes. For example, +`GOPRIVATE=gitlab.example.com/my/private/project` will disable queries for that +one project, but `GOPRIVATE=gitlab.example.com` will disable queries for *all* +projects on GitLab.com. Go will not query module proxies if the module name or a +prefix of it appears in `GOPRIVATE` or `GONOPROXY`. Go will not query checksum +databases if the module name or a prefix of it appears in `GONOPRIVATE` or +`GONOSUMDB`. + +### Authenticate Go requests + +To authenticate requests to private projects made by Go, use a [`.netrc` +file](https://ec.haxx.se/usingcurl-netrc.html) and a [personal access +token](../profile/personal_access_tokens.md) in the password field. **This only +works if your GitLab instance can be accessed with HTTPS.** The `go` command +will not transmit credentials over insecure connections. This will authenticate +all HTTPS requests made directly by Go but will not authenticate requests made +through Git. For example: @@ -259,6 +316,24 @@ login <gitlab_user_name> password <personal_access_token> ``` +NOTE: **Note:** +On Windows, Go reads `~/_netrc` instead of `~/.netrc`. + +### Authenticate Git fetches + +If a module cannot be fetched from a proxy, Go will fall back to using Git (for +GitLab projects). Git will use `.netrc` to authenticate requests. Alternatively, +Git can be configured to embed specific credentials in the request URL, or to +use SSH instead of HTTPS (as Go always uses HTTPS to fetch Git repositories): + +```shell +# embed credentials in any request to GitLab.com: +git config --global url."https://${user}:${personal_access_token}@gitlab.example.com".insteadOf "https://gitlab.example.com" + +# use SSH instead of HTTPS: +git config --global url."git@gitlab.example.com".insteadOf "https://gitlab.example.com" +``` + ## Access project page with project ID > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/53671) in GitLab 11.8. diff --git a/doc/user/project/insights/index.md b/doc/user/project/insights/index.md index 3cc76beb323..4d646ee2f79 100644 --- a/doc/user/project/insights/index.md +++ b/doc/user/project/insights/index.md @@ -271,6 +271,27 @@ you defined. | `week` | 4 | | `month` | 12 | +#### `query.period_field` + +Define the timestamp field used to group "issuables". + +Supported values are: + +- `created_at` (default): Group data using the `created_at` field. +- `closed_at`: Group data using the `closed_at` field (for issues only). +- `merged_at`: Group data using the `merged_at` field (for merge requests only). + +The `period_field` is automatically set to: + +- `closed_at` if `query.issuable_state` is `closed` +- `merged_at` if `query.issuable_state` is `merged` +- `created_at` otherwise + +NOTE: **Note:** +Until [this bug](https://gitlab.com/gitlab-org/gitlab/-/issues/26911), is resolved, you may see `created_at` +in place of `merged_at`. +`created_at` will be used instead. + ### `projects` > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/10904) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.4. diff --git a/doc/user/project/integrations/bugzilla.md b/doc/user/project/integrations/bugzilla.md index de43c504b05..6d44c56743e 100644 --- a/doc/user/project/integrations/bugzilla.md +++ b/doc/user/project/integrations/bugzilla.md @@ -6,7 +6,6 @@ in the table below. | Field | Description | | ----- | ----------- | -| `description` | A name for the issue tracker (to differentiate between instances, for example) | | `project_url` | The URL to the project in Bugzilla which is being linked to this GitLab project. Note that the `project_url` requires PRODUCT_NAME to be updated with the product/project name in Bugzilla. | | `issues_url` | The URL to the issue in Bugzilla project that is linked to this GitLab project. Note that the `issues_url` requires `:id` in the URL. This ID is used by GitLab as a placeholder to replace the issue number. | | `new_issue_url` | This is the URL to create a new issue in Bugzilla for the project linked to this GitLab project. Note that the `new_issue_url` requires PRODUCT_NAME to be updated with the product/project name in Bugzilla. | diff --git a/doc/user/project/integrations/custom_issue_tracker.md b/doc/user/project/integrations/custom_issue_tracker.md index 4eaf3a0d4b4..7d15ae82b6f 100644 --- a/doc/user/project/integrations/custom_issue_tracker.md +++ b/doc/user/project/integrations/custom_issue_tracker.md @@ -11,8 +11,6 @@ To enable the Custom Issue Tracker integration in a project: | Field | Description | | --------------- | ----------- | - | **Title** | A title for the issue tracker (for example, to differentiate between instances). | - | **Description** | A name for the issue tracker (for example, to differentiate between instances). | | **Project URL** | The URL to the project in the custom issue tracker. | | **Issues URL** | The URL to the issue in the issue tracker project that is linked to this GitLab project. Note that the `issues_url` requires `:id` in the URL. This ID is used by GitLab as a placeholder to replace the issue number. For example, `https://customissuetracker.com/project-name/:id`. | | **New issue URL** | Currently unused. Will be changed in a future release. | diff --git a/doc/user/project/integrations/generic_alerts.md b/doc/user/project/integrations/generic_alerts.md index 57c1e54e48c..3490a3f2b9e 100644 --- a/doc/user/project/integrations/generic_alerts.md +++ b/doc/user/project/integrations/generic_alerts.md @@ -18,16 +18,21 @@ create an issue with the payload in the body of the issue. You can always The entire payload will be posted in the issue discussion as a comment authored by the GitLab Alert Bot. -NOTE: **Note** -In GitLab versions 13.1 and greater, you can configure [External Prometheus instances](prometheus.md#external-prometheus-instances) to use this endpoint. +NOTE: **Note:** +In GitLab versions 13.1 and greater, you can configure +[External Prometheus instances](../../../operations/metrics/alerts.md#external-prometheus-instances) +to use this endpoint. ## Setting up generic alerts -To set up the generic alerts integration: +To obtain credentials for setting up a generic alerts integration: -1. Navigate to **Settings > Integrations** in a project. -1. Click on **Alerts endpoint**. -1. Toggle the **Active** alert setting. The `URL` and `Authorization Key` for the webhook configuration can be found there. +- Sign in to GitLab as a user with maintainer [permissions](../../permissions.md) for a project. +- Navigate to the **Operations** page for your project, depending on your installed version of GitLab: + - *In GitLab versions 13.1 and greater,* navigate to **{settings}** **Settings > Operations** in your project. + - *In GitLab versions prior to 13.1,* navigate to **{settings}** **Settings > Integrations** in your project. GitLab will display a banner encouraging you to enable the Alerts endpoint in **{settings}** **Settings > Operations** instead. +- Click **Alerts endpoint**. +- Toggle the **Active** alert setting to display the **URL** and **Authorization Key** for the webhook configuration. ## Customizing the payload @@ -44,6 +49,14 @@ You can customize the payload by sending the following parameters. All fields ot | `severity` | String | The severity of the alert. Must be one of `critical`, `high`, `medium`, `low`, `info`, `unknown`. Default is `critical`. | | `fingerprint` | String or Array | The unique identifier of the alert. This can be used to group occurrences of the same alert. | +You can also add custom fields to the alert's payload. The values of extra parameters +are not limited to primitive types, such as strings or numbers, but can be a nested +JSON object. For example: + +```json +{ "foo": { "bar": { "baz": 42 } } } +``` + TIP: **Payload size:** Ensure your requests are smaller than the [payload application limits](../../../administration/instance_limits.md#generic-alert-json-payloads). @@ -70,6 +83,42 @@ Example payload: "monitoring_tool": "value", "hosts": "value", "severity": "high", - "fingerprint": "d19381d4e8ebca87b55cda6e8eee7385" + "fingerprint": "d19381d4e8ebca87b55cda6e8eee7385", + "foo": { + "bar": { + "baz": 42 + } + } } ``` + +## Triggering test alerts + +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3066) in GitLab Core in 13.2. + +After a [project maintainer or owner](#setting-up-generic-alerts) +[configures generic alerts](#setting-up-generic-alerts), you can trigger a +test alert to confirm your integration works properly. + +1. Sign in as a user with Developer or greater [permissions](../../../user/permissions.md). +1. Navigate to **{settings}** **Settings > Operations** in your project. +1. Click **Alerts endpoint** to expand the section. +1. Enter a sample payload in **Alert test payload** (valid JSON is required). +1. Click **Test alert payload**. + +GitLab displays an error or success message, depending on the outcome of your test. + +## Automatic grouping of identical alerts **(PREMIUM)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/214557) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.2. + +In GitLab versions 13.2 and greater, GitLab groups alerts based on their payload. +When an incoming alert contains the same payload as another alert (excluding the +`start_time` and `hosts` attributes), GitLab groups these alerts together and +displays a counter on the +[Alert Management List](../operations/alert_management.md#alert-management-list) +and details pages. + +If the existing alert is already `resolved`, then a new alert will be created instead. + + diff --git a/doc/user/project/integrations/github.md b/doc/user/project/integrations/github.md index f0977a4ea76..416996fb629 100644 --- a/doc/user/project/integrations/github.md +++ b/doc/user/project/integrations/github.md @@ -14,7 +14,7 @@ and is automatically configured on [GitHub import](../../../integration/github.m ### Complete these steps on GitHub -This integration requires a [GitHub API token](https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line) +This integration requires a [GitHub API token](https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token) with `repo:status` access granted: 1. Go to your "Personal access tokens" page at <https://github.com/settings/tokens> diff --git a/doc/user/project/integrations/img/actions_menu_create_new_dashboard_v13_2.png b/doc/user/project/integrations/img/actions_menu_create_new_dashboard_v13_2.png Binary files differnew file mode 100644 index 00000000000..5d530a80421 --- /dev/null +++ b/doc/user/project/integrations/img/actions_menu_create_new_dashboard_v13_2.png diff --git a/doc/user/project/integrations/img/heatmap_chart_too_much_data_v_13_2.png b/doc/user/project/integrations/img/heatmap_chart_too_much_data_v_13_2.png Binary files differnew file mode 100644 index 00000000000..c3a391b06c7 --- /dev/null +++ b/doc/user/project/integrations/img/heatmap_chart_too_much_data_v_13_2.png diff --git a/doc/user/project/integrations/img/jira/open_jira_issues_list_v13.2.png b/doc/user/project/integrations/img/jira/open_jira_issues_list_v13.2.png Binary files differnew file mode 100644 index 00000000000..0cf58433b25 --- /dev/null +++ b/doc/user/project/integrations/img/jira/open_jira_issues_list_v13.2.png diff --git a/doc/user/project/integrations/img/jira_service_page_v12_2.png b/doc/user/project/integrations/img/jira_service_page_v12_2.png Binary files differdeleted file mode 100644 index ba7dad9b438..00000000000 --- a/doc/user/project/integrations/img/jira_service_page_v12_2.png +++ /dev/null diff --git a/doc/user/project/integrations/img/metrics_settings_button_v13_2.png b/doc/user/project/integrations/img/metrics_settings_button_v13_2.png Binary files differnew file mode 100644 index 00000000000..d649f77eded --- /dev/null +++ b/doc/user/project/integrations/img/metrics_settings_button_v13_2.png diff --git a/doc/user/project/integrations/img/prometheus_manual_configuration_v13_2.png b/doc/user/project/integrations/img/prometheus_manual_configuration_v13_2.png Binary files differnew file mode 100644 index 00000000000..b6ec08f492d --- /dev/null +++ b/doc/user/project/integrations/img/prometheus_manual_configuration_v13_2.png diff --git a/doc/user/project/integrations/img/prometheus_service_configuration.png b/doc/user/project/integrations/img/prometheus_service_configuration.png Binary files differdeleted file mode 100644 index a38d1bce197..00000000000 --- a/doc/user/project/integrations/img/prometheus_service_configuration.png +++ /dev/null diff --git a/doc/user/project/integrations/jira.md b/doc/user/project/integrations/jira.md index 442f3229de2..541c65041ad 100644 --- a/doc/user/project/integrations/jira.md +++ b/doc/user/project/integrations/jira.md @@ -55,29 +55,41 @@ In order to enable the Jira service in GitLab, you need to first configure the p > **Notes:** > -> - The currently supported Jira versions are `v6.x, v7.x, v8.x` . GitLab 7.8 or -> higher is required. -> - GitLab 8.14 introduced a new way to integrate with Jira which greatly simplified -> the configuration options you have to enter. If you are using an older version, -> [follow this documentation](https://gitlab.com/gitlab-org/gitlab/blob/8-13-stable-ee/doc/project_services/jira.md). +> - The supported Jira versions are `v6.x`, `v7.x`, and `v8.x`. > - In order to support Oracle's Access Manager, GitLab will send additional cookies > to enable Basic Auth. The cookie being added to each request is `OBBasicAuth` with > a value of `fromDialog`. To enable the Jira integration in a project, navigate to the -[Integrations page](overview.md#accessing-integrations), click -the **Jira** service, and fill in the required details on the page as described -in the table below. +[Integrations page](overview.md#accessing-integrations) and click +the **Jira** service. + +Select **Enable integration**. + +Select a **Trigger** action. This determines whether a mention of a Jira issue in GitLab commits, merge requests, or both, should link the Jira issue back to that source commit/MR and transition the Jira issue, if indicated. + +To include a comment on the Jira issue when the above referene is made in GitLab, check **Enable comments**. + +Enter the further details on the page as described in the following table. | Field | Description | | ----- | ----------- | | `Web URL` | The base URL to the Jira instance web interface which is being linked to this GitLab project. E.g., `https://jira.example.com`. | | `Jira API URL` | The base URL to the Jira instance API. Web URL value will be used if not set. E.g., `https://jira-api.example.com`. Leave this field blank (or use the same value of `Web URL`) if using **Jira Cloud**. | -| `Username/Email` | Created when [configuring Jira step](#configuring-jira). Use `username` for **Jira Server** or `email` for **Jira Cloud**. | -| `Password/API token` |Created in [configuring Jira step](#configuring-jira). Use `password` for **Jira Server** or `API token` for **Jira Cloud**. | -| `Transition ID` | This is the ID of a transition that moves issues to the desired state. It is possible to insert transition ids separated by `,` or `;` which means the issue will be moved to each state after another using the given order. **Closing Jira issues via commits or Merge Requests won't work if you don't set the ID correctly.** | +| `Username or Email` | Created in [configuring Jira](#configuring-jira) step. Use `username` for **Jira Server** or `email` for **Jira Cloud**. | +| `Password/API token` |Created in [configuring Jira](#configuring-jira) step. Use `password` for **Jira Server** or `API token` for **Jira Cloud**. | +| `Transition ID` | Required for closing Jira issues via commits or merge requests. This is the ID of a transition in Jira that moves issues to a desired state. (See [Obtaining a transition ID](#obtaining-a-transition-id).) If you insert multiple transition IDs separated by `,` or `;`, the issue is moved to each state, one after another, using the given order. | + +To enable users to view Jira issues inside GitLab, select **Enable Jira issues** and enter a project key. **(PREMIUM)** + +CAUTION: **Caution:** +If you enable Jira issues with the setting above, all users that have access to this GitLab project will be able to view all issues from the specified Jira project. + +When you have configured all settings, click **Test settings and save changes**. -### Obtaining a transition ID +Your GitLab project can now interact with all Jira projects in your instance and the project now displays a Jira link that opens the Jira project. + +#### Obtaining a transition ID In the most recent Jira user interface, you can no longer see transition IDs in the workflow administration UI. You can get the ID you need in either of the following ways: @@ -90,19 +102,11 @@ administration UI. You can get the ID you need in either of the following ways: Note that the transition ID may vary between workflows (e.g., bug vs. story), even if the status you are changing to is the same. -After saving the configuration, your GitLab project will be able to interact -with all Jira projects in your Jira instance and you'll see the Jira link on the GitLab project pages that takes you to the appropriate Jira project. - - - -### Disabling comments on Jira issues - -When you reference a Jira issue, it will always link back to the source commit/MR in GitLab, however, you can control whether GitLab will also cross-post a comment to the Jira issue. That functionality is enabled by default. +#### Disabling comments on Jira issues -To disable the automated commenting on Jira issues: +You can continue to have GitLab cross-link a source commit/MR with a Jira issue while disabling the comment added to the issue. -1. Open the [Integrations page](overview.md#accessing-integrations) and select **Jira**. -1. In the **Event Action** section, uncheck **Comment**. +See the [Configuring GitLab](#configuring-gitlab) section and uncheck the **Enable comments** setting. ## Jira issues @@ -111,7 +115,7 @@ By now you should have [configured Jira](#configuring-jira) and enabled the you should be able to reference and close Jira issues by just mentioning their ID in GitLab commits and merge requests. -### Referencing Jira Issues +### Reference Jira issues When GitLab project has Jira issue tracker configured and enabled, mentioning Jira issue in GitLab will automatically add a comment in Jira issue with the @@ -138,7 +142,7 @@ For example, the following commit will reference the Jira issue with `PROJECT-1` git commit -m "PROJECT-1 Fix spelling and grammar" ``` -### Closing Jira Issues +### Close Jira issues Jira issues can be closed directly from GitLab by using trigger words in commits and merge requests. When a commit which contains the trigger word @@ -162,8 +166,6 @@ where `PROJECT-1` is the ID of the Jira issue. > [project settings](img/jira_project_settings.png). > - The Jira issue will not be transitioned if it has a resolution. -### Jira issue closing example - Let's consider the following example: 1. For the project named `PROJECT` in Jira, we implemented a new feature @@ -185,6 +187,45 @@ with a link to the commit that resolved the issue.  +### View Jira issues **(PREMIUM)** + +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3622) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.2. + +You can browse and search issues from a selected Jira project directly in GitLab. This requires [configuration](#configuring-gitlab) in GitLab by an administrator. + + + +From the **Jira Issues** menu, click **Issues List**. The issue list defaults to sort by **Created date**, with the newest issues listed at the top. You can change this to **Last updated**. + +Issues are grouped into tabs based on their [Jira status](https://confluence.atlassian.com/adminjiraserver070/defining-status-field-values-749382903.html). + +- The **Open** tab displays all issues with a Jira status in any category other than Done. +- The **Closed** tab displays all issues with a Jira status categorized as Done. +- The **All** tab displays all issues of any status. + +Click an issue title to open its original Jira issue page for full details. + +#### Search and filter the issues list + +To refine the list of issues, use the search bar to search for any text +contained in an issue summary (title) or description. + +You can also filter by labels, status, reporter, and assignee using URL parameters. +Enhancements to be able to use these through the user interface are [planned](https://gitlab.com/groups/gitlab-org/-/epics/3622). + +- To filter issues by `labels`, specify one or more labels as part of the `labels[]` +parameter in the URL. When using multiple labels, only issues that contain all specified +labels are listed. `/-/integrations/jira/issues?labels[]=backend&labels[]=feature&labels[]=QA` + +- To filter issues by `status`, specify the `status` parameter in the URL. +`/-/integrations/jira/issues?status=In Progress` + +- To filter issues by `reporter`, specify a reporter's Jira display name for the +`author_username` parameter in the URL. `/-/integrations/jira/issues?author_username=John Smith` + +- To filter issues by `assignee`, specify their Jira display name for the +`assignee_username` parameter in the URL. `/-/integrations/jira/issues?assignee_username=John Smith` + ## Troubleshooting If these features do not work as expected, it is likely due to a problem with the way the integration settings were configured. diff --git a/doc/user/project/integrations/jira_cloud_configuration.md b/doc/user/project/integrations/jira_cloud_configuration.md index 619c94b282b..c7157b6bd0e 100644 --- a/doc/user/project/integrations/jira_cloud_configuration.md +++ b/doc/user/project/integrations/jira_cloud_configuration.md @@ -5,7 +5,7 @@ below to create one: 1. Log in to [`id.atlassian.com`](https://id.atlassian.com/manage-profile/security/api-tokens) with your email address. - NOTE: **Note** + NOTE: **Note:** It is important that the user associated with this email address has *write* access to projects in Jira. diff --git a/doc/user/project/integrations/jira_server_configuration.md b/doc/user/project/integrations/jira_server_configuration.md index 1efd0ff3944..c8278a0f083 100644 --- a/doc/user/project/integrations/jira_server_configuration.md +++ b/doc/user/project/integrations/jira_server_configuration.md @@ -6,7 +6,7 @@ need to integrate with GitLab. As an example, we'll create a user named `gitlab` and add it to a new group named `gitlab-developers`. -NOTE: **Note** +NOTE: **Note:** It is important that the Jira user created for the integration is given 'write' access to your Jira projects. This is covered in the process below. diff --git a/doc/user/project/integrations/overview.md b/doc/user/project/integrations/overview.md index 88668ab6c7d..79c55e2d140 100644 --- a/doc/user/project/integrations/overview.md +++ b/doc/user/project/integrations/overview.md @@ -14,8 +14,6 @@ want to configure.  -Below, you will find a list of the currently supported ones accompanied with comprehensive documentation. - ## Integrations listing Click on the service links to see further configuration instructions and details. @@ -28,6 +26,7 @@ Click on the service links to see further configuration instructions and details | Buildkite | Continuous integration and deployments | Yes | | [Bugzilla](bugzilla.md) | Bugzilla issue tracker | No | | Campfire | Simple web-based real-time group chat | No | +| [Confluence](../../../api/services.md#confluence-service) | Replaces the link to the internal wiki with a link to a Confluence Cloud Workspace | No | | Custom Issue Tracker | Custom issue tracker | No | | [Discord Notifications](discord_notifications.md) | Receive event notifications in Discord | No | | Drone CI | Continuous Integration platform built on Docker, written in Go | Yes | @@ -68,16 +67,16 @@ supported by `push_hooks` and `tag_push_hooks` events won't be executed. The number of branches or tags supported can be changed via [`push_event_hooks_limit` application setting](../../../api/settings.md#list-of-settings-that-can-be-accessed-via-api-calls). -## Services templates +## Service templates -Services templates is a way to set some predefined values in the Service of -your liking which will then be pre-filled on each project's Service. +Service templates are a way to set predefined values for an integration across +all new projects on the instance. -Read more about [Services templates in this document](services_templates.md). +Read more about [Service templates in this document](services_templates.md). ## Troubleshooting integrations -Some integrations use service hooks for integration with external applications. To confirm which ones use service hooks, see the [integrations listing](#integrations-listing). GitLab stores details of service hook requests made within the last 2 days. To view details of the requests, go to the service's configuration page. +Some integrations use service hooks for integration with external applications. To confirm which ones use service hooks, see the [integrations listing](#integrations-listing) above. GitLab stores details of service hook requests made within the last 2 days. To view details of the requests, go to that integration's configuration page. The **Recent Deliveries** section lists the details of each request made within the last 2 days: @@ -88,17 +87,17 @@ The **Recent Deliveries** section lists the details of each request made within - Relative time in which the request was made To view more information about the request's execution, click the respective **View details** link. -On the details page, you can see the data sent by GitLab (request headers and body) and the data received by GitLab (response headers and body). +On the details page, you can see the request headers and body sent and received by GitLab. -From this page, you can repeat delivery with the same data by clicking **Resend Request**. +To repeat a delivery using the same data, click **Resend Request**.  ### Uninitialized repositories Some integrations fail with an error `Test Failed. Save Anyway` when you attempt to set them up on -uninitialized repositories. This is because the default service test uses push data to build the -payload for the test request, and it fails, because there are no push events for the project. +uninitialized repositories. Some integrations use push data to build the test payload, +and this error occurs when no push events exist in the project yet. To resolve this error, initialize the repository by pushing a test file to the project and set up the integration again. diff --git a/doc/user/project/integrations/prometheus.md b/doc/user/project/integrations/prometheus.md index 0d17ff51372..f1567208a8f 100644 --- a/doc/user/project/integrations/prometheus.md +++ b/doc/user/project/integrations/prometheus.md @@ -19,7 +19,8 @@ There are two ways to set up Prometheus integration, depending on where your app - For deployments on Kubernetes, GitLab can automatically [deploy and manage Prometheus](#managed-prometheus-on-kubernetes). - For other deployment targets, simply [specify the Prometheus server](#manual-configuration-of-prometheus). -Once enabled, GitLab will automatically detect metrics from known services in the [metric library](#monitoring-cicd-environments). You can also [add your own metrics](#adding-custom-metrics). +Once enabled, GitLab will automatically detect metrics from known services in the [metric library](prometheus_library/index.md). You can also [add your own metrics](../../../operations/metrics/index.md#adding-custom-metrics) and create +[custom dashboards](../../../operations/metrics/dashboards/index.md). ## Enabling Prometheus Integration @@ -32,11 +33,10 @@ GitLab can seamlessly deploy and manage Prometheus on a [connected Kubernetes cl #### Requirements - A [connected Kubernetes cluster](../clusters/index.md) -- Helm Tiller [installed by GitLab](../clusters/index.md#installing-applications) #### Getting started -Once you have a connected Kubernetes cluster with Helm installed, deploying a managed Prometheus is as easy as a single click. +Once you have a connected Kubernetes cluster, deploying a managed Prometheus is as easy as a single click. 1. Go to the **Operations > Kubernetes** page to view your connected clusters 1. Select the cluster you would like to deploy Prometheus to @@ -44,53 +44,6 @@ Once you have a connected Kubernetes cluster with Helm installed, deploying a ma  -#### Getting metrics to display on the Metrics Dashboard - -After completing the steps above, you will also need deployments in order to view the -**Operations > Metrics** page. Setting up [Auto DevOps](../../../topics/autodevops/index.md) -will help you to quickly create a deployment: - -1. Navigate to your project's **Operations > Kubernetes** page, and ensure that, - in addition to "Prometheus" and "Helm Tiller", you also have "Runner" and "Ingress" - installed. Once "Ingress" is installed, copy its endpoint. -1. Navigate to your project's **Settings > CI/CD** page. In the Auto DevOps section, - select a deployment strategy and save your changes. -1. On the same page, in the Variables section, add a variable named `KUBE_INGRESS_BASE_DOMAIN` - with the value of the Ingress endpoint you have copied in the previous step. Leave the type - as "Variable". -1. Navigate to your project's **CI/CD > Pipelines** page, and run a pipeline on any branch. -1. When the pipeline has run successfully, graphs will be available on the **Operations > Metrics** page. - - - -#### Using the Metrics Dashboard - -##### Select an environment - -The **Environment** dropdown box above the dashboard displays the list of all [environments](#monitoring-cicd-environments). -It enables you to search as you type through all environments and select the one you're looking for. - - - -##### Select a dashboard - -The **dashboard** dropdown box above the dashboard displays the list of all dashboards available for the project. -It enables you to search as you type through all dashboards and select the one you're looking for. - - - -##### Mark a dashboard as favorite - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/214582) in GitLab 13.0. - -When viewing a dashboard, click the empty **Star dashboard** **{star-o}** button to mark a -dashboard as a favorite. Starred dashboards display a solid star **{star}** button, -and appear at the top of the dashboard select list. - -To remove dashboard from the favorites list, click the solid **Unstar Dashboard** **{star}** button. - - - #### About managed Prometheus deployments Prometheus is deployed into the `gitlab-managed-apps` namespace, using the [official Helm chart](https://github.com/helm/charts/tree/master/stable/prometheus). Prometheus is only accessible within the cluster, with GitLab communicating through the [Kubernetes API](https://kubernetes.io/docs/concepts/overview/kubernetes-api/). @@ -128,14 +81,24 @@ Installing and configuring Prometheus to monitor applications is fairly straight The actual configuration of Prometheus integration within GitLab is very simple. All you will need is the domain name or IP address of the Prometheus server you'd like -to integrate with. - -1. Navigate to the [Integrations page](overview.md#accessing-integrations). +to integrate with. If the Prometheus resource is secured with Google's Identity-Aware Proxy (IAP), +additional information like Client ID and Service Account credentials can be passed which +GitLab can use to access the resource. More information about authentication from a +service account can be found at Google's documentation for +[Authenticating from a service account](https://cloud.google.com/iap/docs/authentication-howto#authenticating_from_a_service_account). + +1. Navigate to the [Integrations page](overview.md#accessing-integrations) at + **{settings}** **Settings > Integrations**. 1. Click the **Prometheus** service. -1. Provide the domain name or IP address of your server, for example `http://prometheus.example.com/` or `http://192.0.2.1/`. +1. For **API URL**, provide the domain name or IP address of your server, such as + `http://prometheus.example.com/` or `http://192.0.2.1/`. +1. (Optional) In **Google IAP Audience Client ID**, provide the Client ID of the + Prometheus OAuth Client secured with Google IAP. +1. (Optional) In **Google IAP Service Account JSON**, provide the contents of the + Service Account credentials file that is authorized to access the Prometheus resource. 1. Click **Save changes**. - + #### Thanos configuration in GitLab @@ -158,7 +121,7 @@ one of them will be used: [Prometheus manual configuration](#manual-configuration-of-prometheus) and a [managed Prometheus on Kubernetes](#managed-prometheus-on-kubernetes), the manual configuration takes precedence and is used to run queries from - [dashboards](#defining-custom-dashboards-per-project) and [custom metrics](#adding-custom-metrics). + [dashboards](../../../operations/metrics/dashboards/index.md#defining-custom-dashboards-per-project) and [custom metrics](../../../operations/metrics/index.md#adding-custom-metrics). - If you have managed Prometheus applications installed on Kubernetes clusters at **different** levels (project, group, instance), the order of precedence is described in [Cluster precedence](../../instance/clusters/index.md#cluster-precedence). @@ -166,884 +129,6 @@ one of them will be used: clusters at the **same** level, the Prometheus application of a cluster with a matching [environment scope](../../../ci/environments/index.md#scoping-environments-with-specs) is used. -## Monitoring CI/CD Environments - -Once configured, GitLab will attempt to retrieve performance metrics for any -environment which has had a successful deployment. - -GitLab will automatically scan the Prometheus server for metrics from known servers like Kubernetes and NGINX, and attempt to identify individual environments. The supported metrics and scan process is detailed in our [Prometheus Metrics Library documentation](prometheus_library/index.md). - -You can view the performance dashboard for an environment by [clicking on the monitoring button](../../../ci/environments/index.md#monitoring-environments). - -### Adding custom metrics - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/3799) in [GitLab Premium](https://about.gitlab.com/pricing/) 10.6. -> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/28527) to [GitLab Core](https://about.gitlab.com/pricing/) 12.10. - -Custom metrics can be monitored by adding them on the monitoring dashboard page. Once saved, they will be displayed on the environment performance dashboard provided that either: - -- A [connected Kubernetes cluster](../clusters/add_remove_clusters.md) with the environment scope of `*` is used and [Prometheus installed on the cluster](#enabling-prometheus-integration) -- Prometheus is [manually configured](#manual-configuration-of-prometheus). - - - -A few fields are required: - -- **Name**: Chart title -- **Type**: Type of metric. Metrics of the same type will be shown together. -- **Query**: Valid [PromQL query](https://prometheus.io/docs/prometheus/latest/querying/basics/). -- **Y-axis label**: Y axis title to display on the dashboard. -- **Unit label**: Query units, for example `req / sec`. Shown next to the value. - -Multiple metrics can be displayed on the same chart if the fields **Name**, **Type**, and **Y-axis label** match between metrics. For example, a metric with **Name** `Requests Rate`, **Type** `Business`, and **Y-axis label** `rec / sec` would display on the same chart as a second metric with the same values. A **Legend label** is suggested if this feature is used. - -#### Query Variables - -##### Predefined variables - -GitLab supports a limited set of [CI variables](../../../ci/variables/README.md) in the Prometheus query. This is particularly useful for identifying a specific environment, for example with `ci_environment_slug`. The supported variables are: - -- `ci_environment_slug` -- `kube_namespace` -- `ci_project_name` -- `ci_project_namespace` -- `ci_project_path` -- `ci_environment_name` -- `__range` - -NOTE: **Note:** -Variables for Prometheus queries must be lowercase. - -###### __range - -The `__range` variable is useful in Prometheus -[range vector selectors](https://prometheus.io/docs/prometheus/latest/querying/basics/#range-vector-selectors). -Its value is the total number of seconds in the dashboard's time range. -For example, if the dashboard time range is set to 8 hours, the value of -`__range` is `28800s`. - -##### User-defined variables - -[Variables can be defined](#templating-templating-properties) in a custom dashboard YAML file. - -##### Using variables - -Variables can be specified using double curly braces, such as `"{{ci_environment_slug}}"` ([added](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/20793) in GitLab 12.7). - -Support for the `"%{ci_environment_slug}"` format was -[removed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/31581) in GitLab 13.0. -Queries that continue to use the old format will show no data. - -#### Query Variables from URL - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/214500) in GitLab 13.0. - -GitLab supports setting custom variables through URL parameters. Surround the variable -name with double curly braces (`{{example}}`) to interpolate the variable in a query: - -```plaintext -avg(sum(container_memory_usage_bytes{container_name!="{{pod}}"}) by (job)) without (job) /1024/1024/1024' -``` - -The URL for this query would be: - -```plaintext -http://gitlab.com/<user>/<project>/-/environments/<environment_id>/metrics?dashboard=.gitlab%2Fdashboards%2Fcustom.yml&pod=POD -``` - -#### Editing additional metrics from the dashboard - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/208976) in GitLab 12.9. - -You can edit existing additional custom metrics by clicking the **{ellipsis_v}** **More actions** dropdown and selecting **Edit metric**. - - - -### Defining custom dashboards per project - -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/59974) in GitLab 12.1. - -By default, all projects include a GitLab-defined Prometheus dashboard, which -includes a few key metrics, but you can also define your own custom dashboards. - -You may create a new file from scratch or duplicate a GitLab-defined Prometheus -dashboard. - -NOTE: **Note:** -The metrics as defined below do not support alerts, unlike -[custom metrics](#adding-custom-metrics). - -#### Adding a new dashboard to your project - -You can configure a custom dashboard by adding a new YAML file into your project's -`.gitlab/dashboards/` directory. In order for the dashboards to be displayed on -the project's **Operations > Metrics** page, the files must have a `.yml` -extension and should be present in the project's **default** branch. - -For example: - -1. Create `.gitlab/dashboards/prom_alerts.yml` under your repository's root - directory with the following contents: - - ```yaml - dashboard: 'Dashboard Title' - panel_groups: - - group: 'Group Title' - panels: - - type: area-chart - title: "Chart Title" - y_label: "Y-Axis" - y_axis: - format: number - precision: 0 - metrics: - - id: my_metric_id - query_range: 'http_requests_total' - label: "Instance: {{instance}}, method: {{method}}" - unit: "count" - ``` - - The above sample dashboard would display a single area chart. Each file should - define the layout of the dashboard and the Prometheus queries used to populate - data. - -1. Save the file, commit, and push to your repository. The file must be present in your **default** branch. -1. Navigate to your project's **Operations > Metrics** and choose the custom - dashboard from the dropdown. - -NOTE: **Note:** -Configuration files nested under subdirectories of `.gitlab/dashboards` are not -supported and will not be available in the UI. - -#### Duplicating a GitLab-defined dashboard - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/37238) in GitLab 12.7. -> - From [GitLab 12.8 onwards](https://gitlab.com/gitlab-org/gitlab/-/issues/39505), custom metrics are also duplicated when you duplicate a dashboard. - -You can save a complete copy of a GitLab defined dashboard along with all custom metrics added to it. -Resulting `.yml` file can be customized and adapted to your project. -You can decide to save the dashboard `.yml` file in the project's **default** branch or in a -new branch. - -1. Click **Duplicate dashboard** in the dashboard dropdown. - - NOTE: **Note:** - You can duplicate only GitLab-defined dashboards. - -1. Enter the file name and other information, such as the new commit's message, and click **Duplicate**. - -If you select your **default** branch, the new dashboard becomes immediately available. -If you select another branch, this branch should be merged to your **default** branch first. - -#### Dashboard YAML syntax validation - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/33202) in GitLab 13.1. - -To confirm your dashboard definition contains valid YAML syntax: - -1. Navigate to **{doc-text}** **Repository > Files**. -1. Navigate to your dashboard file in your repository. -1. Review the information pane about the file, displayed above the file contents. - -Files with valid syntax display **Metrics Dashboard YAML definition is valid**, -and files with invalid syntax display **Metrics Dashboard YAML definition is invalid**. - - - -When **Metrics Dashboard YAML definition is invalid** at least one of the following messages is displayed: - -1. `dashboard: can't be blank` [learn more](#dashboard-top-level-properties) -1. `panel_groups: can't be blank` [learn more](#dashboard-top-level-properties) -1. `group: can't be blank` [learn more](#panel-group-panel_groups-properties) -1. `panels: can't be blank` [learn more](#panel-group-panel_groups-properties) -1. `metrics: can't be blank` [learn more](#panel-panels-properties) -1. `title: can't be blank` [learn more](#panel-panels-properties) -1. `query: can't be blank` [learn more](#metrics-metrics-properties) -1. `query_range: can't be blank` [learn more](#metrics-metrics-properties) -1. `unit: can't be blank` [learn more](#metrics-metrics-properties) -1. `YAML syntax: The parsed YAML is too big` - - This is displayed when the YAML file is larger than 1 MB. - -1. `YAML syntax: Invalid configuration format` - - This is displayed when the YAML file is empty or does not contain valid YAML. - -Metrics Dashboard YAML definition validation information is also available as a [GraphQL API field](../../../api/graphql/reference/index.md#metricsdashboard) - -#### Dashboard YAML properties - -Dashboards have several components: - -- Templating variables. -- Panel groups, which consist of panels. -- Panels, which support one or more metrics. - -The following tables outline the details of expected properties. - -##### **Dashboard (top-level) properties** - -| Property | Type | Required | Description | -| ------ | ------ | ------ | ------ | -| `dashboard` | string | yes | Heading for the dashboard. Only one dashboard should be defined per file. | -| `panel_groups` | array | yes | The panel groups which should be on the dashboard. | -| `templating` | hash | no | Top level key under which templating related options can be added. | -| `links` | array | no | Add links to display on the dashboard. | - -##### **Templating (`templating`) properties** - -| Property | Type | Required | Description | -| -------- | ---- | -------- | ----------- | -| `variables` | hash | yes | Variables can be defined here. | - -Read the documentation on [templating](#templating-variables-for-metrics-dashboards). - -##### **Links (`links`) properties** - -| Property | Type | Required | Description | -| -------- | ---- | -------- | ----------- | -| `url` | string | yes | The address of the link. | -| `title` | string | no | Display title for the link. | -| `type` | string | no | Type of the link. Specifies the link type, can be: `grafana` | - -Read the documentation on [links](#add-related-links-to-custom-dashboards). - -##### **Panel group (`panel_groups`) properties** - -| Property | Type | Required | Description | -| ------ | ------ | ------ | ------ | -| `group` | string | required | Heading for the panel group. | -| `priority` | number | optional, defaults to order in file | Order to appear on the dashboard. Higher number means higher priority, which will be higher on the page. Numbers do not need to be consecutive. | -| `panels` | array | required | The panels which should be in the panel group. | - -##### **Panel (`panels`) properties** - -| Property | Type | Required | Description | -| ------ | ------ | ------ | ------- | -| `type` | enum | no, defaults to `area-chart` | Specifies the chart type to use, can be: `area-chart`, `line-chart` or `anomaly-chart`. | -| `title` | string | yes | Heading for the panel. | -| `y_label` | string | no, but highly encouraged | Y-Axis label for the panel. | -| `y_axis` | string | no | Y-Axis configuration for the panel. | -| `max_value` | number | no | Denominator value used for calculating [percentile based results](#percentile-based-results) | -| `weight` | number | no, defaults to order in file | Order to appear within the grouping. Lower number means higher priority, which will be higher on the page. Numbers do not need to be consecutive. | -| `metrics` | array | yes | The metrics which should be displayed in the panel. Any number of metrics can be displayed when `type` is `area-chart` or `line-chart`, whereas only 3 can be displayed when `type` is `anomaly-chart`. | -| `links` | array | no | Add links to display on the chart's [context menu](#chart-context-menu). | - -##### **Axis (`panels[].y_axis`) properties** - -| Property | Type | Required | Description | -| ----------- | ------ | ----------------------------- | -------------------------------------------------------------------- | -| `name` | string | no, but highly encouraged | Y-Axis label for the panel. Replaces `y_label` if set. | -| `format` | string | no, defaults to `engineering` | Unit format used. See the [full list of units](prometheus_units.md). | -| `precision` | number | no, defaults to `2` | Number of decimal places to display in the number. | | - -##### **Metrics (`metrics`) properties** - -| Property | Type | Required | Description | -| ------ | ------ | ------ | ------ | -| `id` | string | no | Used for associating dashboard metrics with database records. Must be unique across dashboard configuration files. Required for [alerting](#setting-up-alerts-for-prometheus-metrics) (support not yet enabled, see [relevant issue](https://gitlab.com/gitlab-org/gitlab/-/issues/27980)). | -| `unit` | string | yes | Defines the unit of the query's return data. | -| `label` | string | no, but highly encouraged | Defines the legend-label for the query. Should be unique within the panel's metrics. Can contain time series labels as interpolated variables. | -| `query` | string | yes if `query_range` is not defined | Defines the Prometheus query to be used to populate the chart/panel. If defined, the `query` endpoint of the [Prometheus API](https://prometheus.io/docs/prometheus/latest/querying/api/) will be utilized. | -| `query_range` | string | yes if `query` is not defined | Defines the Prometheus query to be used to populate the chart/panel. If defined, the `query_range` endpoint of the [Prometheus API](https://prometheus.io/docs/prometheus/latest/querying/api/) will be utilized. | -| `step` | number | no, value is calculated if not defined | Defines query resolution step width in float number of seconds. Metrics on the same panel should use the same `step` value. | - -##### Dynamic labels - -Dynamic labels are useful when multiple time series are returned from a Prometheus query. - -When a static label is used and a query returns multiple time series, then all the legend items will be labeled the same, which makes identifying each time series difficult: - -```yaml -metrics: - - id: my_metric_id - query_range: 'http_requests_total' - label: "Time Series" - unit: "count" -``` - -This may render a legend like this: - - - -For labels to be more explicit, using variables that reflect time series labels is a good practice. The variables will be replaced by the values of the time series labels when the legend is rendered: - -```yaml -metrics: - - id: my_metric_id - query_range: 'http_requests_total' - label: "Instance: {{instance}}, method: {{method}}" - unit: "count" -``` - -The resulting rendered legend will look like this: - - - -There is also a shorthand value for dynamic dashboard labels that make use of only one time series label: - -```yaml -metrics: - - id: my_metric_id - query_range: 'http_requests_total' - label: "Method" - unit: "count" -``` - -This works by lowercasing the value of `label` and, if there are more words separated by spaces, replacing those spaces with an underscore (`_`). The transformed value is then checked against the labels of the time series returned by the Prometheus query. If a time series label is found that is equal to the transformed value, then the label value will be used and rendered in the legend like this: - - - -#### Panel types for dashboards - -The below panel types are supported in monitoring dashboards. - -##### Area or Line Chart - -To add an area chart panel type to a dashboard, look at the following sample dashboard file: - -```yaml -dashboard: 'Dashboard Title' -panel_groups: - - group: 'Group Title' - panels: - - type: area-chart # or line-chart - title: 'Area Chart Title' - y_label: "Y-Axis" - y_axis: - format: number - precision: 0 - metrics: - - id: area_http_requests_total - query_range: 'http_requests_total' - label: "Instance: {{instance}}, Method: {{method}}" - unit: "count" -``` - -Note the following properties: - -| Property | Type | Required | Description | -| ------ | ------ | ------ | ------ | -| type | string | no | Type of panel to be rendered. Optional for area panel types | -| query_range | string | required | For area panel types, you must use a [range query](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries) | - - - -Starting in [version 12.8](https://gitlab.com/gitlab-org/gitlab/-/issues/202696), the y-axis values will automatically scale according to the data. Previously, it always started from 0. - -##### Anomaly chart - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/16530) in GitLab 12.5. - -To add an anomaly chart panel type to a dashboard, add a panel with *exactly* 3 metrics. - -The first metric represents the current state, and the second and third metrics represent the upper and lower limit respectively: - -```yaml -dashboard: 'Dashboard Title' -panel_groups: - - group: 'Group Title' - panels: - - type: anomaly-chart - title: "Chart Title" - y_label: "Y-Axis" - metrics: - - id: anomaly_requests_normal - query_range: 'http_requests_total' - label: "# of Requests" - unit: "count" - metrics: - - id: anomaly_requests_upper_limit - query_range: 10000 - label: "Max # of requests" - unit: "count" - metrics: - - id: anomaly_requests_lower_limit - query_range: 2000 - label: "Min # of requests" - unit: "count" -``` - -Note the following properties: - -| Property | Type | Required | Description | -| ------ | ------ | ------ | ------ | -| type | string | required | Must be `anomaly-chart` for anomaly panel types | -| query_range | yes | required | For anomaly panel types, you must use a [range query](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries) in every metric. | - - - -##### Bar chart - -To add a bar chart to a dashboard, look at the following sample dashboard file: - -```yaml -dashboard: 'Dashboard Title' -panel_groups: - - group: 'Group title' - panels: - - type: bar - title: "Http Handlers" - x_label: 'Response Size' - y_axis: - name: "Handlers" - metrics: - - id: prometheus_http_response_size_bytes_bucket - query_range: "sum(increase(prometheus_http_response_size_bytes_bucket[1d])) by (handler)" - unit: 'Bytes' -``` - -Note the following properties: - -| Property | Type | Required | Description | -| ------ | ------ | ------ | ------ | -| `type` | string | yes | Type of panel to be rendered. For bar chart types, set to `bar` | -| `query_range` | yes | yes | For bar chart, you must use a [range query](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries) - - - -##### Column chart - -To add a column panel type to a dashboard, look at the following sample dashboard file: - -```yaml -dashboard: 'Dashboard Title' -panel_groups: - - group: 'Group title' - panels: - - title: "Column" - type: "column" - metrics: - - id: 1024_memory - query: 'avg(sum(container_memory_usage_bytes{container_name!="POD",pod_name=~"^%{ci_environment_slug}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="%{kube_namespace}"}) by (job)) without (job) / count(avg(container_memory_usage_bytes{container_name!="POD",pod_name=~"^%{ci_environment_slug}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="%{kube_namespace}"}) without (job)) /1024/1024' - unit: MB - label: "Memory Usage" -``` - -Note the following properties: - -| Property | Type | Required | Description | -| ------ | ------ | ------ | ------ | -| type | string | yes | Type of panel to be rendered. For column panel types, set to `column` | -| query_range | yes | yes | For column panel types, you must use a [range query](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries) | - - - -##### Stacked column - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/30583) in GitLab 12.8. - -To add a stacked column panel type to a dashboard, look at the following sample dashboard file: - -```yaml -dashboard: 'Dashboard title' -priority: 1 -panel_groups: -- group: 'Group Title' - priority: 5 - panels: - - type: 'stacked-column' - title: "Stacked column" - y_label: "y label" - x_label: 'x label' - metrics: - - id: memory_1 - query_range: 'memory_query' - label: "memory query 1" - unit: "count" - series_name: 'group 1' - - id: memory_2 - query_range: 'memory_query_2' - label: "memory query 2" - unit: "count" - series_name: 'group 2' - -``` - - - -| Property | Type | Required | Description | -| ------ | ------ | ------ | ------ | -| `type` | string | yes | Type of panel to be rendered. For stacked column panel types, set to `stacked-column` | -| `query_range` | yes | yes | For stacked column panel types, you must use a [range query](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries) | - -##### Single Stat - -To add a single stat panel type to a dashboard, look at the following sample dashboard file: - -```yaml -dashboard: 'Dashboard Title' -panel_groups: - - group: 'Group Title' - panels: - - title: "Single Stat" - type: "single-stat" - metrics: - - id: 10 - query: 'max(go_memstats_alloc_bytes{job="prometheus"})' - unit: MB - label: "Total" -``` - -Note the following properties: - -| Property | Type | Required | Description | -| ------ | ------ | ------ | ------ | -| type | string | yes | Type of panel to be rendered. For single stat panel types, set to `single-stat` | -| query | string | yes | For single stat panel types, you must use an [instant query](https://prometheus.io/docs/prometheus/latest/querying/api/#instant-queries) | - - - -###### Percentile based results - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/201946) in GitLab 12.8. - -Query results sometimes need to be represented as a percentage value out of 100. You can use the `max_value` property at the root of the panel definition: - -```yaml -dashboard: 'Dashboard Title' -panel_groups: - - group: 'Group Title' - panels: - - title: "Single Stat" - type: "single-stat" - max_value: 100 - metrics: - - id: 10 - query: 'max(go_memstats_alloc_bytes{job="prometheus"})' - unit: '%' - label: "Total" -``` - -For example, if you have a query value of `53.6`, adding `%` as the unit results in a single stat value of `53.6%`, but if the maximum expected value of the query is `120`, the value would be `44.6%`. Adding the `max_value` causes the correct percentage value to display. - -##### Heatmaps - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/30581) in GitLab 12.5. - -To add a heatmap panel type to a dashboard, look at the following sample dashboard file: - -```yaml -dashboard: 'Dashboard Title' -panel_groups: - - group: 'Group Title' - panels: - - title: "Heatmap" - type: "heatmap" - metrics: - - id: 10 - query: 'sum(rate(nginx_upstream_responses_total{upstream=~"%{kube_namespace}-%{ci_environment_slug}-.*"}[60m])) by (status_code)' - unit: req/sec - label: "Status code" -``` - -Note the following properties: - -| Property | Type | Required | Description | -| ------ | ------ | ------ | ------ | -| type | string | yes | Type of panel to be rendered. For heatmap panel types, set to `heatmap` | -| query_range | yes | yes | For area panel types, you must use a [range query](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries) | - - - -### Templating variables for metrics dashboards - -Templating variables can be used to make your metrics dashboard more versatile. - -#### Templating variable types - -`templating` is a top-level key in the -[dashboard YAML](#dashboard-top-level-properties). -Define your variables in the `variables` key, under `templating`. The value of -the `variables` key should be a hash, and each key under `variables` -defines a templating variable on the dashboard, and may contain alphanumeric and underscore characters. - -A variable can be used in a Prometheus query in the same dashboard using the syntax -described [here](#using-variables). - -##### `text` variable type - -CAUTION: **Warning:** -This variable type is an _alpha_ feature, and is subject to change at any time -without prior notice! - -For each `text` variable defined in the dashboard YAML, there will be a free text -box on the dashboard UI, allowing you to enter a value for each variable. - -The `text` variable type supports a simple and a full syntax. - -###### Simple syntax - -This example creates a variable called `variable1`, with a default value -of `default value`: - -```yaml -templating: - variables: - variable1: 'default value' # `text` type variable with `default value` as its default. -``` - -###### Full syntax - -This example creates a variable called `variable1`, with a default value of `default`. -The label for the text box on the UI will be the value of the `label` key: - -```yaml -templating: - variables: - variable1: # The variable name that can be used in queries. - label: 'Variable 1' # (Optional) label that will appear in the UI for this text box. - type: text - options: - default_value: 'default' # (Optional) default value. -``` - -##### `custom` variable type - -CAUTION: **Warning:** -This variable type is an _alpha_ feature, and is subject to change at any time -without prior notice! - -Each `custom` variable defined in the dashboard YAML creates a dropdown -selector on the dashboard UI, allowing you to select a value for each variable. - -The `custom` variable type supports a simple and a full syntax. - -###### Simple syntax - -This example creates a variable called `variable1`, with a default value of `value1`. -The dashboard UI will display a dropdown with `value1`, `value2` and `value3` -as the choices. - -```yaml -templating: - variables: - variable1: ['value1', 'value2', 'value3'] -``` - -###### Full syntax - -This example creates a variable called `variable1`, with a default value of `value_option_2`. -The label for the text box on the UI will be the value of the `label` key. -The dashboard UI will display a dropdown with `Option 1` and `Option 2` -as the choices. - -If you select `Option 1` from the dropdown, the variable will be replaced with `value option 1`. -Similarly, if you select `Option 2`, the variable will be replaced with `value_option_2`: - -```yaml -templating: - variables: - variable1: # The variable name that can be used in queries. - label: 'Variable 1' # (Optional) label that will appear in the UI for this dropdown. - type: custom - options: - values: - - value: 'value option 1' # The value that will replace the variable in queries. - text: 'Option 1' # (Optional) Text that will appear in the UI dropdown. - - value: 'value_option_2' - text: 'Option 2' - default: true # (Optional) This option should be the default value of this variable. -``` - -### Add related links to custom dashboards - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/216385) in GitLab 13.1. - -You can embed links to other dashboards or external services in your custom -dashboard by adding **Related links** to your dashboard's YAML file. Related links -open in the same tab as the dashboard. Related links can be displayed in the -following locations on your dashboard: - -- At the top of your dashboard as the top level [`links` dashboard property](#dashboard-top-level-properties). -- In charts context menus as the [`links` property of a panel](#panel-panels-properties). - -Related links can contain the following attributes: - -- `url`: The full URL to the link. Required. -- `title`: A phrase describing the link. Optional. If this attribute is not set, - the full URL is used for the link title. -- `type`: A string declaring the type of link. Optional. If set to `grafana`, the - dashboard's time range values are converted to Grafana's time range format and - appended to the `url`. - -The dashboard's time range is appended to the `url` as URL parameters. - -The following example shows two related links (`GitLab.com` and `GitLab Documentation`) -added to a dashboard: - - - -#### Links Syntax - -```yaml -links: - - title: GitLab.com - url: https://gitlab.com - - title: GitLab Documentation - url: https://docs.gitlab.com - - title: Public Grafana playground dashboard - url: https://play.grafana.org/d/000000012/grafana-play-home?orgId=1 - type: grafana -``` - -### View and edit the source file of a custom dashboard - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/34779) in GitLab 12.5. - -When viewing a custom dashboard of a project, you can view the original -`.yml` file by clicking on the **Edit dashboard** button. - -### Chart Context Menu - -From each of the panels in the dashboard, you can access the context menu by clicking the **{ellipsis_v}** **More actions** dropdown box above the upper right corner of the panel to take actions related to the chart's data. - - - -The options are: - -- [Expand panel](#expand-panel) -- [View logs](#view-logs-ultimate) -- [Download CSV](#downloading-data-as-csv) -- [Copy link to chart](#embedding-gitlab-managed-kubernetes-metrics) -- [Alerts](#setting-up-alerts-for-prometheus-metrics) - -### Dashboard Annotations - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/211330) in GitLab 12.10 (enabled by feature flag `metrics_dashboard_annotations`). -> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/215224) in GitLab 13.0. - -You can use **Metrics Dashboard Annotations** to mark any important events on -every metrics dashboard by adding annotations to it. While viewing a dashboard, -annotation entries assigned to the selected time range will be automatically -fetched and displayed on every chart within that dashboard. On mouse hover, each -annotation presents additional details, including the exact time of an event and -its description. - -You can create annotations by making requests to the -[Metrics dashboard annotations API](../../../api/metrics_dashboard_annotations.md) - - - -#### Retention policy - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/211433) in GitLab 13.01. - -To avoid excessive storage space consumption by stale annotations, records attached -to time periods older than two weeks are removed daily. This recurring background -job runs at 1:00 a.m. local server time. - -### Expand panel - -> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3100) in GitLab 13.0. - -To view a larger version of a visualization, expand the panel by clicking the -**{ellipsis_v}** **More actions** icon and selecting **Expand panel**. - -To return to the metrics dashboard, click the **Back** button in your -browser, or pressing the <kbd>Esc</kbd> key. - -### View Logs **(ULTIMATE)** - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/122013) in GitLab 12.8. - -If you have [Logs](../clusters/kubernetes_pod_logs.md) enabled, -you can navigate from the charts in the dashboard to view Logs by -clicking on the context menu in the upper-right corner. - -If you use the **Timeline zoom** function at the bottom of the chart, logs will narrow down to the time range you selected. - -### Timeline zoom and URL sharing - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/198910) in GitLab 12.8. - -You can use the **Timeline zoom** function at the bottom of a chart to zoom in -on a date and time of your choice. When you click and drag the sliders to select -a different beginning or end date of data to display, GitLab adds your selected start -and end times to the URL, enabling you to share specific timeframes more easily. - -### Downloading data as CSV - -Data from Prometheus charts on the metrics dashboard can be downloaded as CSV. - -### Setting up alerts for Prometheus metrics - -#### Managed Prometheus instances - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/6590) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.2 for [custom metrics](#adding-custom-metrics), and 11.3 for [library metrics](prometheus_library/metrics.md). - -For managed Prometheus instances using auto configuration, alerts for metrics [can be configured](#adding-custom-metrics) directly in the performance dashboard. - -To set an alert: - -1. Click on the ellipsis icon in the top right corner of the metric you want to create the alert for. -1. Choose **Alerts** -1. Set threshold and operator. -1. Click **Add** to save and activate the alert. - - - -To remove the alert, click back on the alert icon for the desired metric, and click **Delete**. - -#### External Prometheus instances - ->- [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9258) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.8. ->- [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/42640) to [GitLab Core](https://about.gitlab.com/pricing/) in 12.10. - -For manually configured Prometheus servers, a notify endpoint is provided to use with Prometheus webhooks. If you have manual configuration enabled, an **Alerts** section is added to **Settings > Integrations > Prometheus**. This contains the *URL* and *Authorization Key*. The **Reset Key** button will invalidate the key and generate a new one. - - - -To send GitLab alert notifications, copy the *URL* and *Authorization Key* into the [`webhook_configs`](https://prometheus.io/docs/alerting/configuration/#webhook_config) section of your Prometheus Alertmanager configuration: - -```yaml -receivers: - name: gitlab - webhook_configs: - - http_config: - bearer_token: 9e1cbfcd546896a9ea8be557caf13a76 - send_resolved: true - url: http://192.168.178.31:3001/root/manual_prometheus/prometheus/alerts/notify.json - ... -``` - -In order for GitLab to associate your alerts with an [environment](../../../ci/environments/index.md), you need to configure a `gitlab_environment_name` label on the alerts you set up in Prometheus. The value of this should match the name of your Environment in GitLab. - -NOTE: **Note** -In GitLab versions 13.1 and greater, you can configure your manually configured Prometheus server to use the [Generic alerts integration](generic_alerts.md). - -### Taking action on incidents **(ULTIMATE)** - ->- [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/4925) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.11. ->- [From GitLab Ultimate 12.5](https://gitlab.com/gitlab-org/gitlab/-/issues/13401), when GitLab receives a recovery alert, it will automatically close the associated issue. - -Alerts can be used to trigger actions, like opening an issue automatically (disabled by default since `13.1`). To configure the actions: - -1. Navigate to your project's **Settings > Operations > Incidents**. -1. Enable the option to create issues. -1. Choose the [issue template](../description_templates.md) to create the issue from. -1. Optionally, select whether to send an email notification to the developers of the project. -1. Click **Save changes**. - -Once enabled, an issue will be opened automatically when an alert is triggered which contains values extracted from [alert's payload](https://prometheus.io/docs/alerting/configuration/#webhook_config -): - -- Issue author: `GitLab Alert Bot` -- Issue title: Extract from `annotations/title`, `annotations/summary` or `labels/alertname` -- Alert `Summary`: A list of properties - - `starts_at`: Alert start time via `startsAt` - - `full_query`: Alert query extracted from `generatorURL` - - Optional list of attached annotations extracted from `annotations/*` -- Alert [GFM](../../markdown.md): GitLab Flavored Markdown from `annotations/gitlab_incident_markdown` - -When GitLab receives a **Recovery Alert**, it will automatically close the associated issue. This action will be recorded as a system message on the issue indicating that it was closed automatically by the GitLab Alert bot. - -To further customize the issue, you can add labels, mentions, or any other supported [quick action](../quick_actions.md) in the selected issue template, which will apply to all incidents. To limit quick actions or other information to only specific types of alerts, use the `annotations/gitlab_incident_markdown` field. - -Since [version 12.2](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/63373), GitLab will tag each incident issue with the `incident` label automatically. If the label does not yet exist, it will be created automatically as well. - -If the metric exceeds the threshold of the alert for over 5 minutes, an email will be sent to all [Maintainers and Owners](../../permissions.md#project-members-permissions) of the project. - ## Determining the performance impact of a merge > - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/10408) in GitLab 9.2. @@ -1069,174 +154,3 @@ Performance data will be available for the duration it is persisted on the Prometheus server.  - -## Embedding metric charts within GitLab Flavored Markdown - -### Embedding GitLab-managed Kubernetes metrics - -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/29691) in GitLab 12.2. - -It is possible to display metrics charts within [GitLab Flavored Markdown](../../markdown.md#gitlab-flavored-markdown-gfm) fields such as issue or merge request descriptions. The maximum number of embedded charts allowed in a GitLab Flavored Markdown field is 100. - -This can be useful if you are sharing an application incident or performance -metrics to others and want to have relevant information directly available. - -NOTE: **Note:** -Requires [Kubernetes](prometheus_library/kubernetes.md) metrics. - -To display metric charts, include a link of the form `https://<root_url>/<project>/-/environments/<environment_id>/metrics`: - - - -GitLab unfurls the link as an embedded metrics panel: - - - -You can also embed a single chart. To get a link to a chart, click the -**{ellipsis_v}** **More actions** menu in the upper right corner of the chart, -and select **Copy link to chart**, as shown in this example: - - - -The following requirements must be met for the metric to unfurl: - -- The `<environment_id>` must correspond to a real environment. -- Prometheus must be monitoring the environment. -- The GitLab instance must be configured to receive data from the environment. -- The user must be allowed access to the monitoring dashboard for the environment ([Reporter or higher](../../permissions.md)). -- The dashboard must have data within the last 8 hours. - - If all of the above are true, then the metric will unfurl as seen below: - - - -Metric charts may also be hidden: - - - -You can open the link directly into your browser for a [detailed view of the data](#expand-panel). - -### Embedding metrics in issue templates - -It is also possible to embed either the default dashboard metrics or individual metrics in issue templates. For charts to render side-by-side, links to the entire metrics dashboard or individual metrics should be separated by either a comma or a space. - - - -### Embedding metrics based on alerts in incident issues - -For [GitLab-managed alerting rules](#setting-up-alerts-for-prometheus-metrics), the issue will include an embedded chart for the query corresponding to the alert. The chart displays an hour of data surrounding the starting point of the incident, 30 minutes before and after. - -For [manually configured Prometheus instances](#manual-configuration-of-prometheus), a chart corresponding to the query can be included if these requirements are met: - -- The alert corresponds to an environment managed through GitLab. -- The alert corresponds to a [range query](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries). -- The alert contains the required attributes listed in the chart below. - -| Attributes | Required | Description | -| ---------- | -------- | ----------- | -| `annotations/gitlab_environment_name` | Yes | Name of the GitLab-managed environment corresponding to the alert | -| One of `annotations/title`, `annotations/summary`, `labels/alertname` | Yes | Will be used as the chart title | -| `annotations/gitlab_y_label` | No | Will be used as the chart's y-axis label | - -### Embedding Cluster Health Charts **(ULTIMATE)** - -> [Introduced](<https://gitlab.com/gitlab-org/gitlab/-/issues/40997>) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.9. - -[Cluster Health Metrics](../clusters/index.md#monitoring-your-kubernetes-cluster-ultimate) can also be embedded in [GitLab-flavored Markdown](../../markdown.md). - -To embed a metric chart, include a link to that chart in the form `https://<root_url>/<project>/-/cluster/<cluster_id>?<query_params>` anywhere that GitLab-flavored Markdown is supported. To generate and copy a link to the chart, follow the instructions in the [Cluster Health Metric documentation](../clusters/index.md#monitoring-your-kubernetes-cluster-ultimate). - -The following requirements must be met for the metric to unfurl: - -- The `<cluster_id>` must correspond to a real cluster. -- Prometheus must be monitoring the cluster. -- The user must be allowed access to the project cluster metrics. -- The dashboards must be reporting data on the [Cluster Health Page](../clusters/index.md#monitoring-your-kubernetes-cluster-ultimate) - - If the above requirements are met, then the metric will unfurl as seen below. - - - -### Embedding Grafana charts - -Grafana metrics can be embedded in [GitLab Flavored Markdown](../../markdown.md). - -#### Embedding charts via Grafana Rendered Images - -It is possible to embed live [Grafana](https://docs.gitlab.com/omnibus/settings/grafana.html) charts in issues, as a [direct linked rendered image](https://grafana.com/docs/grafana/latest/reference/share_panel/#direct-link-rendered-image). - -The sharing dialog within Grafana provides the link, as highlighted below. - - - -NOTE: **Note:** -For this embed to display correctly, the Grafana instance must be available to the target user, either as a public dashboard, or on the same network. - -Copy the link and add an image tag as [inline HTML](../../markdown.md#inline-html) in your Markdown. You may tweak the query parameters as required. For instance, removing the `&from=` and `&to=` parameters will give you a live chart. Here is example markup for a live chart from GitLab's public dashboard: - -```html -<img src="https://dashboards.gitlab.com/d/RZmbBr7mk/gitlab-triage?orgId=1&refresh=30s&var-env=gprd&var-environment=gprd&var-prometheus=prometheus-01-inf-gprd&var-prometheus_app=prometheus-app-01-inf-gprd&var-backend=All&var-type=All&var-stage=main&from=1580444107655&to=1580465707655"/> -``` - -This will render like so: - - - -#### Embedding charts via integration with Grafana HTTP API - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/31376) in GitLab 12.5. - -Each project can support integration with one Grafana instance. This configuration allows a user to copy a link to a panel in Grafana, then paste it into a GitLab Markdown field. The chart will be rendered in the GitLab chart format. - -Prerequisites for embedding from a Grafana instance: - -1. The datasource must be a Prometheus instance. -1. The datasource must be proxyable, so the HTTP Access setting should be set to `Server`. - - - -##### Setting up the Grafana integration - -1. [Generate an Admin-level API Token in Grafana.](https://grafana.com/docs/grafana/latest/http_api/auth/#create-api-token) -1. In your GitLab project, navigate to **Settings > Operations > Grafana Authentication**. -1. To enable the integration, check the "Active" checkbox. -1. For "Grafana URL", enter the base URL of the Grafana instance. -1. For "API Token", enter the Admin API Token you just generated. -1. Click **Save Changes**. - -##### Generating a link to a chart - -1. In Grafana, navigate to the dashboard you wish to embed a panel from. -  -1. In the upper-left corner of the page, select a specific value for each variable required for the queries in the chart. -  -1. In Grafana, click on a panel's title, then click **Share** to open the panel's sharing dialog to the **Link** tab. If you click the _dashboard's_ share panel instead, GitLab will attempt to embed the first supported panel on the dashboard (if available). -1. If your Prometheus queries use Grafana's custom template variables, ensure that the "Template variables" option is toggled to **On**. Of Grafana global template variables, only `$__interval`, `$__from`, and `$__to` are currently supported. Toggle **On** the "Current time range" option to specify the time range of the chart. Otherwise, the default range will be the last 8 hours. -  -1. Click **Copy** to copy the URL to the clipboard. -1. In GitLab, paste the URL into a Markdown field and save. The chart will take a few moments to render. -  - -## Metrics dashboard visibility - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/201924) in GitLab 13.0. - -You can set the visibility of the metrics dashboard to **Only Project Members** -or **Everyone With Access**. When set to **Everyone with Access**, the metrics -dashboard is visible to authenticated and non-authenticated users. - -## Troubleshooting - -When troubleshooting issues with a managed Prometheus app, it is often useful to -[view the Prometheus UI](../../../development/prometheus.md#access-the-ui-of-a-prometheus-managed-application-in-kubernetes). - -### "No data found" error on Metrics dashboard page - -If the "No data found" screen continues to appear, it could be due to: - -- No successful deployments have occurred to this environment. -- Prometheus does not have performance data for this environment, or the metrics - are not labeled correctly. To test this, connect to the Prometheus server and - [run a query](prometheus_library/kubernetes.md#metrics-supported), replacing `$CI_ENVIRONMENT_SLUG` - with the name of your environment. -- You may need to re-add the GitLab predefined common metrics. This can be done by running the [import common metrics Rake task](../../../administration/raketasks/maintenance.md#import-common-metrics). diff --git a/doc/user/project/integrations/prometheus_library/nginx_ingress.md b/doc/user/project/integrations/prometheus_library/nginx_ingress.md index b2bc217e8bf..7bebe7b1e65 100644 --- a/doc/user/project/integrations/prometheus_library/nginx_ingress.md +++ b/doc/user/project/integrations/prometheus_library/nginx_ingress.md @@ -8,7 +8,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/22133) in GitLab 11.7. -NOTE: **Note:** NGINX Ingress versions prior to 0.16.0 offer an included [VTS Prometheus metrics exporter](nginx_ingress_vts.md), which exports metrics different than the built-in metrics. +NOTE: **Note:** +NGINX Ingress versions prior to 0.16.0 offer an included [VTS Prometheus metrics exporter](nginx_ingress_vts.md), which exports metrics different than the built-in metrics. GitLab has support for automatically detecting and monitoring the Kubernetes NGINX Ingress controller. This is provided by leveraging the built-in Prometheus metrics included with Kubernetes NGINX Ingress controller [version 0.16.0](https://github.com/kubernetes/ingress-nginx/blob/master/Changelog.md#0160) onward. diff --git a/doc/user/project/integrations/prometheus_library/nginx_ingress_vts.md b/doc/user/project/integrations/prometheus_library/nginx_ingress_vts.md index 6ba0a7610f6..326931e9790 100644 --- a/doc/user/project/integrations/prometheus_library/nginx_ingress_vts.md +++ b/doc/user/project/integrations/prometheus_library/nginx_ingress_vts.md @@ -8,7 +8,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/13438) in GitLab 9.5. -NOTE: **Note:** [NGINX Ingress version 0.16](nginx_ingress.md) and above have built-in Prometheus metrics, which are different than the VTS based metrics. +NOTE: **Note:** +[NGINX Ingress version 0.16](nginx_ingress.md) and above have built-in Prometheus metrics, which are different than the VTS based metrics. GitLab has support for automatically detecting and monitoring the Kubernetes NGINX Ingress controller. This is provided by leveraging the included VTS Prometheus metrics exporter in [version 0.9.0](https://github.com/kubernetes/ingress-nginx/blob/master/Changelog.md#09-beta1) through [0.15.x](https://github.com/kubernetes/ingress-nginx/blob/master/Changelog.md#0150). diff --git a/doc/user/project/integrations/prometheus_units.md b/doc/user/project/integrations/prometheus_units.md index 7b216ced1cc..ee4f3ed77d4 100644 --- a/doc/user/project/integrations/prometheus_units.md +++ b/doc/user/project/integrations/prometheus_units.md @@ -1,175 +1,5 @@ --- -stage: Monitor -group: APM -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +redirect_to: '../../../operations/metrics/dashboards/yaml_number_format.md' --- -# Unit formats reference - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/201999) in GitLab 12.9. - -You can select units to format your charts by adding `format` to your -[axis configuration](prometheus.md#dashboard-yaml-properties). - -## Internationalization and localization - -Currently, your [internationalization and localization options](https://en.wikipedia.org/wiki/Internationalization_and_localization) for number formatting are dependent on the system you are using i.e. your OS or browser. - -## Engineering Notation - -For generic or default data, numbers are formatted according to the current locale in [engineering notation](https://en.wikipedia.org/wiki/Engineering_notation). - -While an [engineering notation exists for the web](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Intl/NumberFormat), GitLab uses a version based off the [scientific notation](https://en.wikipedia.org/wiki/Scientific_notation). GitLab formatting acts in accordance with SI prefixes. For example, using GitLab notation, `1500.00` becomes `1.5k` instead of `1.5E3`. Keep this distinction in mind when using the engineering notation for your metrics. - -Formats: `engineering` - -SI prefixes: - -| Name | Symbol | Value | -| ---------- | ------- | -------------------------- | -| `yotta` | Y | 1000000000000000000000000 | -| `zetta` | Z | 1000000000000000000000 | -| `exa` | E | 1000000000000000000 | -| `peta` | P | 1000000000000000 | -| `tera` | T | 1000000000000 | -| `giga` | G | 1000000000 | -| `mega` | M | 1000000 | -| `kilo` | k | 1000 | -| `milli` | m | 0.001 | -| `micro` | μ | 0.000001 | -| `nano` | n | 0.000000001 | -| `pico` | p | 0.000000000001 | -| `femto` | f | 0.000000000000001 | -| `atto` | a | 0.000000000000000001 | -| `zepto` | z | 0.000000000000000000001 | -| `yocto` | y | 0.000000000000000000000001 | - -**Examples:** - -| Data | Displayed | -| --------------------------------- | --------- | -| `0.000000000000000000000008` | 8y | -| `0.000000000000000000008` | 8z | -| `0.000000000000000008` | 8a | -| `0.000000000000008` | 8f | -| `0.000000000008` | 8p | -| `0.000000008` | 8n | -| `0.000008` | 8μ | -| `0.008` | 8m | -| `10` | 10 | -| `1080` | 1.08k | -| `18000` | 18k | -| `18888` | 18.9k | -| `188888` | 189k | -| `18888888` | 18.9M | -| `1888888888` | 1.89G | -| `1888888888888` | 1.89T | -| `1888888888888888` | 1.89P | -| `1888888888888888888` | 1.89E | -| `1888888888888888888888` | 1.89Z | -| `1888888888888888888888888` | 1.89Y | -| `1888888888888888888888888888` | 1.89e+27 | - -## Numbers - -For number data, numbers are formatted according to the current locale. - -Formats: `number` - -**Examples:** - -| Data | Displayed | -| ---------- | --------- | -| `10` | 1 | -| `1000` | 1,000 | -| `1000000` | 1,000,000 | - -## Percentage - -For percentage data, format numbers in the chart with a `%` symbol. - -Formats supported: `percent`, `percentHundred` - -**Examples:** - -| Format | Data | Displayed | -| ---------------- | ----- | --------- | -| `percent` | `0.5` | 50% | -| `percent` | `1` | 100% | -| `percent` | `2` | 200% | -| `percentHundred` | `50` | 50% | -| `percentHundred` | `100` | 100% | -| `percentHundred` | `200` | 200% | - -## Duration - -For time durations, format numbers in the chart with a time unit symbol. - -Formats supported: `milliseconds`, `seconds` - -**Examples:** - -| Format | Data | Displayed | -| -------------- | ------ | --------- | -| `milliseconds` | `10` | 10ms | -| `milliseconds` | `500` | 100ms | -| `milliseconds` | `1000` | 1000ms | -| `seconds` | `10` | 10s | -| `seconds` | `500` | 500s | -| `seconds` | `1000` | 1000s | - -## Digital (Metric) - -Converts a number of bytes using metric prefixes. It scales to -use the unit that's the best fit. - -Formats supported: - -- `decimalBytes` -- `kilobytes` -- `megabytes` -- `gigabytes` -- `terabytes` -- `petabytes` - -**Examples:** - -| Format | Data | Displayed | -| -------------- | --------- | --------- | -| `decimalBytes` | `1` | 1B | -| `decimalBytes` | `1000` | 1kB | -| `decimalBytes` | `1000000` | 1MB | -| `kilobytes` | `1` | 1kB | -| `kilobytes` | `1000` | 1MB | -| `kilobytes` | `1000000` | 1GB | -| `megabytes` | `1` | 1MB | -| `megabytes` | `1000` | 1GB | -| `megabytes` | `1000000` | 1TB | - -## Digital (IEC) - -Converts a number of bytes using binary prefixes. It scales to -use the unit that's the best fit. - -Formats supported: - -- `bytes` -- `kibibytes` -- `mebibytes` -- `gibibytes` -- `tebibytes` -- `pebibytes` - -**Examples:** - -| Format | Data | Displayed | -| ----------- | ------------- | --------- | -| `bytes` | `1` | 1B | -| `bytes` | `1024` | 1KiB | -| `bytes` | `1024 * 1024` | 1MiB | -| `kibibytes` | `1` | 1KiB | -| `kibibytes` | `1024` | 1MiB | -| `kibibytes` | `1024 * 1024` | 1GiB | -| `mebibytes` | `1` | 1MiB | -| `mebibytes` | `1024` | 1GiB | -| `mebibytes` | `1024 * 1024` | 1TiB | +This document was moved to [another location](../../../operations/metrics/dashboards/yaml_number_format.md). diff --git a/doc/user/project/integrations/redmine.md b/doc/user/project/integrations/redmine.md index 9e1cdf0490c..c92ddf38ad2 100644 --- a/doc/user/project/integrations/redmine.md +++ b/doc/user/project/integrations/redmine.md @@ -7,7 +7,6 @@ | Field | Description | | ----- | ----------- | - | `description` | A name for the issue tracker (to differentiate between instances, for example) | | `project_url` | The URL to the project in Redmine which is being linked to this GitLab project | | `issues_url` | The URL to the issue in Redmine project that is linked to this GitLab project. Note that the `issues_url` requires `:id` in the URL. This ID is used by GitLab as a placeholder to replace the issue number. | | `new_issue_url` | This is the URL to create a new issue in Redmine for the project linked to this GitLab project. **This is currently not being used and will be removed in a future release.** | diff --git a/doc/user/project/integrations/services_templates.md b/doc/user/project/integrations/services_templates.md index 8a88df88629..bc2bdde2f64 100644 --- a/doc/user/project/integrations/services_templates.md +++ b/doc/user/project/integrations/services_templates.md @@ -1,28 +1,25 @@ -# Services templates +# Service templates -A GitLab administrator can add a service template that sets a default for each -project. After a service template is enabled, it will be applied to **all** -projects that don't have it already enabled and its details will be pre-filled -on the project's Service page. By disabling the template, it will be disabled -for new projects only. +Using a service template, GitLab administrators can provide default values for configuring integrations at the project level. + +When you enable a service template, the defaults are applied to **all** projects that do not +already have the integration enabled or do not otherwise have custom values saved. +The values are pre-filled on each project's configuration page for the applicable integration. + +If you disable the template, these values no longer appear as defaults, while +any values already saved for an integration remain unchanged. ## Enable a service template Navigate to the **Admin Area > Service Templates** and choose the service template you wish to create. -## Services for external issue trackers +## Service for external issue trackers -In the image below you can see how a service template for Redmine would look -like. +The following image shows an example service template for Redmine.  ---- - For each project, you will still need to configure the issue tracking URLs by replacing `:issues_tracker_id` in the above screenshot with the ID used -by your external issue tracker. Prior to GitLab v7.8, this ID was configured in -the project settings, and GitLab would automatically update the URL configured -in `gitlab.yml`. This behavior is now deprecated and all issue tracker URLs -must be configured directly within the project's **Integrations** settings. +by your external issue tracker. diff --git a/doc/user/project/integrations/slack.md b/doc/user/project/integrations/slack.md index 79fc8eceddf..6c5dc787c5e 100644 --- a/doc/user/project/integrations/slack.md +++ b/doc/user/project/integrations/slack.md @@ -1,25 +1,45 @@ # Slack Notifications Service -The Slack Notifications Service allows your GitLab project to send events (e.g. issue created) to your existing Slack team as notifications. This requires configurations in both Slack and GitLab. +The Slack Notifications Service allows your GitLab project to send events +(such as issue creation) to your existing Slack team as notifications. Setting up +Slack notifications requires configuration changes for both Slack and GitLab. -> Note: You can also use Slack slash commands to control GitLab inside Slack. This is the separately configured [Slack slash commands](slack_slash_commands.md). +NOTE: **Note:** +You can also use Slack slash commands to control GitLab inside Slack. This is the +separately configured [Slack slash commands](slack_slash_commands.md). -## Slack Configuration +## Slack configuration 1. Sign in to your Slack team and [start a new Incoming WebHooks configuration](https://my.slack.com/services/new/incoming-webhook). -1. Select the Slack channel where notifications will be sent to by default. Click the **Add Incoming WebHooks integration** button to add the configuration. -1. Copy the **Webhook URL**, which we'll use later in the GitLab configuration. +1. Select the Slack channel where notifications will be sent to by default. + Click the **Add Incoming WebHooks integration** button to add the configuration. +1. Copy the **Webhook URL**, which we will use later in the GitLab configuration. -## GitLab Configuration +## GitLab configuration -1. Navigate to the [Integrations page](overview.md#accessing-integrations) in your project's settings, i.e. **Project > Settings > Integrations**. +1. Open your project's page, and navigate to your project's + [Integrations page](overview.md#accessing-integrations) at + **{settings}** **Settings > Integrations**. 1. Select the **Slack notifications** integration to configure it. -1. Ensure that the **Active** toggle is enabled. -1. Check the checkboxes corresponding to the GitLab events you want to send to Slack as a notification. -1. For each event, optionally enter the Slack channel names where you want to send the event, separated by a comma. If left empty, the event will be sent to the default channel that you configured in the Slack Configuration step. **Note:** Usernames and private channels are not supported. To send direct messages, use the Member ID found under user's Slack profile. -1. Paste the **Webhook URL** that you copied from the Slack Configuration step. -1. Optionally customize the Slack bot username that will be sending the notifications. -1. Configure the remaining options and click `Save changes`. +1. Click **Enable integration**. +1. In **Trigger**, select the checkboxes for each type of GitLab event to send to Slack as a + notification. See [Triggers available for Slack notifications](#triggers-available-for-slack-notifications) + for a full list. By default, messages are sent to the channel you configured during + [Slack integration](#slack-configuration). +1. (Optional) To send messages to a different channel, multiple channels, or as a direct message: + - To send messages to channels, enter the Slack channel names, separated by commas. + - To send direct messages, use the Member ID found in the user's Slack profile. + + NOTE: **Note:** + Usernames and private channels are not supported. + +1. In **Webhook**, provide the webhook URL that you copied from the + [Slack integration](#slack-configuration) step. +1. (Optional) In **Username**, provide the username of the Slack bot that sends the notifications. +1. Select the **Notify only broken pipelines** check box to only notify on failures. +1. In the **Branches to be notified** select box, choose which types of branches + to send notifications for. +1. Click **Test settings and save changes**. Your Slack team will now start receiving GitLab event notifications as configured. @@ -43,14 +63,14 @@ The following triggers are available for Slack notifications: ## Troubleshooting -If you're having trouble with the Slack integration not working, then start by +If your Slack integration is not working, start troubleshooting by searching through the [Sidekiq logs](../../../administration/logs.md#sidekiqlog) for errors relating to your Slack service. ### Something went wrong on our end -This is a generic error shown in the GitLab UI and doesn't mean much by itself. -You'll need to look in [the logs](../../../administration/logs.md#productionlog) to find +This is a generic error shown in the GitLab UI and does not mean much by itself. +Review [the logs](../../../administration/logs.md#productionlog) to find an error message and keep troubleshooting from there. ### `certificate verify failed` @@ -83,10 +103,10 @@ result = Net::HTTP.get(URI('https://<SLACK URL>'));0 result = Net::HTTP.get(URI('https://<GITLAB URL>'));0 ``` -If it's an issue with GitLab not trusting HTTPS connections to itself, then you may simply +If GitLab is not trusting HTTPS connections to itself, then you may need to [add your certificate to GitLab's trusted certificates](https://docs.gitlab.com/omnibus/settings/ssl.html#install-custom-public-certificates). -If it's an issue with GitLab not trusting connections to Slack, then the GitLab -OpenSSL trust store probably got messed up somehow. Typically this is from overriding -the trust store with `gitlab_rails['env'] = {"SSL_CERT_FILE" => "/path/to/file.pem"}` +If GitLab is not trusting connections to Slack, then the GitLab +OpenSSL trust store is incorrect. Some typical causes: overriding +the trust store with `gitlab_rails['env'] = {"SSL_CERT_FILE" => "/path/to/file.pem"}`, or by accidentally modifying the default CA bundle `/opt/gitlab/embedded/ssl/certs/cacert.pem`. diff --git a/doc/user/project/integrations/youtrack.md b/doc/user/project/integrations/youtrack.md index 119a53f5c35..e067ab6071e 100644 --- a/doc/user/project/integrations/youtrack.md +++ b/doc/user/project/integrations/youtrack.md @@ -13,7 +13,6 @@ To enable YouTrack integration in a project: | Field | Description | |:----------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| - | **Description** | Name for the issue tracker (to differentiate between instances, for example). | | **Project URL** | URL to the project in YouTrack which is being linked to this GitLab project. | | **Issues URL** | URL to the issue in YouTrack project that is linked to this GitLab project. Note that the **Issues URL** requires `:id` in the URL. This ID is used by GitLab as a placeholder to replace the issue number. | diff --git a/doc/user/project/issue_board.md b/doc/user/project/issue_board.md index 89b17609698..1831563332f 100644 --- a/doc/user/project/issue_board.md +++ b/doc/user/project/issue_board.md @@ -42,8 +42,6 @@ Check all the [GitLab Enterprise features for issue boards](#gitlab-enterprise-f  ---- - ## How it works The Issue Board feature builds on GitLab's existing @@ -98,8 +96,6 @@ If you have the labels "**backend**", "**frontend**", "**staging**", and  ---- - ### Use cases for multiple issue boards With [multiple issue boards](#multiple-issue-boards), @@ -252,8 +248,6 @@ clicking **View scope**.  ---- - ### Focus mode > - [Introduced]((https://about.gitlab.com/releases/2017/04/22/gitlab-9-1-released/#issue-boards-focus-mode-ees-eep)) in [GitLab Starter](https://about.gitlab.com/pricing/) 9.1. @@ -275,8 +269,6 @@ especially in combination with [assignee lists](#assignee-lists-premium).  ---- - ### Group issue boards **(PREMIUM)** > [Introduced](https://about.gitlab.com/releases/2017/09/22/gitlab-10-0-released/#group-issue-boards) in [GitLab Premium](https://about.gitlab.com/pricing/) 10.0. @@ -292,8 +284,6 @@ Multiple group issue boards were originally [introduced](https://about.gitlab.co  ---- - ### Assignee lists **(PREMIUM)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/5784) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.0. @@ -313,8 +303,6 @@ To remove an assignee list, just as with a label list, click the trash icon.  ---- - ### Milestone lists **(PREMIUM)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/6469) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.2. @@ -332,8 +320,6 @@ As in other list types, click the trash icon to remove a list.  ---- - ## Work In Progress limits **(STARTER)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/11403) in GitLab 12.7 @@ -365,8 +351,6 @@ status.  ---- - ## Actions you can take on an issue board - [Create a new list](#create-a-new-list). @@ -437,8 +421,6 @@ the list by filtering by author, assignee, milestone, and label.  ---- - ### Remove an issue from a list Removing an issue from a list can be done by clicking the issue card and then @@ -447,8 +429,6 @@ respective label is removed.  ---- - ### Filter issues You should be able to use the filters on top of your issue board to show only @@ -492,8 +472,6 @@ to another list the label changes and a system not is recorded.  ---- - ### Drag issues between lists When dragging issues between lists, different behavior occurs depending on the source list and the target list. @@ -518,8 +496,6 @@ To select and move multiple cards:  ---- - ### Issue ordering in a list When visiting a board, issues appear ordered in any list. You're able to change diff --git a/doc/user/project/issues/crosslinking_issues.md b/doc/user/project/issues/crosslinking_issues.md index 6cc31a45309..c721bef8f4d 100644 --- a/doc/user/project/issues/crosslinking_issues.md +++ b/doc/user/project/issues/crosslinking_issues.md @@ -31,7 +31,8 @@ git commit -m "this is my commit message. Related to https://gitlab.com/<usernam Of course, you can replace `gitlab.com` with the URL of your own GitLab instance. -NOTE: **Note:** Linking your first commit to your issue is going to be relevant +NOTE: **Note:** +Linking your first commit to your issue is going to be relevant for tracking your process with [GitLab Cycle Analytics](https://about.gitlab.com/stages-devops-lifecycle/value-stream-analytics/). It will measure the time taken for planning the implementation of that issue, which is the time between creating an issue and making the first commit. diff --git a/doc/user/project/issues/csv_import.md b/doc/user/project/issues/csv_import.md index d67b135186f..807f4fcffdf 100644 --- a/doc/user/project/issues/csv_import.md +++ b/doc/user/project/issues/csv_import.md @@ -7,7 +7,8 @@ Issues can be imported to a project by uploading a CSV file with the columns The user uploading the CSV file will be set as the author of the imported issues. -NOTE: **Note:** A permission level of [Developer](../../permissions.md), or higher, is required +NOTE: **Note:** +A permission level of [Developer](../../permissions.md), or higher, is required to import issues. ## Prepare for the import diff --git a/doc/user/project/issues/design_management.md b/doc/user/project/issues/design_management.md index ac397592a3b..618506ea7ee 100644 --- a/doc/user/project/issues/design_management.md +++ b/doc/user/project/issues/design_management.md @@ -30,9 +30,11 @@ to be enabled: project level, navigate to your project's **Settings > General**, expand **Visibility, project features, permissions** and enable **Git Large File Storage**. -Design Management requires that projects are using -[hashed storage](../../../administration/repository_storage_types.md#hashed-storage) -(the default storage type since v10.0). +Design Management also requires that projects are using +[hashed storage](../../../administration/raketasks/storage.md#migrate-to-hashed-storage). Since + GitLab 10.0, newly created projects use hashed storage by default. A GitLab admin can verify the storage type of a +project by navigating to **Admin Area > Projects** and then selecting the project in question. +A project can be identified as hashed-stored if its *Gitaly relative path* contains `@hashed`. If the requirements are not met, the **Designs** tab displays a message to the user. @@ -47,6 +49,7 @@ and [PDFs](https://gitlab.com/gitlab-org/gitlab/-/issues/32811) is planned for a ## Limitations - Design uploads are limited to 10 files at a time. +- From GitLab 13.1, Design filenames are limited to 255 characters. - Design Management data [isn't deleted when a project is destroyed](https://gitlab.com/gitlab-org/gitlab/-/issues/13429) yet. - Design Management data [won't be moved](https://gitlab.com/gitlab-org/gitlab/-/issues/13426) @@ -57,20 +60,62 @@ and [PDFs](https://gitlab.com/gitlab-org/gitlab/-/issues/32811) is planned for a - Only the latest version of the designs can be deleted. - Deleted designs cannot be recovered but you can see them on previous designs versions. -## The Design Management page +## GitLab-Figma plugin -Navigate to the **Design Management** page from any issue by clicking the **Designs** tab: +> [Introduced](https://gitlab.com/gitlab-org/gitlab-figma-plugin/-/issues/2) in GitLab 13.2. - +Connect your design environment with your source code management in a seamless workflow. The GitLab-Figma plugin makes it quick and easy to collaborate in GitLab by bringing the work of product designers directly from Figma to GitLab Issues as uploaded Designs. + +To use the plugin, install it from the [Figma Directory](https://www.figma.com/community/plugin/860845891704482356) +and connect to GitLab through a personal access token. The details are explained in the [plugin documentation](https://gitlab.com/gitlab-org/gitlab-figma-plugin/-/wikis/home). + +## The Design Management section + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/223193) in GitLab 13.2, Designs are displayed directly on the issue description rather than on a separate tab. +> - The new display is deployed behind a feature flag, enabled by default. +> - It's enabled on GitLab.com. +> - It cannot be enabled or disabled per-project. +> - It's recommended for production use. +> - For GitLab self-managed instances, GitLab administrators can opt to [disable it](#enable-or-disable-displaying-designs-on-the-issue-description-core-only). If disabled, it will move Designs back to the **Designs** tab. + +You can find to the **Design Management** section in the issue description: + + + +### Enable or disable displaying Designs on the issue description **(CORE ONLY)** + +Displaying Designs on the issue description is under development but ready for +production use. It is deployed behind a feature flag that is **enabled by +default**. +[GitLab administrators with access to the GitLab Rails console](../../../administration/feature_flags.md) +can opt to disable it for your instance. + +To disable it: + +```ruby +Feature.disable(:design_management_moved) +``` + +To enable it: + +```ruby +Feature.enable(:design_management_moved) +``` + +By disabling this feature, designs will be displayed on the **Designs** tab +instead of directly on the issue description. ## Adding designs -To upload design images, click the **Upload Designs** button and select images to upload. +To upload Design images, drag files from your computer and drop them in the Design Management section, +or click **upload** to select images from your file browser: + + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/34353) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.9, you can drag and drop designs onto the dedicated drop zone to upload them. - + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/202634) in GitLab 12.10, you can also copy images from your file system and @@ -239,3 +284,13 @@ To disable it: ```ruby Feature.disable(:design_management_reference_filter_gfm_pipeline) ``` + +## Design activity records + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/33051) in GitLab 13.1. +> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/225205) in GitLab 13.2. + +User activity events on designs (creation, deletion, and updates) are tracked by GitLab and +displayed on the [user profile](../../profile/index.md#user-profile), +[group](../../group/index.md#view-group-activity), +and [project](../index.md#project-activity) activity pages. diff --git a/doc/user/project/issues/img/design_drag_and_drop_uploads_v13_2.png b/doc/user/project/issues/img/design_drag_and_drop_uploads_v13_2.png Binary files differnew file mode 100644 index 00000000000..d60f1234b6d --- /dev/null +++ b/doc/user/project/issues/img/design_drag_and_drop_uploads_v13_2.png diff --git a/doc/user/project/issues/img/design_management_upload_v13.2.png b/doc/user/project/issues/img/design_management_upload_v13.2.png Binary files differnew file mode 100644 index 00000000000..1d4b10307fc --- /dev/null +++ b/doc/user/project/issues/img/design_management_upload_v13.2.png diff --git a/doc/user/project/issues/img/design_management_v13_2.png b/doc/user/project/issues/img/design_management_v13_2.png Binary files differnew file mode 100644 index 00000000000..0a6e2be17ab --- /dev/null +++ b/doc/user/project/issues/img/design_management_v13_2.png diff --git a/doc/user/project/issues/index.md b/doc/user/project/issues/index.md index 06a80672769..9113f5344ba 100644 --- a/doc/user/project/issues/index.md +++ b/doc/user/project/issues/index.md @@ -175,8 +175,6 @@ requires [GraphQL](../../../api/graphql/index.md) to be enabled.  ---- - ### Health status **(ULTIMATE)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/36427) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.10. diff --git a/doc/user/project/issues/issue_data_and_actions.md b/doc/user/project/issues/issue_data_and_actions.md index 6d34f91d37f..7f236d04fb6 100644 --- a/doc/user/project/issues/issue_data_and_actions.md +++ b/doc/user/project/issues/issue_data_and_actions.md @@ -88,7 +88,7 @@ An issue can be assigned to: - Yourself. - Another person. -- [Many people](#multiple-assignees-STARTER). **(STARTER)** +- [Many people](#multiple-assignees-starter). **(STARTER)** The assignee(s) can be changed as often as needed. The idea is that the assignees are responsible for that issue until it's reassigned to someone else to take it from there. @@ -253,7 +253,7 @@ Also: ### Create Merge Request -Create a new branch and [WIP merge request](../merge_requests/work_in_progress_merge_requests.md) +Create a new branch and [**Draft** merge request](../merge_requests/work_in_progress_merge_requests.md) in one action. The branch will be named `issuenumber-title` by default, but you can choose any name, and GitLab will verify that it is not already in use. The merge request will automatically inherit the milestone and labels of the issue, and will be set to diff --git a/doc/user/project/issues/managing_issues.md b/doc/user/project/issues/managing_issues.md index 08e3164b2eb..babc5030337 100644 --- a/doc/user/project/issues/managing_issues.md +++ b/doc/user/project/issues/managing_issues.md @@ -45,8 +45,7 @@ There are many ways to get to the New Issue form from within a project: ### Elements of the New Issue form -> Ability to add the new issue to an epic [was introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/13847) -> in [GitLab Premium](https://about.gitlab.com/pricing/) 13.1. +> Ability to add the new issue to an epic [was introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/13847) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.1.  @@ -76,7 +75,7 @@ create issues for the same project.  -### New issue via Service Desk **(STARTER)** +### New issue via Service Desk Enable [Service Desk](../service_desk.md) for your project and offer email support. By doing so, when your customer sends a new email, a new issue can be created in diff --git a/doc/user/project/labels.md b/doc/user/project/labels.md index 406938519b1..9886ef91f16 100644 --- a/doc/user/project/labels.md +++ b/doc/user/project/labels.md @@ -54,7 +54,7 @@ and edit labels. View the project labels list by going to the project and clicking **Issues > Labels**. The list includes all labels that are defined at the project level, as well as all -labels inherited from the parent group. You can filter the list by entering a search +labels inherited from the immediate parent group. You can filter the list by entering a search query at the top and clicking search (**{search}**). To create a new project label: @@ -94,7 +94,7 @@ also be merged. All issues, merge requests, issue board lists, issue board filters, and label subscriptions with the old labels will be assigned to the new group label. -WARNING: **Caution:** +CAUTION: **Caution:** Promoting a label is a permanent action, and cannot be reversed. To promote a project label to a group label: @@ -251,3 +251,16 @@ If you sort by `Priority`, GitLab uses this sort comparison order: Ties are broken arbitrarily.  + +## Troubleshooting + +### Some label titles end with `_duplicate<number>` + +In specific circumstances it was possible to create labels with duplicate titles in the same +namespace. + +To resolve the duplication, [in GitLab 13.2](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/21384) +and later, some duplicate labels have `_duplicate<number>` appended to their titles. + +You can safely change these labels' titles if you prefer. +For details of the original problem, see [issue 30390](https://gitlab.com/gitlab-org/gitlab/issues/30390). diff --git a/doc/user/project/members/index.md b/doc/user/project/members/index.md index 787a74b0065..3eb411aef18 100644 --- a/doc/user/project/members/index.md +++ b/doc/user/project/members/index.md @@ -128,3 +128,30 @@ If you change your mind before your request is approved, just click the ## Share project with group Alternatively, you can [share a project with an entire group](share_project_with_groups.md) instead of adding users one by one. + +## Remove a member from the project + +Only users with permissions of [Owner](../../permissions.md#group-members-permissions) can manage +project members. + +You can remove a user from the project if the given member has a direct membership in the project. +If membership is inherited from a parent group, then the member can be removed only from the parent +group itself. + +When removing a member, you can decide whether to unassign the user from all issues and merge +requests they are currently assigned or leave the assignments as they are. + +- **Unassigning the removed member** from all issues and merge requests might be helpful when a user + is leaving a private project and you wish to revoke their access to any issues and merge requests + they are assigned. +- **Keeping the issues and merge requests assigned** might be helpful for projects that accept public + contributions where a user doesn't have to be a member to be able to contribute to issues and + merge requests. + +To remove a member from a project: + +1. In a project, go to **{users}** **Members**. +1. Click the **Delete** **{remove}** button next to a project member you want to remove. + A **Remove member** modal appears. +1. (Optional) Select the **Also unassign this user from related issues and merge requests** checkbox. +1. Click **Remove member**. diff --git a/doc/user/project/merge_requests/accessibility_testing.md b/doc/user/project/merge_requests/accessibility_testing.md index 417b85a6082..f3a0aac9ff4 100644 --- a/doc/user/project/merge_requests/accessibility_testing.md +++ b/doc/user/project/merge_requests/accessibility_testing.md @@ -1,4 +1,7 @@ --- +stage: Verify +group: Testing +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference, howto --- @@ -20,7 +23,10 @@ analyzed to a file called `accessibility`. ## Accessibility Merge Request widget -[Since GitLab 13.0](https://gitlab.com/gitlab-org/gitlab/-/issues/39425), in addition to the report artifact that is created, GitLab will also show the +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/39425) in GitLab 13.0 behind the disabled [feature flag](../../../administration/feature_flags.md) `:accessibility_report_view`. +> - [Feature Flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/217372) in GitLab 13.1. + +In addition to the report artifact that is created, GitLab will also show the Accessibility Report in the merge request widget area:  diff --git a/doc/user/project/merge_requests/browser_performance_testing.md b/doc/user/project/merge_requests/browser_performance_testing.md index 390d480724d..10457e40e0b 100644 --- a/doc/user/project/merge_requests/browser_performance_testing.md +++ b/doc/user/project/merge_requests/browser_performance_testing.md @@ -1,4 +1,7 @@ --- +stage: Verify +group: Testing +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference, howto --- @@ -7,20 +10,16 @@ type: reference, howto > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/3507) in [GitLab Premium](https://about.gitlab.com/pricing/) 10.3. If your application offers a web interface and you're using -[GitLab CI/CD](../../../ci/README.md), you can quickly determine the performance -impact of pending code changes. +[GitLab CI/CD](../../../ci/README.md), you can quickly determine the rendering performance +impact of pending code changes in the browser. ## Overview GitLab uses [Sitespeed.io](https://www.sitespeed.io), a free and open source -tool, for measuring the performance of web sites. GitLab has built a simple -[Sitespeed plugin](https://gitlab.com/gitlab-org/gl-performance) which outputs -the performance score for each page analyzed in a file called `performance.json`. -The [Sitespeed.io performance score](https://examples.sitespeed.io/6.0/2017-11-23-23-43-35/help.html) -is a composite value based on best practices. - -GitLab can [show the Performance report](#how-browser-performance-testing-works) -in the merge request widget area. +tool, for measuring the rendering performance of web sites. The +[Sitespeed plugin](https://gitlab.com/gitlab-org/gl-performance) that GitLab built outputs +the performance score for each page analyzed in a file called `browser-performance.json` +this data can be shown on Merge Requests. ## Use cases @@ -38,7 +37,7 @@ Consider the following workflow: ## How browser performance testing works First, define a job in your `.gitlab-ci.yml` file that generates the -[Performance report artifact](../../../ci/pipelines/job_artifacts.md#artifactsreportsperformance-premium). +[Browser Performance report artifact](../../../ci/pipelines/job_artifacts.md#artifactsreportsperformance-premium). GitLab then checks this report, compares key performance metrics for each page between the source and target branches, and shows the information in the merge request. @@ -46,12 +45,13 @@ For an example Performance job, see [Configuring Browser Performance Testing](#configuring-browser-performance-testing). NOTE: **Note:** -If the Performance report has no data to compare, such as when you add the -Performance job in your `.gitlab-ci.yml` for the very first time, no information -displays in the merge request widget area. Consecutive merge requests will have data for -comparison, and the Performance report will be shown properly. +If the Browser Performance report has no data to compare, such as when you add the +Browser Performance job in your `.gitlab-ci.yml` for the very first time, +the Browser Performance report widget won't show. It must have run at least +once on the target branch (`master`, for example), before it will display in a +merge request targeting that branch. - + ## Configuring Browser Performance Testing @@ -61,21 +61,7 @@ using Docker-in-Docker. 1. First, set up GitLab Runner with a [Docker-in-Docker build](../../../ci/docker/using_docker_build.md#use-docker-in-docker-workflow-with-docker-executor). -1. After configuring the Runner, add a new job to `.gitlab-ci.yml` that generates - the expected report. -1. Define the `performance` job according to your version of GitLab: - - - For GitLab 12.4 and later - [include](../../../ci/yaml/README.md#includetemplate) the - [`Browser-Performance.gitlab-ci.yml` template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml) provided as a part of your GitLab installation. - - For GitLab versions earlier than 12.4 - Copy and use the job as defined in the - [`Browser-Performance.gitlab-ci.yml` template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml). - - CAUTION: **Caution:** - The job definition provided by the template does not support Kubernetes yet. - For a complete example of a more complex setup that works in Kubernetes, see - [`Browser-Performance-Testing.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml). - -1. Add the following to your `.gitlab-ci.yml` file: +1. Configure the default Browser Performance Testing CI job as follows in your `.gitlab-ci.yml` file: ```yaml include: @@ -86,24 +72,32 @@ using Docker-in-Docker. URL: https://example.com ``` - CAUTION: **Caution:** - The job definition provided by the template is supported in GitLab 11.5 and later versions. - It also requires GitLab Runner 11.5 or later. For earlier versions, use the - [previous job definitions](#previous-job-definitions). +NOTE: **Note:** +For versions before 12.4, see the information for [older GitLab versions](#gitlab-versions-123-and-older). +If you are using a Kubernetes cluster, use [`template: Jobs/Browser-Performance-Testing.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml) +instead. The above example creates a `performance` job in your CI/CD pipeline and runs sitespeed.io against the webpage you defined in `URL` to gather key metrics. -The [GitLab plugin for sitespeed.io](https://gitlab.com/gitlab-org/gl-performance) -is downloaded to save the report as a [Performance report artifact](../../../ci/pipelines/job_artifacts.md#artifactsreportsperformance-premium) -that you can later download and analyze. Due to implementation limitations, we always -take the latest Performance artifact available. -The full HTML sitespeed.io report is saved as an artifact, and if -[GitLab Pages](../pages/index.md) is enabled, it can be viewed directly in your browser. +The example uses a CI/CD template that is included in all GitLab installations since +12.4, but it will not work with Kubernetes clusters. If you are using GitLab 12.3 +or older, you must [add the configuration manually](#gitlab-versions-123-and-older) + +The template uses the [GitLab plugin for sitespeed.io](https://gitlab.com/gitlab-org/gl-performance), +and it saves the full HTML sitespeed.io report as a [Browser Performance report artifact](../../../ci/pipelines/job_artifacts.md#artifactsreportsperformance-premium) +that you can later download and analyze. This implementation always takes the latest +Browser Performance artifact available. If [GitLab Pages](../pages/index.md) is enabled, +you can view the report directly in your browser. + +You can also customize the jobs with environment variables: + +- `SITESPEED_IMAGE`: Configure the Docker image to use for the job (default `sitespeedio/sitespeed.io`), but not the image version. +- `SITESPEED_VERSION`: Configure the version of the Docker image to use for the job (default `13.3.0`). +- `SITESPEED_OPTIONS`: Configure any additional sitespeed.io options as required (default `nil`). Refer to the [sitespeed.io documentation](https://www.sitespeed.io/documentation/sitespeed.io/configuration/) for more details. -You can also customize options by setting the `SITESPEED_OPTIONS` variable. For example, you can override the number of runs sitespeed.io -makes on the given URL: +makes on the given URL, and change the version: ```yaml include: @@ -111,18 +105,11 @@ include: performance: variables: - URL: https://example.com + URL: https://www.sitespeed.io/ + SITESPEED_VERSION: 13.2.0 SITESPEED_OPTIONS: -n 5 ``` -For further customization options for sitespeed.io, including the ability to provide a -list of URLs to test, please see the -[Sitespeed.io Configuration](https://www.sitespeed.io/documentation/sitespeed.io/configuration/) -documentation. - -TIP: **Tip:** -Key metrics are automatically extracted and shown in the merge request widget. - ### Configuring degradation threshold > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/27599) in GitLab 13.0. @@ -149,15 +136,12 @@ The above CI YAML configuration is great for testing against static environments be extended for dynamic environments, but a few extra steps are required: 1. The `performance` job should run after the dynamic environment has started. -1. In the `review` job, persist the hostname and upload it as an artifact so - it's available to the `performance` job. The same can be done for static - environments like staging and production to unify the code path. You can save it - as an artifact with `echo $CI_ENVIRONMENT_URL > environment_url.txt` - in your job's `script`. -1. In the `performance` job, read the previous artifact into an environment - variable. In this case, use `$URL` because the sitespeed.io command - uses it for the URL parameter. Because Review App URLs are dynamic, define - the `URL` variable through `before_script` instead of `variables`. +1. In the `review` job: + 1. Generate a URL list file with the dynamic URL. + 1. Save the file as an artifact, for example with `echo $CI_ENVIRONMENT_URL > environment_url.txt` + in your job's `script`. + 1. Pass the list as the URL environment variable (which can be a URL or a file containing URLs) + to the `performance` job. 1. You can now run the sitespeed.io container against the desired hostname and paths. @@ -190,20 +174,21 @@ review: performance: dependencies: - review - before_script: - - export URL=$(cat environment_url.txt) + variables: + URL: environment_url.txt ``` -### Previous job definitions +### GitLab versions 12.3 and older -CAUTION: **Caution:** -Before GitLab 11.5, the Performance job and artifact had to be named specifically -to automatically extract report data and show it in the merge request widget. -While these old job definitions are still maintained, they have been deprecated -and may be removed in next major release, GitLab 12.0. -GitLab recommends you update your current `.gitlab-ci.yml` configuration to reflect that change. +Browser Performance Testing has gone through several changes since it's introduction. +In this section we'll detail these changes and how you can run the test based on your +GitLab version: -For GitLab 11.4 and earlier, the job should look like: +- In GitLab 12.4 [a job template was made available](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml). +- In 13.2 the feature was renamed from `Performance` to `Browser Performance` with +additional template variables. The job name in the template is still `performance` +for compatibility reasons, but may be renamed to match in a future iteration. +- For 11.5 to 12.3 no template is available and the job has to be defined manually as follows: ```yaml performance: @@ -211,28 +196,45 @@ performance: image: docker:git variables: URL: https://example.com + SITESPEED_VERSION: 13.3.0 + SITESPEED_OPTIONS: '' services: - docker:stable-dind script: - mkdir gitlab-exporter - wget -O ./gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/master/index.js - mkdir sitespeed-results - - docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:6.3.1 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results $URL + - docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --outputFolder sitespeed-results $URL $SITESPEED_OPTIONS - mv sitespeed-results/data/performance.json performance.json artifacts: paths: - performance.json - sitespeed-results/ + reports: + performance: performance.json ``` -<!-- ## Troubleshooting +- For 11.4 and earlier the job should be defined as follows: -Include any troubleshooting steps that you can foresee. If you know beforehand what issues -one might have when setting this up, or when something is changed, or on upgrading, it's -important to describe those, too. Think of things that may go wrong and include them here. -This is important to minimize requests for support, and to avoid doc comments with -questions that you know someone might ask. +```yaml +performance: + stage: performance + image: docker:git + variables: + URL: https://example.com + services: + - docker:stable-dind + script: + - mkdir gitlab-exporter + - wget -O ./gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/master/index.js + - mkdir sitespeed-results + - docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:6.3.1 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results $URL + - mv sitespeed-results/data/performance.json performance.json + artifacts: + paths: + - performance.json + - sitespeed-results/ +``` -Each scenario can be a third-level heading, e.g. `### Getting error message X`. -If you have none to add when creating a doc, leave this section in place -but commented out to help encourage others to add to it in the future. --> +Upgrading to the latest version and using the templates is recommended, to ensure +you receive the latest updates, including updates to the sitespeed.io versions. diff --git a/doc/user/project/merge_requests/code_quality.md b/doc/user/project/merge_requests/code_quality.md index 7b4d4b668d5..36acba032ff 100644 --- a/doc/user/project/merge_requests/code_quality.md +++ b/doc/user/project/merge_requests/code_quality.md @@ -1,8 +1,11 @@ --- +stage: Verify +group: Testing +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference, howto --- -# Code Quality **(STARTER)** +# Code Quality > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/1984) in [GitLab Starter](https://about.gitlab.com/pricing/) 9.3. @@ -22,8 +25,13 @@ Code Quality: DevOps](../../../topics/autodevops/stages.md#auto-code-quality-starter). - Can be extended through [Analysis Plugins](https://docs.codeclimate.com/docs/list-of-engines) or a [custom tool](#implementing-a-custom-tool). +## Code Quality Widget + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/1984) in [GitLab Starter](https://about.gitlab.com/pricing/) 9.3. +> - Made [available in all tiers](https://gitlab.com/gitlab-org/gitlab/-/issues/212499) in 13.2. + Going a step further, GitLab can show the Code Quality report right -in the merge request widget area: +in the merge request widget area if a report from the target branch is available to compare to:  @@ -79,7 +87,7 @@ include: The above example will create a `code_quality` job in your CI/CD pipeline which will scan your source code for code quality issues. The report will be saved as a -[Code Quality report artifact](../../../ci/pipelines/job_artifacts.md#artifactsreportscodequality-starter) +[Code Quality report artifact](../../../ci/pipelines/job_artifacts.md#artifactsreportscodequality) that you can later download and analyze. It's also possible to override the URL to the Code Quality image by @@ -237,7 +245,7 @@ do this: 1. Define a job in your `.gitlab-ci.yml` file that generates the [Code Quality report - artifact](../../../ci/pipelines/job_artifacts.md#artifactsreportscodequality-starter). + artifact](../../../ci/pipelines/job_artifacts.md#artifactsreportscodequality). 1. Configure your tool to generate the Code Quality report artifact as a JSON file that implements a subset of the [Code Climate spec](https://github.com/codeclimate/platform/blob/master/spec/analyzers/SPEC.md#data-types). @@ -273,11 +281,11 @@ NOTE: **Note:** Although the Code Climate spec supports more properties, those are ignored by GitLab. -## Code Quality reports +## Code Quality reports **(STARTER)** Once the Code Quality job has completed: -- The full list of code quality violations generated by a pipeline is available in the +- The full list of code quality violations generated by a pipeline is shown in the Code Quality tab of the Pipeline Details page. - Potential changes to code quality are shown directly in the merge request. The Code Quality widget in the merge request compares the reports from the base and head of the branch, @@ -286,8 +294,43 @@ Once the Code Quality job has completed: [downloadable artifact](../../../ci/pipelines/job_artifacts.md#downloading-artifacts) for the `code_quality` job. +## Extending functionality + +### Using Analysis Plugins + +Should there be a need to extend the default functionality provided by Code Quality, as stated in [Code Quality](#code-quality), [Analysis Plugins](https://docs.codeclimate.com/docs/list-of-engines) are available. + +For example, to use the [SonarJava analyzer](https://docs.codeclimate.com/docs/sonar-java), +add a file named `.codeclimate.yml` containing the [enablement code](https://docs.codeclimate.com/docs/sonar-java#enable-the-plugin) +for the plugin to the root of your repository: + +```yaml +version: "2" +plugins: + sonar-java: + enabled: true +``` + +This adds SonarJava to the `plugins:` section of the [default `.codeclimate.yml`](https://gitlab.com/gitlab-org/ci-cd/codequality/-/blob/master/codeclimate_defaults/.codeclimate.yml) +included in your project. + +Changes to the `plugins:` section do not affect the `exclude_patterns` section of the +defeault `.codeclimate.yml`. See the Code Climate documentation for +[excluding files and folders](https://docs.codeclimate.com/docs/excluding-files-and-folders) +for more details. + +Here's [an example project](https://gitlab.com/jheimbuck_gl/jh_java_example_project) that uses Code Quality with a `.codeclimate.yml` file. + ## Troubleshooting +### Changing the default configuration has no effect + +A common issue is that the terms `Code Quality` (GitLab specific) and `Code Climate` +(Engine used by GitLab) are very similar. You must add a **`.codeclimate.yml`** file +to change the default configuration, **not** a `.codequality.yml` file. If you use +the wrong filename, the [default `.codeclimate.yml`](https://gitlab.com/gitlab-org/ci-cd/codequality/-/blob/master/codeclimate_defaults/.codeclimate.yml) +is still used. + ### No Code Quality report is displayed in a Merge Request This can be due to multiple reasons: @@ -295,6 +338,7 @@ This can be due to multiple reasons: - You just added the Code Quality job in your `.gitlab-ci.yml`. The report does not have anything to compare to yet, so no information can be displayed. Future merge requests will have something to compare to. +- Your pipeline is not set to run the code quality job on your default branch. If there is no report generated from the default branch, your MR branch reports will not have anything to compare to. - If no [degradation or error is detected](https://docs.codeclimate.com/docs/maintainability#section-checks), nothing will be displayed. - The [`artifacts:expire_in`](../../../ci/yaml/README.md#artifactsexpire_in) CI/CD diff --git a/doc/user/project/merge_requests/fail_fast_testing.md b/doc/user/project/merge_requests/fail_fast_testing.md new file mode 100644 index 00000000000..619a6d04577 --- /dev/null +++ b/doc/user/project/merge_requests/fail_fast_testing.md @@ -0,0 +1,87 @@ +--- +stage: Verify +group: Testing +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: reference, howto +--- + +# Fail Fast Testing **(PREMIUM)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/198550) in GitLab 13.1. + +For applications that use RSpec for running tests, we've introduced the `Verify/Failfast` +[template to run subsets of your test suite](https://gitlab.com/gitlab-org/gitlab/-/tree/master/lib/gitlab/ci/templates/Verify/FailFast.gitlab-ci.yml), +based on the changes in your merge request. + +The template uses the [test_file_finder (`tff`) gem](https://gitlab.com/gitlab-org/ci-cd/test_file_finder/) +that accepts a list of files as input, and returns a list of spec (test) files +that it believes to be relevant to the input files. + +`tff` is designed for Ruby on Rails projects, so the `Verify/FailFast` template is +configured to run when changes to Ruby files are detected. By default, it runs in +the [`.pre` stage](../../../ci/yaml/README.md#pre-and-post) of a GitLab CI/CD pipeline, +before all other stages. + +## Example use case + +Fail fast testing is useful when adding new functionality to a project and adding +new automated tests. + +Your project could have hundreds of thousands of tests that take a long time to complete. +You may be confident that a new test will pass, but you have to wait for all the tests +to complete to verify it. This could take an hour or more, even when using parallelization. + +Fail fast testing gives you a faster feedback loop from the pipeline. It lets you +know quickly that the new tests are passing and the new functionality did not break +other tests. + +## Requirements + +This template requires: + +- A project built in Rails that uses RSpec for testing. +- CI/CD configured to: + - Use a Docker image with Ruby available. + - Use [Pipelines for Merge Requests](../../../ci/merge_request_pipelines/index.md#configuring-pipelines-for-merge-requests) +- [Pipelines for Merged Results](../../../ci/merge_request_pipelines/pipelines_for_merged_results/index.md#enable-pipelines-for-merged-results) + enabled in the project settings. + +## Configure Fast RSpec Failure + +We'll use the following plain RSpec configuration as a starting point. It installs all the +project gems and executes `rspec`, on merge request pipelines only. + +```yaml +rspec-complete: + stage: test + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + script: + - bundle install + - bundle exec rspec +``` + +To run the most relevant specs first instead of the whole suite, [`include`](../../../ci/yaml/README.md#include) +the template by adding the following to your CI/CD configuration: + +```yaml +include: + - template: Verify/FailFast.gitlab-ci.yml +``` + +### Example test loads + +For illustrative purposes, let's say our Rails app spec suite consists of 100 specs per model for ten models. + +If no Ruby files are changed: + +- `rspec-rails-modified-paths-specs` will not run any tests. +- `rspec-complete` will run the full suite of 1000 tests. + +If one Ruby model is changed, for example `app/models/example.rb`, then `rspec-rails-modified-paths-specs` +will run the 100 tests for `example.rb`: + +- If all of these 100 tests pass, then the full `rspec-complete` suite of 1000 tests is allowed to run. +- If any of these 100 tests fail, they will fail quickly, and `rspec-complete` will not run any tests. + +The final case saves resources and time as the full 1000 test suite does not run. diff --git a/doc/user/project/merge_requests/getting_started.md b/doc/user/project/merge_requests/getting_started.md index 32eb0c73ed4..9ac0f3ee42e 100644 --- a/doc/user/project/merge_requests/getting_started.md +++ b/doc/user/project/merge_requests/getting_started.md @@ -57,7 +57,7 @@ request's page at the top-right side: - [Close issues automatically](#merge-requests-to-close-issues) when they are merged. - Enable the [delete source branch when merge request is accepted](#deleting-the-source-branch) option to keep your repository clean. - Enable the [squash commits when merge request is accepted](squash_and_merge.md) option to combine all the commits into one before merging, thus keep a clean commit history in your repository. -- Set the merge request as a [Work In Progress (WIP)](work_in_progress_merge_requests.md) to avoid accidental merges before it is ready. +- Set the merge request as a [**Draft**](work_in_progress_merge_requests.md) to avoid accidental merges before it is ready. Once you have created the merge request, you can also: diff --git a/doc/user/project/merge_requests/img/browser_performance_testing.png b/doc/user/project/merge_requests/img/browser_performance_testing.png Binary files differindex eea77fb8b93..c270462f7a8 100644 --- a/doc/user/project/merge_requests/img/browser_performance_testing.png +++ b/doc/user/project/merge_requests/img/browser_performance_testing.png diff --git a/doc/user/project/merge_requests/img/draft_blocked_merge_button_v13_2.png b/doc/user/project/merge_requests/img/draft_blocked_merge_button_v13_2.png Binary files differnew file mode 100644 index 00000000000..beb12c581d6 --- /dev/null +++ b/doc/user/project/merge_requests/img/draft_blocked_merge_button_v13_2.png diff --git a/doc/user/project/merge_requests/img/file_by_file_v13_2.png b/doc/user/project/merge_requests/img/file_by_file_v13_2.png Binary files differnew file mode 100644 index 00000000000..e3114ebabad --- /dev/null +++ b/doc/user/project/merge_requests/img/file_by_file_v13_2.png diff --git a/doc/user/project/merge_requests/img/load_performance_testing.png b/doc/user/project/merge_requests/img/load_performance_testing.png Binary files differnew file mode 100644 index 00000000000..3a58e9c28d4 --- /dev/null +++ b/doc/user/project/merge_requests/img/load_performance_testing.png diff --git a/doc/user/project/merge_requests/img/merge_when_pipeline_succeeds_only_if_succeeds_msg.png b/doc/user/project/merge_requests/img/merge_when_pipeline_succeeds_only_if_succeeds_msg.png Binary files differdeleted file mode 100644 index 761690d1e0c..00000000000 --- a/doc/user/project/merge_requests/img/merge_when_pipeline_succeeds_only_if_succeeds_msg.png +++ /dev/null diff --git a/doc/user/project/merge_requests/img/wip_blocked_accept_button.png b/doc/user/project/merge_requests/img/wip_blocked_accept_button.png Binary files differdeleted file mode 100644 index ab2c8425b83..00000000000 --- a/doc/user/project/merge_requests/img/wip_blocked_accept_button.png +++ /dev/null diff --git a/doc/user/project/merge_requests/index.md b/doc/user/project/merge_requests/index.md index 5d2813f5bfc..f68fc7d7b45 100644 --- a/doc/user/project/merge_requests/index.md +++ b/doc/user/project/merge_requests/index.md @@ -19,7 +19,7 @@ A. Consider you're a software developer working in a team: 1. You checkout a new branch, and submit your changes through a merge request 1. You gather feedback from your team -1. You work on the implementation optimizing code with [Code Quality reports](code_quality.md) **(STARTER)** +1. You work on the implementation optimizing code with [Code Quality reports](code_quality.md) 1. You verify your changes with [JUnit test reports](../../../ci/junit_test_reports.md) in GitLab CI/CD 1. You avoid using dependencies whose license is not compatible with your project with [License Compliance reports](../../compliance/license_compliance/index.md) **(ULTIMATE)** 1. You request the [approval](merge_request_approvals.md) from your manager **(STARTER)** diff --git a/doc/user/project/merge_requests/load_performance_testing.md b/doc/user/project/merge_requests/load_performance_testing.md new file mode 100644 index 00000000000..3239269109d --- /dev/null +++ b/doc/user/project/merge_requests/load_performance_testing.md @@ -0,0 +1,197 @@ +--- +stage: Verify +group: Testing +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: reference, howto +--- + +# Load Performance Testing **(PREMIUM)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/10683) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.2. + +With Load Performance Testing, you can test the impact of any pending code changes +to your application's backend in [GitLab CI/CD](../../../ci/README.md). + +GitLab uses [k6](https://k6.io/), a free and open source +tool, for measuring the system performance of applications under +load. + +Unlike [Browser Performance Testing](browser_performance_testing.md), which is +used to measure how web sites perform in client browsers, Load Performance Testing +can be used to perform various types of [load tests](https://k6.io/docs/#use-cases) +against application endpoints such as APIs, Web Controllers, and so on. +This can be used to test how the backend or the server performs at scale. + +For example, you can use Load Performance Testing to perform many concurrent +GET calls to a popular API endpoint in your application to see how it performs. + +## How Load Performance Testing works + +First, define a job in your `.gitlab-ci.yml` file that generates the +[Load Performance report artifact](../../../ci/pipelines/job_artifacts.md#artifactsreportsload_performance-premium). +GitLab checks this report, compares key load performance metrics +between the source and target branches, and then shows the information in a merge request widget: + + + +Next, you need to configure the test environment and write the k6 test. + +The key performance metrics that the merge request widget shows after the test completes are: + +- Checks: The percentage pass rate of the [checks](https://k6.io/docs/using-k6/checks) configured in the k6 test. +- TTFB P90: The 90th percentile of how long it took to start receiving responses, aka the [Time to First Byte](https://en.wikipedia.org/wiki/Time_to_first_byte) (TTFB). +- TTFB P95: The 95th percentile for TTFB. +- RPS: The average requests per second (RPS) rate the test was able to achieve. + +NOTE: **Note:** +If the Load Performance report has no data to compare, such as when you add the +Load Performance job in your `.gitlab-ci.yml` for the very first time, +the Load Performance report widget won't show. It must have run at least +once on the target branch (`master`, for example), before it will display in a +merge request targeting that branch. + +## Configure the Load Performance Testing job + +Configuring your Load Performance Testing job can be broken down into several distinct parts: + +- Determine the test parameters such as throughput, and so on. +- Set up the target test environment for load performance testing. +- Design and write the k6 test. + +### Determine the test parameters + +The first thing you need to do is determine the [type of load test](https://k6.io/docs/test-types/introduction) +you want to run, and how it will run (for example, the number of users, throughput, and so on). + +Refer to the [k6 docs](https://k6.io/docs/), especially the [k6 testing guides](https://k6.io/docs/testing-guides), +for guidance on the above and more. + +### Test Environment setup + +A large part of the effort around load performance testing is to prepare the target test environment +for high loads. You should ensure it's able to handle the +[throughput](https://k6.io/blog/monthly-visits-concurrent-users) it will be tested with. + +It's also typically required to have representative test data in the target environment +for the load performance test to use. + +We strongly recommend [not running these tests against a production environment](https://k6.io/our-beliefs#load-test-in-a-pre-production-environment). + +### Write the load performance test + +After the environment is prepared, you can write the k6 test itself. k6 is a flexible +tool and can be used to run [many kinds of performance tests](https://k6.io/docs/test-types/introduction). +Refer to the [k6 documentation](https://k6.io/docs/) for detailed information on how to write tests. + +### Configure the test in GitLab CI/CD + +When your k6 test is ready, the next step is to configure the load performance +testing job in GitLab CI/CD. The easiest way to do this is to use the +[`Verify/Load-Performance-Testing.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml) +template that is included with GitLab. + +NOTE: **Note:** +For large scale k6 tests you need to ensure the GitLab Runner instance performing the actual +test is able to handle running the test. Refer to [k6's guidance](https://k6.io/docs/testing-guides/running-large-tests#hardware-considerations) +for spec details. The [default shared GitLab.com runners](../../gitlab_com/#linux-shared-runners) +likely have insufficient specs to handle most large k6 tests. + +This template runs the +[k6 Docker container](https://hub.docker.com/r/loadimpact/k6/) in the job and provides several ways to customize the +job. + +An example configuration workflow: + +1. Set up a GitLab Runner that can run Docker containers, such as a Runner using the + [Docker-in-Docker workflow](../../../ci/docker/using_docker_build.md#use-docker-in-docker-workflow-with-docker-executor). +1. Configure the default Load Performance Testing CI job in your `.gitlab-ci.yml` file. + You need to include the template and configure it with variables: + + ```yaml + include: + template: Verify/Load-Performance-Testing.gitlab-ci.yml + + load_performance: + variables: + K6_TEST_FILE: <PATH TO K6 TEST FILE IN PROJECT> + ``` + +The above example creates a `load_performance` job in your CI/CD pipeline that runs +the k6 test. + +NOTE: **Note:** +For Kubernetes setups a different template should be used: [`Jobs/Load-Performance-Testing.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml). + +k6 has [various options](https://k6.io/docs/using-k6/options) to configure how it will run tests, such as what throughput (RPS) to run with, +how long the test should run, and so on. Almost all options can be configured in the test itself, but as +you can also pass command line options via the `K6_OPTIONS` variable. + +For example, you can override the duration of the test with a CLI option: + +```yaml + include: + template: Verify/Load-Performance-Testing.gitlab-ci.yml + + load_performance: + variables: + K6_TEST_FILE: <PATH TO K6 TEST FILE IN PROJECT> + K6_OPTIONS: '--duration 30s' +``` + +GitLab only displays the key performance metrics in the MR widget if k6's results are saved +via [summary export](https://k6.io/docs/results-visualization/json#summary-export) +as a [Load Performance report artifact](../../../ci/pipelines/job_artifacts.md#artifactsreportsload_performance-premium). +The latest Load Performance artifact available is always used. + +If [GitLab Pages](../pages/index.md) is enabled, you can view the report directly in your browser. + +### Load Performance testing in Review Apps + +The CI/CD YAML configuration example above works for testing against static environments, +but it can be extended to work with [review apps](../../../ci/review_apps) or +[dynamic environments](../../../ci/environments) with a few extra steps. + +The best approach is to capture the dynamic URL into a custom environment variable that +is then [inherited](../../../ci/variables/README.md#inherit-environment-variables) +by the `load_performance` job. The k6 test script to be run should then be configured to +use that environment URL, such as: ``http.get(`${__ENV.ENVIRONMENT_URL`})``. + +For example: + +1. In the `review` job: + 1. Capture the dynamic URL and save it into a `.env` file, e.g. `echo "ENVIRONMENT_URL=$CI_ENVIRONMENT_URL" >> review.env`. + 1. Set the `.env` file to be an [`artifacts:reports:dotenv` report](../../../ci/variables/README.md#inherit-environment-variables). +1. Set the `load_performance` job to depend on the review job, so it inherits the environment variable. +1. Configure the k6 test script to use the environment variable in it's steps. + +Your `.gitlab-ci.yml` file might be similar to: + +```yaml +stages: + - deploy + - performance + +include: + template: Verify/Load-Performance-Testing.gitlab-ci.yml + +review: + stage: deploy + environment: + name: review/$CI_COMMIT_REF_NAME + url: http://$CI_ENVIRONMENT_SLUG.example.com + script: + - run_deploy_script + - echo "ENVIRONMENT_URL=$CI_ENVIRONMENT_URL" >> review.env + artifacts: + reports: + dotenv: + review.env + rules: + - if: '$CI_COMMIT_BRANCH' # Modify to match your pipeline rules, or use `only/except` if needed. + +load_performance: + dependencies: + - review + rules: + - if: '$CI_COMMIT_BRANCH' # Modify to match your pipeline rules, or use `only/except` if needed. +``` diff --git a/doc/user/project/merge_requests/merge_request_dependencies.md b/doc/user/project/merge_requests/merge_request_dependencies.md index a1012e27d32..65f3cb1e0d5 100644 --- a/doc/user/project/merge_requests/merge_request_dependencies.md +++ b/doc/user/project/merge_requests/merge_request_dependencies.md @@ -38,15 +38,15 @@ the `awesome-project` merge request before the `awesome-lib` one would break the `master`branch. The `awesome-project` merge request could be [marked as -WIP](work_in_progress_merge_requests.md), -and the reason for the WIP stated included in the comments. However, this +**Draft**](work_in_progress_merge_requests.md), +and the reason for the draft stated included in the comments. However, this requires the state of the `awesome-lib` merge request to be manually tracked, and doesn't scale well if the `awesome-project` merge request depends on changes to **several** other projects. By making the `awesome-project` merge request depend on the `awesome-lib` merge request instead, this relationship is -automatically tracked by GitLab, and the WIP state can be used to +automatically tracked by GitLab, and the draft state can be used to communicate the readiness of the code in each individual merge request instead. diff --git a/doc/user/project/merge_requests/merge_when_pipeline_succeeds.md b/doc/user/project/merge_requests/merge_when_pipeline_succeeds.md index d45ccdc9be9..7d90c9f3cd7 100644 --- a/doc/user/project/merge_requests/merge_when_pipeline_succeeds.md +++ b/doc/user/project/merge_requests/merge_when_pipeline_succeeds.md @@ -1,36 +1,38 @@ --- +stage: Create +group: Source Code +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference, concepts --- # Merge when pipeline succeeds -When reviewing a merge request that looks ready to merge but still has one or -more CI jobs running, you can set it to be merged automatically when the -jobs pipeline succeeds. This way, you don't have to wait for the jobs to +When reviewing a merge request that looks ready to merge but still has a +pipeline running, you can set it to merge automatically when the +pipeline succeeds. This way, you don't have to wait for the pipeline to finish and remember to merge the request manually.  ## How it works -When you hit the "Merge When Pipeline Succeeds" button, the status of the merge -request will be updated to represent the impending merge. If you cannot wait -for the pipeline to succeed and want to merge immediately, this option is -available in the dropdown menu on the right of the main button. +When you click "Merge When Pipeline Succeeds", the status of the merge +request is updated to show the impending merge. If you can't wait +for the pipeline to succeed, you can choose **Merge immediately** +in the dropdown menu on the right of the main button. -Both team developers and the author of the merge request have the option to -cancel the automatic merge if they find a reason why it shouldn't be merged -after all. +The author of the merge request and project members with developer permissions can +cancel the automatic merge at any time before the pipeline finishes.  -When the pipeline succeeds, the merge request will automatically be merged. +When the pipeline succeeds, the merge request is automatically merged. When the pipeline fails, the author gets a chance to retry any failed jobs, or to push new commits to fix the failure. When the jobs are retried and succeed on the second try, the merge request -will automatically be merged after all. When the merge request is updated with -new commits, the automatic merge is automatically canceled to allow the new +is automatically merged. When the merge request is updated with +new commits, the automatic merge is canceled to allow the new changes to be reviewed. ## Only allow merge requests to be merged if the pipeline succeeds @@ -42,7 +44,7 @@ or if there are threads to be resolved. This works for both: - Pipelines run from an [external CI integration](../integrations/overview.md#integrations-listing) As a result, [disabling GitLab CI/CD pipelines](../../../ci/enable_or_disable_ci.md) -will not disable this feature, as it will still be possible to use pipelines from external +does not disable this feature, as it is possible to use pipelines from external CI providers with this feature. To enable it, you must: 1. Navigate to your project's **Settings > General** page. @@ -50,14 +52,40 @@ CI providers with this feature. To enable it, you must: 1. In the **Merge checks** subsection, select the **Pipelines must succeed** checkbox. 1. Press **Save** for the changes to take effect. -NOTE: **Note:** This setting also prevents merge requests from being merged if there is no pipeline. +This setting also prevents merge requests from being merged if there is no pipeline. - +### Limitations + +When this setting is enabled, a merge request is prevented from being merged if there +is no pipeline. This may conflict with some use cases where [`only/except`](../../../ci/yaml/README.md#onlyexcept-advanced) +or [`rules`](../../../ci/yaml/README.md#rules) are used and they don't generate any pipelines. + +You should ensure that [there is always a pipeline](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/54226) +and that it's successful. + +If both a branch pipeline and a merge request pipeline are triggered for a single +merge request, only the success or failure of the *merge request pipeline* is checked. +If the merge request pipeline is configured with fewer jobs than the branch pipeline, +it could allow code that fails tests to be merged: + +```yaml +branch-pipeline-job: + rules: + - if: '$CI_PIPELINE_SOURCE == "push"' + script: + - echo "Code testing scripts here, for example." -From now on, every time the pipeline fails you will not be able to merge the -merge request from the UI, until you make all relevant jobs pass. +merge-request-pipeline-job: + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + script: + - echo "No tests run, but this pipeline always succeeds and enables merge." + - echo true +``` - +You should avoid configuration like this, and only use branch (`push`) pipelines +or merge request pipelines, when possible. See [`rules` documentation](../../../ci/yaml/README.md#differences-between-rules-and-onlyexcept) +for details on avoiding two pipelines for a single merge request. ### Skipped pipelines @@ -72,20 +100,10 @@ merge requests from being merged. To change this behavior: 1. In the **Merge checks** subsection, select the **Skipped pipelines are considered successful** checkbox. 1. Press **Save** for the changes to take effect. -### Limitations - -When this setting is enabled, a merge request is prevented from being merged if there is no pipeline. This may conflict with some use cases where [`only/except`](../../../ci/yaml/README.md#onlyexcept-advanced) rules are used and they don't generate any pipelines. - -Users that expect to be able to merge a merge request in this scenario should ensure that [there is always a pipeline](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/54226) and that it's successful. +## From the command line -For example, to that on merge requests there is always a passing job even though `only/except` rules may not generate any other jobs: - -```yaml -enable_merge: - only: [merge_requests] - script: - - echo true -``` +You can use [Push Options](../push_options.md) to enable merge when pipeline succeeds +for a merge request when pushing from the command line. <!-- ## Troubleshooting @@ -98,8 +116,3 @@ questions that you know someone might ask. Each scenario can be a third-level heading, e.g. `### Getting error message X`. If you have none to add when creating a doc, leave this section in place but commented out to help encourage others to add to it in the future. --> - -## Use it from the command line - -You can use [Push Options](../push_options.md) to trigger this feature when -pushing. diff --git a/doc/user/project/merge_requests/reviewing_and_managing_merge_requests.md b/doc/user/project/merge_requests/reviewing_and_managing_merge_requests.md index a3ad41d8dd8..162ebdf157d 100644 --- a/doc/user/project/merge_requests/reviewing_and_managing_merge_requests.md +++ b/doc/user/project/merge_requests/reviewing_and_managing_merge_requests.md @@ -64,6 +64,43 @@ list.  +### File-by-file diff navigation + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/222790) in GitLab 13.2. +> - It's deployed behind a feature flag, disabled by default. +> - It's enabled on GitLab.com. +> - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enable-or-disable-file-by-file-diff-navigation-core-only). + +For larger merge requests it might sometimes be useful to review single files at a time. To enable, +from your avatar on the top-right navbar, click **Settings**, and go to **Preferences** on the left +sidebar. Scroll down to the **Behavior** section and select **Show one file at a time on merge request's Changes tab**. +Click **Save changes** to apply. + +From there, when reviewing merge requests' **Changes** tab, you will see only one file at a time. You can then click the buttons **Prev** and **Next** to view the other files changed. + + + +#### Enable or disable file-by-file diff navigation **(CORE ONLY)** + +File-by-file diff navigation is under development but ready for production use. It is +deployed behind a feature flag that is **disabled by default**. +[GitLab administrators with access to the GitLab Rails console](../../../administration/feature_flags.md) +can enable it for your instance. + +To enable it: + +```ruby +# Instance-wide +Feature.enable(:view_diffs_file_by_file) +``` + +To disable it: + +```ruby +# Instance-wide +Feature.disable(:view_diffs_file_by_file>) +``` + ### Merge requests commit navigation > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/18140) in GitLab 13.0. diff --git a/doc/user/project/merge_requests/squash_and_merge.md b/doc/user/project/merge_requests/squash_and_merge.md index 924334055b9..79eec059293 100644 --- a/doc/user/project/merge_requests/squash_and_merge.md +++ b/doc/user/project/merge_requests/squash_and_merge.md @@ -85,6 +85,60 @@ it. This is because squashing is only available when accepting a merge request, so a merge request may need to be rebased before squashing, even though squashing can itself be considered equivalent to rebasing. +## Squash Commits Options + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/17613) in GitLab 13.2. +> - It's deployed behind a feature flag, disabled by default. +> - It's disabled on GitLab.com. +> - It's not recommended for production use. +> - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enable-or-disable-squash-commit-options-core-only). **(CORE ONLY)** + +With Squash Commits Options you can configure the behavior of Squash and Merge for your project. +To set it up, navigate to your project's **Settings > General** and expand **Merge requests**. +You will find the following options to choose, which will affect existing and new merge requests +submitted to your project: + +- **Do not allow**: users cannot use Squash and Merge to squash all the commits immediately before + merging. The checkbox to enable or disable it will be unchecked and hidden from the users. +- **Allow**: users will have the option to enable Squash and Merge on a merge request basis. + The checkbox will be unchecked (disabled) by default, but and the user is allowed to enable it. +- **Encourage**: users will have the option to enable Squash and Merge on a merge request basis. + The checkbox will be checked (enabled) by default to encourage its use, but the user is allowed to + disable it. +- **Require**: Squash and Merge is enabled for all merge requests, so it will always be performed. + The checkbox to enable or disable it will be checked and hidden from the users. + +The Squash and Merge checkbox is displayed when you create a merge request and when you edit the description of an existing one, except when Squash Commit Options is set to **Do not allow** or **Require**. + +NOTE: **Note:** +If your project is set to **Do not allow** Squash and Merge, the users still have the option to +squash commits locally through the command line and force-push to their remote branch before merging. + +### Enable or disable Squash Commit Options **(CORE ONLY)** + +Squash Commit Options is under development and not ready for production use. It is +deployed behind a feature flag that is **disabled by default**. +[GitLab administrators with access to the GitLab Rails console](../../../administration/feature_flags.md) +can enable it for your instance. Squash Commit Options can be enabled or disabled per-project. + +To enable it: + +```ruby +# Instance-wide +Feature.enable(:squash_options) +# or by project +Feature.enable(:squash_options, Project.find(<project id>)) +``` + +To disable it: + +```ruby +# Instance-wide +Feature.disable(:squash_options) +# or by project +Feature.disable(:squash_options, Project.find(<project id>)) +``` + <!-- ## Troubleshooting Include any troubleshooting steps that you can foresee. If you know beforehand what issues diff --git a/doc/user/project/merge_requests/test_coverage_visualization.md b/doc/user/project/merge_requests/test_coverage_visualization.md index 1c0e698aba5..12194423a00 100644 --- a/doc/user/project/merge_requests/test_coverage_visualization.md +++ b/doc/user/project/merge_requests/test_coverage_visualization.md @@ -1,4 +1,7 @@ --- +stage: Verify +group: Testing +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference, howto --- diff --git a/doc/user/project/merge_requests/testing_and_reports_in_merge_requests.md b/doc/user/project/merge_requests/testing_and_reports_in_merge_requests.md index f7614ed7996..e5ebc46d58f 100644 --- a/doc/user/project/merge_requests/testing_and_reports_in_merge_requests.md +++ b/doc/user/project/merge_requests/testing_and_reports_in_merge_requests.md @@ -1,4 +1,7 @@ --- +stage: Verify +group: Testing +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: index description: "Test your code and display reports in merge requests" --- @@ -11,8 +14,9 @@ or link to useful information directly from merge requests: | Feature | Description | |--------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | [Accessibility Testing](accessibility_testing.md) | Automatically report A11y violations for changed pages in merge requests. | -| [Browser Performance Testing](browser_performance_testing.md) **(PREMIUM)** | Quickly determine the performance impact of pending code changes. | -| [Code Quality](code_quality.md) **(STARTER)** | Analyze your source code quality using the [Code Climate](https://codeclimate.com/) analyzer and show the Code Climate report right in the merge request widget area. | +| [Browser Performance Testing](browser_performance_testing.md) **(PREMIUM)** | Quickly determine the browser performance impact of pending code changes. | +| [Load Performance Testing](load_performance_testing.md) **(PREMIUM)** | Quickly determine the server performance impact of pending code changes. | +| [Code Quality](code_quality.md) | Analyze your source code quality using the [Code Climate](https://codeclimate.com/) analyzer and show the Code Climate report right in the merge request widget area. | | [Display arbitrary job artifacts](../../../ci/yaml/README.md#artifactsexpose_as) | Configure CI pipelines with the `artifacts:expose_as` parameter to directly link to selected [artifacts](../../../ci/pipelines/job_artifacts.md) in merge requests. | | [GitLab CI/CD](../../../ci/README.md) | Build, test, and deploy your code in a per-branch basis with built-in CI/CD. | | [JUnit test reports](../../../ci/junit_test_reports.md) | Configure your CI jobs to use JUnit test reports, and let GitLab display a report on the merge request so that it’s easier and faster to identify the failure without having to check the entire job log. | diff --git a/doc/user/project/merge_requests/work_in_progress_merge_requests.md b/doc/user/project/merge_requests/work_in_progress_merge_requests.md index 8ac4131e10b..ece5e7868dc 100644 --- a/doc/user/project/merge_requests/work_in_progress_merge_requests.md +++ b/doc/user/project/merge_requests/work_in_progress_merge_requests.md @@ -2,42 +2,46 @@ type: reference, concepts --- -# "Work In Progress" merge requests +# Draft merge requests If a merge request is not yet ready to be merged, perhaps due to continued development or open threads, you can prevent it from being accepted before it's ready by flagging -it as a **Work In Progress**. This will disable the "Merge" button, preventing it from -being merged, and it will stay disabled until the "WIP" flag has been removed. +it as a **Draft**. This will disable the "Merge" button, preventing it from +being merged, and it will stay disabled until the "Draft" flag has been removed. - + -## Adding the "Work In Progress" flag to a merge request +## Adding the "Draft" flag to a merge request -There are several ways to flag a merge request as a Work In Progress: +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/32692) in GitLab 13.2, Work-In-Progress (WIP) merge requests were renamed to **Draft**. Support for using **WIP** will be removed in GitLab 14.0. -- Add `[WIP]` or `WIP:` to the start of the merge request's title. Clicking on - **Start the title with WIP:**, under the title box, when editing the merge request's +There are several ways to flag a merge request as a Draft: + +- Add `[Draft]`, `Draft:` or `(Draft)` to the start of the merge request's title. Clicking on + **Start the title with Draft:**, under the title box, when editing the merge request's description will have the same effect. +- **Deprecated** Add `[WIP]` or `WIP:` to the start of the merge request's title. + **WIP** still works but was deprecated in favor of **Draft**. It will be removed in the next major version (GitLab 14.0). - Add the `/wip` [quick action](../quick_actions.md#quick-actions-for-issues-merge-requests-and-epics) in a comment in the merge request. This is a toggle, and can be repeated to change the status back. Note that any other text in the comment will be discarded. -- Add "wip" or "WIP" to the start of a commit message targeting the merge request's +- Add `draft:` or `Draft:` to the start of a commit message targeting the merge request's source branch. This is not a toggle, and doing it again in another commit will have no effect. -## Removing the "Work In Progress" flag from a merge request +## Removing the "Draft" flag from a merge request Similar to above, when a Merge Request is ready to be merged, you can remove the -"Work in Progress" flag in several ways: +`Draft` flag in several ways: -- Remove `[WIP]` or `WIP:` from the start of the merge request's title. Clicking on - **Remove the WIP: prefix from the title**, under the title box, when editing the merge +- Remove `[Draft]`, `Draft:` or `(Draft)` from the start of the merge request's title. Clicking on + **Remove the Draft: prefix from the title**, under the title box, when editing the merge request's description, will have the same effect. - Add the `/wip` [quick action](../quick_actions.md#quick-actions-for-issues-merge-requests-and-epics) in a comment in the merge request. This is a toggle, and can be repeated to change the status back. Note that any other text in the comment will be discarded. -- Click on the **Resolve WIP status** button near the bottom of the merge request description, - next to the "Merge" button (see [image above](#work-in-progress-merge-requests)). +- Click on the **Resolve Draft status** button near the bottom of the merge request description, + next to the **Merge** button (see [image above](#draft-merge-requests)). Must have at least Developer level permissions on the project for the button to be visible. diff --git a/doc/user/project/milestones/index.md b/doc/user/project/milestones/index.md index 421cb42de1e..aea5eef5efc 100644 --- a/doc/user/project/milestones/index.md +++ b/doc/user/project/milestones/index.md @@ -29,39 +29,67 @@ Similarly, milestones can be used as releases. To do so: 1. Set the milestone title to the version of your release, such as `Version 9.4`. 1. Add an issue to your release by associating the desired milestone from the issue's right-hand sidebar. -Additionally, you can integrate milestones with GitLab's [Releases feature](../releases/index.md#releases-associated-with-milestones). +Additionally, you can integrate milestones with GitLab's [Releases feature](../releases/index.md#associate-milestones-with-a-release). ## Project milestones and group milestones -- **Project milestones** can be assigned to issues or merge requests in that project only. Navigate to **Issues > Milestones** in a project to view the project milestone list. -- **Group milestones** can be assigned to any issue or merge request of any project in that group. Navigate to **Issues > Milestones** in a group to view the group milestone list. -- All milestones you have access to can also be viewed in the dashboard milestones list. Click on **Milestones** on the top navigation bar to view both project milestones and group milestones you have access to. +You can assign **project milestones** to issues or merge requests in that project only. +To view the project milestone list, in a project, go to **{issues}** **Issues > Milestones**. + +You can assign **group milestones** to any issue or merge request of any project in that group. +To view the group milestone list, in a group, go to **{issues}** **Issues > Milestones**. + +You can also view all milestones you have access to in the dashboard milestones list. +To view both project milestones and group milestones you have access to, click **More > Milestones** +on the top navigation bar. + +For information about project and group milestones API, see: + +- [Project Milestones API](../../../api/milestones.md) +- [Group Milestones API](../../../api/group_milestones.md) + +NOTE: **Note:** +If you're in a group and click **Issues > Milestones**, you'll see group milestones and the milestones +of projects in this group. +If you're in a project and click **Issues > Milestones**, you'll only see this project's milestones. ## Creating milestones ->**Note:** -A permission level of `Developer` or higher is required to create milestones. +NOTE: **Note:** +A permission level of [Developer or higher](../../permissions.md) is required to create milestones. ### New project milestone -To create a **project milestone**, navigate to **Issues > Milestones** in the project. +To create a **project milestone**: -Click the **New milestone** button. Enter the title, an optional description, an optional start date, and an optional due date. Click **Create milestone** to create the milestone. +1. In a project, go to **{issues}** **Issues > Milestones**. +1. Click **New milestone**. +1. Enter the title, an optional description, an optional start date, and an optional due date. +1. Click **New milestone**.  ### New group milestone -To create a **group milestone**, follow similar steps from above to project milestones. Navigate to **Issues > Milestones** in the group and create it from there. +To create a **group milestone**: + +1. In a group, go to **{issues}** **Issues > Milestones**. +1. Click **New milestone**. +1. Enter the title, an optional description, an optional start date, and an optional due date. +1. Click **New milestone**.  ## Editing milestones ->**Note:** -A permission level of `Developer` or higher is required to edit milestones. +NOTE: **Note:** +A permission level of [Developer or higher](../../permissions.md) is required to edit milestones. + +To edit a milestone: -You can update a milestone by navigating to **Issues > Milestones** in the project or group and clicking the **Edit** button. +1. In a project or group, go to **{issues}** **Issues > Milestones**. +1. Click a milestone's title. +1. Click **Edit**. You can delete a milestone by clicking the **Delete** button. diff --git a/doc/user/project/operations/alert_management.md b/doc/user/project/operations/alert_management.md index 411b36411af..3e4b94473bc 100644 --- a/doc/user/project/operations/alert_management.md +++ b/doc/user/project/operations/alert_management.md @@ -17,16 +17,64 @@ being developed, efficiency and awareness can be increased. NOTE: **Note:** You will need at least Maintainer [permissions](../../permissions.md) to enable the Alert Management feature. -1. Follow the [instructions for toggling generic alerts](../integrations/generic_alerts.md#setting-up-generic-alerts) -1. You can now visit **{cloud-gear}** **Operations > Alerts** in your project's sidebar to [view a list](#alert-management-list) of alerts. +There are several ways to accept alerts into your GitLab project, as outlined below. +Enabling any of these methods will allow the Alerts list to display. After configuring +alerts, visit **{cloud-gear}** **Operations > Alerts** in your project's sidebar +to [view the list](#alert-management-list) of alerts. - +### Opsgenie integration **(PREMIUM)** -## Populate Alert data +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3066) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.2. -To populate data, see the instructions for -[customizing the payload](../integrations/generic_alerts.md) and making a -request to the alerts endpoint. +A new way of monitoring Alerts via a GitLab integration is with +[Opsgenie](https://www.atlassian.com/software/opsgenie). + +NOTE: **Note:** +If you enable the Opsgenie integration, you cannot have other GitLab alert services, +such as [Generic Alerts](../integrations/generic_alerts.md) or +Prometheus alerts, active at the same time. + +To enable Opsgenie integration: + +1. Sign in as a user with Maintainer or Owner [permissions](../../permissions.md). +1. Navigate to **{cloud-gear}** **Operations > Alerts**. +1. In the **Integrations** select box, select Opsgenie. +1. Click the **Active** toggle. +1. In the **API URL**, enter the base URL for your Opsgenie integration, such + as `https://app.opsgenie.com/alert/list`. +1. Click **Save changes**. + +After enabling the integration, navigate to the Alerts list page at **{cloud-gear}** **Operations > Alerts**, and click **View alerts in Opsgenie**. + +### Enable a Generic Alerts endpoint + +GitLab provides the Generic Alerts endpoint so you can accept alerts from a third-party +alerts service. See the +[instructions for toggling generic alerts](../integrations/generic_alerts.md#setting-up-generic-alerts) +to add this option. After configuring the endpoint, the +[Alerts list](#alert-management-list) is enabled. + +To populate the alerts with data, see [Customizing the payload](../integrations/generic_alerts.md#customizing-the-payload) for requests to the alerts endpoint. + +### Enable GitLab-managed Prometheus alerts + +You can install the GitLab-managed Prometheus application on your Kubernetes +cluster. For more information, see +[Managed Prometheus on Kubernetes](../integrations/prometheus.md#managed-prometheus-on-kubernetes). +When GitLab-managed Prometheus is installed, the [Alerts list](#alert-management-list) +is also enabled. + +To populate the alerts with data, see +[GitLab-Managed Prometheus instances](../../../operations/metrics/alerts.md#managed-prometheus-instances). + +### Enable external Prometheus alerts + +You can configure an externally-managed Prometheus instance to send alerts +to GitLab. To set up this configuration, see the [configuring Prometheus](../../../operations/metrics/alerts.md#external-prometheus-instances) documentation. Activating the external Prometheus +configuration also enables the [Alerts list](#alert-management-list). + +To populate the alerts with data, see +[External Prometheus instances](../../../operations/metrics/alerts.md#external-prometheus-instances). ## Alert Management severity @@ -55,15 +103,42 @@ You will need at least Developer [permissions](../../permissions.md) to view the You can find the Alert Management list at **{cloud-gear}** **Operations > Alerts** in your project's sidebar. Each alert contains the following metrics: - + - **Severity** - The current importance of a alert and how much attention it should receive. - **Start time** - How long ago the alert fired. This field uses the standard GitLab pattern of `X time ago`, but is supported by a granular date/time tooltip depending on the user's locale. -- **End time** - How long ago the alert fired was resolved. This field uses the standard GitLab pattern of `X time ago`, but is supported by a granular date/time tooltip depending on the user's locale. - **Alert description** - The description of the alert, which attempts to capture the most meaningful data. - **Event count** - The number of times that an alert has fired. +- **Issue** - A link to the incident issue that has been created for the alert. - **Status** - The [current status](#alert-management-statuses) of the alert. +### Alert Management list sorting + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/217745) in GitLab 13.1. + +The Alert Management list displays alerts sorted by start time, but you can +change the sort order by clicking the headers in the Alert Management list. + +To see if a column is sortable, point your mouse at the header. Sortable columns +display an arrow next to the column name, as shown in this example: + + + +### Searching alerts + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/213884) in GitLab 13.1. + +The alert list supports a simple free text search. + + + +This search filters on the following fields: + +- Title +- Description +- Monitoring tool +- Service + ### Alert Management statuses Each alert contains a status dropdown to indicate which alerts need investigation. @@ -138,14 +213,43 @@ deselect the user from the list of assignees, or click **Unassigned**. > [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3066) in GitLab 13.1. -NOTE: **Note:** -GitLab currently only supports creating system notes when assigning an Alert. +When you take action on an alert, this is logged as a system note, +which is visible in the Alert Details view. This gives you a linear +timeline of the alert's investigation and assignment history. -Assigning a user an Alert creates a system note, visible in the Alert Details view, -giving you a linear timeline of the alert's investigation and assignment history. +The following actions will result in a system note: + +- [Updating the status of an alert](#update-an-alerts-status) +- [Creating an issue based on an alert](#create-an-issue-from-an-alert) +- [Assignment of an alert to a user](#update-an-alerts-assignee)  +### View an Alert's metrics data + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/217768) in GitLab 13.2. + +To view the metrics for an alert: + + 1. Sign in as a user with Developer or higher [permissions](../../permissions.md). + 1. Navigate to **{cloud-gear}** **Operations > Alerts**. + 1. Click the alert you want to view. + 1. Below the title of the alert, click the **Metrics** tab. + + + +For GitLab-managed Prometheus instances, metrics data is automatically available +for the alert, making it easy to see surrounding behavior. See +[Managed Prometheus instances](../../../operations/metrics/alerts.md#managed-prometheus-instances) +for information on setting up alerts. + +For externally-managed Prometheus instances, you can configure your alerting rules to +display a chart in the alert. See +[Embedding metrics based on alerts in incident issues](../../../operations/metrics/embed.md#embedding-metrics-based-on-alerts-in-incident-issues) +for information on how to appropriately configure your alerting rules. See +[External Prometheus instances](../../../operations/metrics/alerts.md#external-prometheus-instances) +for information on setting up alerts for your self-managed Prometheus instance. + ## Use cases for assigning alerts Consider a team formed by different sections of monitoring, collaborating on a diff --git a/doc/user/project/operations/dashboard_settings.md b/doc/user/project/operations/dashboard_settings.md index e14c478ab7a..f30ce5024d6 100644 --- a/doc/user/project/operations/dashboard_settings.md +++ b/doc/user/project/operations/dashboard_settings.md @@ -9,6 +9,9 @@ info: To determine the technical writer assigned to the Stage/Group associated w You can configure your [Monitoring dashboard](../integrations/prometheus.md) to display the time zone of your choice, and the links of your choice. +To configure these settings you must have Manage Project +Operations [permissions](../../permissions.md). + ## Change the dashboard time zone > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/214370) in GitLab 13.1. @@ -17,6 +20,7 @@ By default, your monitoring dashboard displays dates and times in your local time zone, but you can display dates and times in UTC format. To change the time zone: +1. Sign in as a user with Manage Project Operations [permissions](../../permissions.md). 1. Navigate to **{settings}** **Settings > Operations**, and scroll to **Metrics Dashboard**. 1. In the **Dashboard timezone** select box, select *User's local timezone* @@ -32,6 +36,7 @@ time zone: You can add a button on your monitoring dashboard that links directly to your existing external dashboards: +1. Sign in as a user with Manage Project Operations [permissions](../../permissions.md). 1. Navigate to **{settings}** **Settings > Operations**, and scroll to **Metrics Dashboard**. 1. In **External dashboard URL**, provide the URL to your external dashboard: diff --git a/doc/user/project/operations/feature_flags.md b/doc/user/project/operations/feature_flags.md index a9729204cd7..b0f7cfc966a 100644 --- a/doc/user/project/operations/feature_flags.md +++ b/doc/user/project/operations/feature_flags.md @@ -1,261 +1,5 @@ --- -stage: Release -group: Progressive Delivery -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +redirect_to: '../../../operations/feature_flags.md' --- -# Feature Flags **(PREMIUM)** - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/7433) in GitLab 11.4. - -With Feature Flags, you can deploy your application's new features to production in smaller batches. -You can toggle a feature on and off to subsets of users, helping you achieve Continuous Delivery. -Feature flags help reduce risk, allowing you to do controlled testing, and separate feature -delivery from customer launch. - -<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> -For an example of feature flags in action, see [GitLab for Deploys, Feature Flags, and Error Tracking](https://www.youtube.com/embed/5tw2p6lwXxo). - -## How it works - -GitLab uses [Unleash](https://github.com/Unleash/unleash), a feature -toggle service. - -By enabling or disabling a flag in GitLab, your application -can determine which features to enable or disable. - -You can create feature flags in GitLab and use the API from your application -to get the list of feature flags and their statuses. The application must be configured to communicate -with GitLab, so it's up to developers to use a compatible client library and -[integrate the feature flags in your app](#integrate-feature-flags-with-your-application). - -## Create a feature flag - -To create and enable a feature flag: - -1. Navigate to your project's **Operations > Feature Flags**. -1. Click the **New feature flag** button. -1. Enter a name that starts with a letter and contains only lowercase letters, digits, underscores (`_`) - and dashes (`-`), and does not end with a dash (`-`) or underscore (`_`). -1. Enter a description (optional, 255 characters max). -1. Enter details about how the flag should be applied: - - In GitLab 13.0 and earlier: Enter an environment spec, - enable or disable the flag in this environment, and select a rollout strategy. - - In GitLab 13.1 and later (when [this feature flag](#feature-flag-behavior-change-in-130) is enabled): Select a strategy and then - the environments to apply the strategy to. -1. Click **Create feature flag**. - -The feature flag is displayed in the list. It is enabled by default. - -## Disable a feature flag for a specific environment - -In [GitLab 13.0 and earlier](https://gitlab.com/gitlab-org/gitlab/-/issues/8621), -to disable a feature flag for a specific environment: - -1. Navigate to your project's **Operations > Feature Flags**. -1. For the feature flag you want to disable, click the Pencil icon. -1. To disable the flag: - - In GitLab 13.0 and earlier: Slide the Status toggle for the environment. Or, to delete the - environment spec, on the right, click the **Remove (X)** icon. - - In GitLab 13.1 and later (when [this feature flag](#feature-flag-behavior-change-in-130) is - enabled): For each strategy it applies to, under **Environments**, delete the environment. -1. Click **Save changes**. - -## Disable a feature flag for all environments - -To disable a feature flag for all environments: - -1. Navigate to your project's **Operations > Feature Flags**. -1. For the feature flag you want to disable, slide the Status toggle to **Disabled**. - -The feature flag is displayed on the **Disabled** tab. - -## Feature flag behavior change in 13.0 - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/35555) in GitLab 13.0. - -Starting in GitLab 13.0, you can apply a feature flag strategy across multiple environments, -without defining the strategy multiple times. - -This feature is under development and not ready for production use. It is -deployed behind a feature flag that is **disabled by default**. -[GitLab administrators with access to the GitLab Rails console](../../../administration/feature_flags.md) -can enable it for your instance. - -To enable it: - -```ruby -Feature.enable(:feature_flags_new_version) -``` - -To disable it: - -```ruby -Feature.disable(:feature_flags_new_version) -``` - -## Feature flag strategies - -GitLab Feature Flag uses [Unleash](https://unleash.github.io) -as the feature flag engine. In Unleash, there is a concept of rulesets for granular feature flag controls, -called [strategies](https://unleash.github.io/docs/activation_strategy). -Supported strategies for GitLab Feature Flags are described below. - -### Rollout strategy - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/8240) in GitLab 12.2. - -The selected rollout strategy affects which users will experience the feature as enabled. - -The status of an environment spec ultimately determines whether or not a feature is enabled at all. -For instance, a feature will always be disabled for every user if the matching environment spec has a disabled status, regardless of the chosen rollout strategy. -However, a feature will be enabled for 50% of logged-in users if the matching environment spec has an enabled status along with a **Percent rollout (logged in users)** strategy set to 50%. - -#### All users - -Enables the feature for all users. It is implemented using the Unleash -[`default`](https://unleash.github.io/docs/activation_strategy#default) -activation strategy. - -#### Percent rollout (logged in users) - -Enables the feature for a percentage of authenticated users. It is -implemented using the Unleash -[`gradualRolloutUserId`](https://unleash.github.io/docs/activation_strategy#gradualrolloutuserid) -activation strategy. - -Set a value of 15%, for example, to enable the feature for 15% of authenticated users. - -A rollout percentage may be between 0% and 100%. - -CAUTION: **Caution:** -If this strategy is selected, then the Unleash client **must** be given a user -ID for the feature to be enabled. See the [Ruby example](#ruby-application-example) below. - -#### User IDs - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/8240) in GitLab 12.2. [Updated](https://gitlab.com/gitlab-org/gitlab/-/issues/34363) to be defined per environment in GitLab 12.6. - -A feature flag may be enabled for a list of target users. It is implemented -using the Unleash [`userWithId`](https://unleash.github.io/docs/activation_strategy#userwithid) -activation strategy. - -User IDs should be a comma-separated list of values. For example, `user@example.com, user2@example.com`, or `username1,username2,username3`, etc. - -CAUTION: **Caution:** -The Unleash client **must** be given a user ID for the feature to be enabled for -target users. See the [Ruby example](#ruby-application-example) below. - -## Integrate feature flags with your application - -To use feature flags with your application, get access credentials from GitLab. -Then prepare your application with a client library. - -### Get access credentials - -To get the access credentials that your application needs to communicate with GitLab: - -1. Navigate to your project's **Operations > Feature Flags**. -1. Click the **Configure** button to view the following: - - **API URL**: URL where the client (application) connects to get a list of feature flags. - - **Instance ID**: Unique token that authorizes the retrieval of the feature flags. - - **Application name**: The name of the running environment. For instance, - if the application runs for a production server, the application name would be - `production` or similar. This value is used for the environment spec evaluation. - -NOTE: **Note:** -The meaning of these fields might change over time. For example, we are not sure -if **Instance ID** will be single token or multiple tokens, assigned to the -**Environment**. Also, **Application name** could describe the version of -application instead of the running environment. - -### Choose a client library - -GitLab implements a single backend that is compatible with Unleash clients. - -With the Unleash client, developers can define, in the application code, the default values for flags. -Each feature flag evaluation can express the desired outcome if the flag isn't present in the -provided configuration file. - -Unleash currently [offers many SDKs for various languages and frameworks](https://github.com/Unleash/unleash#client-implementations). - -### Feature flags API information - -For API content, see: - -- [Feature Flags API](../../../api/feature_flags.md) -- [Feature Flag Specs API](../../../api/feature_flag_specs.md) (Deprecated and [scheduled for removal in GitLab 14.0](https://gitlab.com/gitlab-org/gitlab/-/issues/213369).) -- [Feature Flag User Lists API](../../../api/feature_flag_user_lists.md) -- [Legacy Feature Flags API](../../../api/feature_flags_legacy.md) - -### Golang application example - -Here's an example of how to integrate feature flags in a Golang application: - -```golang -package main - -import ( - "io" - "log" - "net/http" - - "github.com/Unleash/unleash-client-go" -) - -type metricsInterface struct { -} - -func init() { - unleash.Initialize( - unleash.WithUrl("https://gitlab.com/api/v4/feature_flags/unleash/42"), - unleash.WithInstanceId("29QmjsW6KngPR5JNPMWx"), - unleash.WithAppName("production"), - unleash.WithListener(&metricsInterface{}), - ) -} - -func helloServer(w http.ResponseWriter, req *http.Request) { - if unleash.IsEnabled("my_feature_name") { - io.WriteString(w, "Feature enabled\n") - } else { - io.WriteString(w, "hello, world!\n") - } -} - -func main() { - http.HandleFunc("/", helloServer) - log.Fatal(http.ListenAndServe(":8080", nil)) -} -``` - -### Ruby application example - -Here's an example of how to integrate feature flags in a Ruby application. - -The Unleash client is given a user ID for use with a **Percent rollout (logged in users)** rollout strategy or a list of **Target Users**. - -```ruby -#!/usr/bin/env ruby - -require 'unleash' -require 'unleash/context' - -unleash = Unleash::Client.new({ - url: 'http://gitlab.com/api/v4/feature_flags/unleash/42', - app_name: 'production', - instance_id: '29QmjsW6KngPR5JNPMWx' -}) - -unleash_context = Unleash::Context.new -# Replace "123" with the id of an authenticated user. -# Note that the context's user id must be a string: -# https://unleash.github.io/docs/unleash_context -unleash_context.user_id = "123" - -if unleash.is_enabled?("my_feature_name", unleash_context) - puts "Feature enabled" -else - puts "hello, world!" -end -``` +This document was moved to [another location](../../../operations/feature_flags.md). diff --git a/doc/user/project/operations/img/alert_detail_metrics_v13_2.png b/doc/user/project/operations/img/alert_detail_metrics_v13_2.png Binary files differnew file mode 100644 index 00000000000..84d83365ea8 --- /dev/null +++ b/doc/user/project/operations/img/alert_detail_metrics_v13_2.png diff --git a/doc/user/project/operations/img/alert_list_search_v13_1.png b/doc/user/project/operations/img/alert_list_search_v13_1.png Binary files differnew file mode 100644 index 00000000000..ba993fe530b --- /dev/null +++ b/doc/user/project/operations/img/alert_list_search_v13_1.png diff --git a/doc/user/project/operations/img/alert_list_sort_v13_1.png b/doc/user/project/operations/img/alert_list_sort_v13_1.png Binary files differnew file mode 100644 index 00000000000..8e06c3478f7 --- /dev/null +++ b/doc/user/project/operations/img/alert_list_sort_v13_1.png diff --git a/doc/user/project/operations/img/alert_list_v13_1.png b/doc/user/project/operations/img/alert_list_v13_1.png Binary files differnew file mode 100644 index 00000000000..7a1a5f5191e --- /dev/null +++ b/doc/user/project/operations/img/alert_list_v13_1.png diff --git a/doc/user/project/operations/img/alert_management_1_v13_0.png b/doc/user/project/operations/img/alert_management_1_v13_0.png Binary files differdeleted file mode 100644 index dbc1e795b16..00000000000 --- a/doc/user/project/operations/img/alert_management_1_v13_0.png +++ /dev/null diff --git a/doc/user/project/operations/img/alert_management_1_v13_1.png b/doc/user/project/operations/img/alert_management_1_v13_1.png Binary files differdeleted file mode 100644 index 00aa56a6050..00000000000 --- a/doc/user/project/operations/img/alert_management_1_v13_1.png +++ /dev/null diff --git a/doc/user/project/operations/index.md b/doc/user/project/operations/index.md index ca38eab9455..9cec578bc5e 100644 --- a/doc/user/project/operations/index.md +++ b/doc/user/project/operations/index.md @@ -1,13 +1,5 @@ -# Project operations +--- +redirect_to: '../../../operations/index.md' +--- -GitLab provides a variety of tools to help operate and maintain -your applications: - -- Collect [Prometheus metrics](../integrations/prometheus_library/index.md). -- Deploy to different [environments](../../../ci/environments/index.md). -- Connect your project to a [Kubernetes cluster](../clusters/index.md). -- Manage your infrastructure with [Infrastructure as Code](../../infrastructure/index.md) approaches. -- Discover and view errors generated by your applications with [Error Tracking](error_tracking.md). -- Create, toggle, and remove [Feature Flags](feature_flags.md). **(PREMIUM)** -- [Trace](tracing.md) the performance and health of a deployed application. **(ULTIMATE)** -- Change the [settings of the Monitoring Dashboard](dashboard_settings.md). +This document was moved to [another location](../../../operations/index.md). diff --git a/doc/user/project/operations/tracing.md b/doc/user/project/operations/tracing.md index 07f60c37f1b..87567f45560 100644 --- a/doc/user/project/operations/tracing.md +++ b/doc/user/project/operations/tracing.md @@ -1,40 +1,5 @@ --- -stage: Monitor -group: APM -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +redirect_to: '../../../operations/tracing.md' --- -# Tracing **(ULTIMATE)** - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/7903) in GitLab Ultimate 11.5. - -Tracing provides insight into the performance and health of a deployed application, -tracking each function or microservice which handles a given request. - -This makes it easy to -understand the end-to-end flow of a request, regardless of whether you are using a monolithic or distributed system. - -## Jaeger tracing - -[Jaeger](https://www.jaegertracing.io/) is an open source, end-to-end distributed -tracing system used for monitoring and troubleshooting microservices-based distributed -systems. - -### Deploying Jaeger - -To learn more about deploying Jaeger, read the official -[Getting Started documentation](https://www.jaegertracing.io/docs/latest/getting-started/). -There is an easy to use [all-in-one Docker image](https://www.jaegertracing.io/docs/latest/getting-started/#AllinoneDockerimage), -as well as deployment options for [Kubernetes](https://github.com/jaegertracing/jaeger-kubernetes) -and [OpenShift](https://github.com/jaegertracing/jaeger-openshift). - -### Enabling Jaeger - -GitLab provides an easy way to open the Jaeger UI from within your project: - -1. [Set up Jaeger](https://www.jaegertracing.io) and configure your application using one of the - [client libraries](https://www.jaegertracing.io/docs/latest/client-libraries/). -1. Navigate to your project's **Settings > Operations** and provide the Jaeger URL. -1. Click **Save changes** for the changes to take effect. -1. You can now visit **Operations > Tracing** in your project's sidebar and - GitLab will redirect you to the configured Jaeger URL. +This document was moved to [another location](../../../operations/tracing.md). diff --git a/doc/user/project/pages/custom_domains_ssl_tls_certification/index.md b/doc/user/project/pages/custom_domains_ssl_tls_certification/index.md index bf9b58acf14..0425ca56285 100644 --- a/doc/user/project/pages/custom_domains_ssl_tls_certification/index.md +++ b/doc/user/project/pages/custom_domains_ssl_tls_certification/index.md @@ -287,7 +287,7 @@ To enable this setting: 1. Navigate to your project's **Settings > Pages**. 1. Tick the checkbox **Force HTTPS (requires valid certificates)**. -NOTE: **Note** +NOTE: **Note:** If you use CloudFlare CDN in front of GitLab Pages, make sure to set the SSL connection setting to `full` instead of `flexible`. For more details, see the [CloudFlare CDN directions](https://support.cloudflare.com/hc/en-us/articles/200170416-End-to-end-HTTPS-with-Cloudflare-Part-3-SSL-options#h_4e0d1a7c-eb71-4204-9e22-9d3ef9ef7fef). <!-- ## Troubleshooting diff --git a/doc/user/project/pages/getting_started/fork_sample_project.md b/doc/user/project/pages/getting_started/fork_sample_project.md index de9bd97b262..250e90f0302 100644 --- a/doc/user/project/pages/getting_started/fork_sample_project.md +++ b/doc/user/project/pages/getting_started/fork_sample_project.md @@ -1,56 +1,5 @@ --- -type: reference, howto -stage: Release -group: Release Management -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +redirect_to: 'pages_forked_sample_project.md' --- -# Create a Pages website from a forked sample - -GitLab provides [sample projects for the most popular Static Site Generators](https://gitlab.com/pages). -You can fork one of the sample projects and run the CI/CD pipeline to generate a Pages website. - -Fork a sample project when you want to test GitLab Pages or start a new project that's already -configured to generate a Pages site. - -<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> Watch a [video tutorial](https://www.youtube.com/watch?v=TWqh9MtT4Bg) of how this works. - -To fork a sample project and create a Pages website: - -1. View the sample projects by going to the [GitLab Pages examples](https://gitlab.com/pages) group. -1. Click the name of the project you want to [fork](../../../../gitlab-basics/fork-project.md). -1. In the top right, click the **Fork** button, and then choose a namespace to fork to. -1. Go to your project's **CI/CD > Pipelines** and click **Run pipeline**. - GitLab CI/CD builds and deploys your site. - -The site can take approximately 30 minutes to deploy. -When the pipeline is finished, go to **Settings > Pages** to find the link to your website from your project. - -For every change pushed to your repository, GitLab CI/CD runs a new pipeline -that immediately publishes your changes to the Pages site. - -You can take some **optional** further steps: - -- _Remove the fork relationship._ If you want to contribute to the project you forked from, - you can keep this relationship. Otherwise, go to your project's **Settings > General**, - expand **Advanced settings**, and scroll down to **Remove fork relationship**: - -  - -- _Change the URL to match your namespace._ If your Pages site is hosted on GitLab.com, - you can rename it to `<namespace>.gitlab.io`, where `<namespace>` is your GitLab namespace - (the one you chose when you forked the project). - - - Go to your project's **Settings > General** and expand **Advanced**. Scroll down to - **Change path** and change the path to `<namespace>.gitlab.io`. - - For example, if your project's URL is `gitlab.com/gitlab-tests/jekyll`, your namespace is - `gitlab-tests`. - - If you set the repository path to `gitlab-tests.gitlab.io`, - the resulting URL for your Pages website is `https://gitlab-tests.gitlab.io`. - -  - - - Now go to your SSG's configuration file and change the [base URL](../getting_started_part_one.md#urls-and-baseurls) - from `"project-name"` to `""`. The project name setting varies by SSG and may not be in the config file. +This document was moved to [pages_forked_sample_project.md](pages_forked_sample_project.md). diff --git a/doc/user/project/pages/getting_started/new_or_existing_website.md b/doc/user/project/pages/getting_started/new_or_existing_website.md index 5d7126ab22e..86f36447b93 100644 --- a/doc/user/project/pages/getting_started/new_or_existing_website.md +++ b/doc/user/project/pages/getting_started/new_or_existing_website.md @@ -1,49 +1,5 @@ --- -type: reference, howto -stage: Release -group: Release Management -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +redirect_to: 'pages_ci_cd_template.md' --- -# Create a Pages website by using a CI/CD template - -GitLab provides `.gitlab-ci.yml` templates for the most popular Static Site Generators (SSGs). -You can create your own `.gitlab-ci.yml` file from one of these templates, and run -the CI/CD pipeline to generate a Pages website. - -Use a `.gitlab-ci.yml` template when you have an existing project that you want to add a Pages site to. - -Your GitLab repository should contain files specific to an SSG, or plain HTML. -After you complete these steps, you may need to do additional -configuration for the Pages site to generate properly. - -1. In the left sidebar, click **Project overview**. -1. Click **Set up CI/CD**. - -  - - If this button is not available, CI/CD is already configured for - your project. You may want to browse the `.gitlab-ci.yml` files - [in these projects instead](https://gitlab.com/pages). - -1. From the **Apply a template** list, choose a template for the SSG you're using. - You can also choose plain HTML. - -  - - If you don't find a corresponding template, you can view the - [GitLab Pages group of sample projects](https://gitlab.com/pages). - These projects contain `.gitlab-ci.yml` files that you can modify for your needs. - You can also [learn how to write your own `.gitlab-ci.yml` - file for GitLab Pages](../getting_started_part_four.md). - -1. Save and commit the `.gitlab-ci.yml` file. - -If everything is configured correctly, the site can take approximately 30 minutes to deploy. - -You can watch the pipeline run by going to **CI / CD > Pipelines**. -When the pipeline is finished, go to **Settings > Pages** to find the link to -your Pages website. - -For every change pushed to your repository, GitLab CI/CD runs a new pipeline -that immediately publishes your changes to the Pages site. +This document was moved to [pages_ci_cd_template.md](pages_ci_cd_template.md). diff --git a/doc/user/project/pages/getting_started/pages_bundled_template.md b/doc/user/project/pages/getting_started/pages_bundled_template.md index cfa4e0910db..bc706105606 100644 --- a/doc/user/project/pages/getting_started/pages_bundled_template.md +++ b/doc/user/project/pages/getting_started/pages_bundled_template.md @@ -1,34 +1,5 @@ --- -type: reference, howto -stage: Release -group: Release Management -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +redirect_to: 'pages_new_project_template.md' --- -# Create a Pages website from a new project template - -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/47857) in GitLab 11.8. - -GitLab provides templates for the most popular Static Site Generators (SSGs). -You can create a new project from a template and run the CI/CD pipeline to generate a Pages website. - -Use a template when you want to test GitLab Pages or start a new project that's already -configured to generate a Pages site. - -1. From the top navigation, click the **+** button and select **New project**. -1. Select **Create from Template**. -1. Next to one of the templates starting with **Pages**, click **Use template**. - -  - -1. Complete the form and click **Create project**. -1. From the left sidebar, navigate to your project's **CI/CD > Pipelines** - and click **Run pipeline** to trigger GitLab CI/CD to build and deploy your - site. - -The site can take approximately 30 minutes to deploy. -When the pipeline is finished, go to **Settings > Pages** to find the link to -your Pages website. - -For every change pushed to your repository, GitLab CI/CD runs a new pipeline -that immediately publishes your changes to the Pages site. +This document was moved to [pages_new_project_template.md](pages_new_project_template.md). diff --git a/doc/user/project/pages/getting_started/pages_ci_cd_template.md b/doc/user/project/pages/getting_started/pages_ci_cd_template.md new file mode 100644 index 00000000000..906ffe43285 --- /dev/null +++ b/doc/user/project/pages/getting_started/pages_ci_cd_template.md @@ -0,0 +1,49 @@ +--- +type: reference, howto +stage: Release +group: Release Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Create a Pages website by using a CI/CD template + +GitLab provides `.gitlab-ci.yml` templates for the most popular Static Site Generators (SSGs). +You can create your own `.gitlab-ci.yml` file from one of these templates, and run +the CI/CD pipeline to generate a Pages website. + +Use a `.gitlab-ci.yml` template when you have an existing project that you want to add a Pages site to. + +Your GitLab repository should contain files specific to an SSG, or plain HTML. +After you complete these steps, you may need to do additional +configuration for the Pages site to generate properly. + +1. In the left sidebar, click **Project overview**. +1. Click **Set up CI/CD**. + +  + + If this button is not available, CI/CD is already configured for + your project. You may want to browse the `.gitlab-ci.yml` files + [in these projects instead](https://gitlab.com/pages). + +1. From the **Apply a template** list, choose a template for the SSG you're using. + You can also choose plain HTML. + +  + + If you don't find a corresponding template, you can view the + [GitLab Pages group of sample projects](https://gitlab.com/pages). + These projects contain `.gitlab-ci.yml` files that you can modify for your needs. + You can also [learn how to write your own `.gitlab-ci.yml` + file for GitLab Pages](pages_from_scratch.md). + +1. Save and commit the `.gitlab-ci.yml` file. + +If everything is configured correctly, the site can take approximately 30 minutes to deploy. + +You can watch the pipeline run by going to **CI / CD > Pipelines**. +When the pipeline is finished, go to **Settings > Pages** to find the link to +your Pages website. + +For every change pushed to your repository, GitLab CI/CD runs a new pipeline +that immediately publishes your changes to the Pages site. diff --git a/doc/user/project/pages/getting_started/pages_forked_sample_project.md b/doc/user/project/pages/getting_started/pages_forked_sample_project.md new file mode 100644 index 00000000000..de9bd97b262 --- /dev/null +++ b/doc/user/project/pages/getting_started/pages_forked_sample_project.md @@ -0,0 +1,56 @@ +--- +type: reference, howto +stage: Release +group: Release Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Create a Pages website from a forked sample + +GitLab provides [sample projects for the most popular Static Site Generators](https://gitlab.com/pages). +You can fork one of the sample projects and run the CI/CD pipeline to generate a Pages website. + +Fork a sample project when you want to test GitLab Pages or start a new project that's already +configured to generate a Pages site. + +<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> Watch a [video tutorial](https://www.youtube.com/watch?v=TWqh9MtT4Bg) of how this works. + +To fork a sample project and create a Pages website: + +1. View the sample projects by going to the [GitLab Pages examples](https://gitlab.com/pages) group. +1. Click the name of the project you want to [fork](../../../../gitlab-basics/fork-project.md). +1. In the top right, click the **Fork** button, and then choose a namespace to fork to. +1. Go to your project's **CI/CD > Pipelines** and click **Run pipeline**. + GitLab CI/CD builds and deploys your site. + +The site can take approximately 30 minutes to deploy. +When the pipeline is finished, go to **Settings > Pages** to find the link to your website from your project. + +For every change pushed to your repository, GitLab CI/CD runs a new pipeline +that immediately publishes your changes to the Pages site. + +You can take some **optional** further steps: + +- _Remove the fork relationship._ If you want to contribute to the project you forked from, + you can keep this relationship. Otherwise, go to your project's **Settings > General**, + expand **Advanced settings**, and scroll down to **Remove fork relationship**: + +  + +- _Change the URL to match your namespace._ If your Pages site is hosted on GitLab.com, + you can rename it to `<namespace>.gitlab.io`, where `<namespace>` is your GitLab namespace + (the one you chose when you forked the project). + + - Go to your project's **Settings > General** and expand **Advanced**. Scroll down to + **Change path** and change the path to `<namespace>.gitlab.io`. + + For example, if your project's URL is `gitlab.com/gitlab-tests/jekyll`, your namespace is + `gitlab-tests`. + + If you set the repository path to `gitlab-tests.gitlab.io`, + the resulting URL for your Pages website is `https://gitlab-tests.gitlab.io`. + +  + + - Now go to your SSG's configuration file and change the [base URL](../getting_started_part_one.md#urls-and-baseurls) + from `"project-name"` to `""`. The project name setting varies by SSG and may not be in the config file. diff --git a/doc/user/project/pages/getting_started/pages_from_scratch.md b/doc/user/project/pages/getting_started/pages_from_scratch.md new file mode 100644 index 00000000000..7278c734b07 --- /dev/null +++ b/doc/user/project/pages/getting_started/pages_from_scratch.md @@ -0,0 +1,402 @@ +--- +last_updated: 2020-01-06 +type: reference, howto +stage: Release +group: Release Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Create a GitLab Pages website from scratch + +This tutorial shows you how to create a Pages site from scratch. You will start with +a blank project and create your own CI file, which gives instruction to the +[GitLab Runner](https://docs.gitlab.com/runner/). When your CI/CD +[pipeline](../../../../ci/pipelines/index.md) runs, the Pages site is created. + +This example uses the [Jekyll](https://jekyllrb.com/) Static Site Generator (SSG). +Other SSGs follow similar steps. You do not need to be familiar with Jekyll or SSGs +to complete this tutorial. + +## Prerequisites + +To follow along with this example, start with a blank project in GitLab. +Create three files in the root (top-level) directory. + +- `.gitlab-ci.yml` A YAML file that contains the commands you want to run. + For now, leave the file's contents blank. + +- `index.html` An HTML file you can populate with whatever HTML content you'd like, + for example: + + ```html + <html> + <head> + <title>Home</title> + </head> + <body> + <h1>Hello World!</h1> + </body> + </html> + ``` + +- [`Gemfile`](https://bundler.io/gemfile.html) A file that describes dependencies for Ruby programs. + Populate it with this content: + + ```ruby + source "https://rubygems.org" + + gem "jekyll" + ``` + +## Choose a Docker image + +In this example, the Runner uses a [Docker image](../../../../ci/docker/using_docker_images.md) +to run scripts and deploy the site. + +This specific Ruby image is maintained on [DockerHub](https://hub.docker.com/_/ruby). + +Edit your `.gitlab-ci.yml` and add this text as the first line. + +```yaml +image: ruby:2.7 +``` + +If your SSG needs [NodeJS](https://nodejs.org/) to build, you must specify an +image that contains NodeJS as part of its file system. For example, for a +[Hexo](https://gitlab.com/pages/hexo) site, you can use `image: node:12.17.0`. + +## Install Jekyll + +To run [Jekyll](https://jekyllrb.com/) locally, you would open your terminal and: + +- Install [Bundler](https://bundler.io/) by running `gem install bundler`. +- Create `Gemfile.lock` by running `bundle install`. +- Install Jekyll by running `bundle exec jekyll build`. + +In the `.gitlab-ci.yml` file, this is written as: + +```yaml +script: + - gem install bundler + - bundle install + - bundle exec jekyll build +``` + +In addition, in the `.gitlab-ci.yml` file, each `script` is organized by a `job`. +A `job` includes the scripts and settings you want to apply to that specific +task. + +```yaml +job: + script: + - gem install bundler + - bundle install + - bundle exec jekyll build +``` + +For GitLab Pages, this `job` has a specific name, called `pages`. +This setting tells the Runner you want the job to deploy your website +with GitLab Pages: + +```yaml +pages: + script: + - gem install bundler + - bundle install + - bundle exec jekyll build +``` + +## Specify the `public` directory for output + +Jekyll needs to know where to generate its output. +GitLab Pages only considers files in a directory called `public`. + +Jekyll uses destination (`-d`) to specify an output directory for the built website: + +```yaml +pages: + script: + - gem install bundler + - bundle install + - bundle exec jekyll build -d public +``` + +## Specify the `public` directory for artifacts + +Now that Jekyll has output the files to the `public` directory, +the Runner needs to know where to get them. The artifacts are stored +in the `public` directory: + +```yaml +pages: + script: + - gem install bundler + - bundle install + - bundle exec jekyll build -d public + artifacts: + paths: + - public +``` + +Paste this into `.gitlab-ci.yml` file, so it now looks like this: + +```yaml +image: ruby:2.7 + +pages: + script: + - gem install bundler + - bundle install + - bundle exec jekyll build -d public + artifacts: + paths: + - public +``` + +Now save and commit the `.gitlab-ci.yml` file. You can watch the pipeline run +by going to **CI / CD > Pipelines**. + +When it succeeds, go to **Settings > Pages** to view the URL where your site +is now available. + +If you want to do more advanced tasks, you can update your `.gitlab-ci.yml` file +with [any of the available settings](../../../../ci/yaml/README.md). You can check +your CI syntax with the [GitLab CI/CD Lint Tool](../../../../ci/yaml/README.md#validate-the-gitlab-ciyml). + +The following topics show other examples of other options you can add to your CI/CD file. + +## Deploy specific branches to a Pages site + +You may want to deploy to a Pages site only from specific branches. + +First, add a `workflow` section to force the pipeline to run only when changes are +pushed to branches: + +```yaml +image: ruby:2.7 + +workflow: + rules: + - if: '$CI_COMMIT_BRANCH' + +pages: + script: + - gem install bundler + - bundle install + - bundle exec jekyll build -d public + artifacts: + paths: + - public +``` + +Then configure the pipeline to run the job for the master branch only. + +```yaml +image: ruby:2.7 + +workflow: + rules: + - if: '$CI_COMMIT_BRANCH' + +pages: + script: + - gem install bundler + - bundle install + - bundle exec jekyll build -d public + artifacts: + paths: + - public + rules: + - if: '$CI_COMMIT_BRANCH == "master"' +``` + +## Specify a stage to deploy + +There are three default stages for GitLab CI/CD: build, test, +and deploy. + +If you want to test your script and check the built site before deploying +to production, you can run the test exactly as it will run when you +push to `master`. + +To specify a stage for your job to run in, +add a `stage` line to your CI file: + +```yaml +image: ruby:2.7 + +workflow: + rules: + - if: '$CI_COMMIT_BRANCH' + +pages: + stage: deploy + script: + - gem install bundler + - bundle install + - bundle exec jekyll build -d public + artifacts: + paths: + - public + rules: + - if: '$CI_COMMIT_BRANCH == "master"' +``` + +Now add another job to the CI file, telling it to +test every push to every branch **except** the `master` branch: + +```yaml +image: ruby:2.7 + +workflow: + rules: + - if: '$CI_COMMIT_BRANCH' + +pages: + stage: deploy + script: + - gem install bundler + - bundle install + - bundle exec jekyll build -d public + artifacts: + paths: + - public + rules: + - if: '$CI_COMMIT_BRANCH == "master"' + +test: + stage: test + script: + - gem install bundler + - bundle install + - bundle exec jekyll build -d test + artifacts: + paths: + - test + rules: + - if: '$CI_COMMIT_BRANCH != "master"' +``` + +When the `test` job runs in the `test` stage, Jekyll +builds the site in a directory called `test`. The job affects +all branches except `master`. + +When you apply stages to different jobs, every job in the same +stage builds in parallel. If your web application needs more than +one test before being deployed, you can run all your tests at the +same time. + +## Remove duplicate commands + +To avoid duplicating the same scripts in every job, you can add them +to a `before_script` section. + +In the example, `gem install bundler` and `bundle install` were running +for both jobs, `pages` and `test`. + +Move these commands to a `before_script` section: + +```yaml +image: ruby:2.7 + +workflow: + rules: + - if: '$CI_COMMIT_BRANCH' + +before_script: + - gem install bundler + - bundle install + +pages: + stage: deploy + script: + - bundle exec jekyll build -d public + artifacts: + paths: + - public + rules: + - if: '$CI_COMMIT_BRANCH == "master"' + +test: + stage: test + script: + - bundle exec jekyll build -d test + artifacts: + paths: + - test + rules: + - if: '$CI_COMMIT_BRANCH != "master"' +``` + +## Build faster with cached dependencies + +To build faster, you can cache the installation files for your +project's dependencies by using the `cache` parameter. + +This example caches Jekyll dependencies in a `vendor` directory +when you run `bundle install`: + +```yaml +image: ruby:2.7 + +workflow: + rules: + - if: '$CI_COMMIT_BRANCH' + +cache: + paths: + - vendor/ + +before_script: + - gem install bundler + - bundle install --path vendor + +pages: + stage: deploy + script: + - bundle exec jekyll build -d public + artifacts: + paths: + - public + rules: + - if: '$CI_COMMIT_BRANCH == "master"' + +test: + stage: test + script: + - bundle exec jekyll build -d test + artifacts: + paths: + - test + rules: + - if: '$CI_COMMIT_BRANCH != "master"' +``` + +In this case, you need to exclude the `/vendor` +directory from the list of folders Jekyll builds. Otherwise, Jekyll +will try to build the directory contents along with the site. + +In the root directory, create a file called `_config.yml` +and add this content: + +```yaml +exclude: + - vendor +``` + +Now GitLab CI/CD not only builds the website, +but also pushes with **continuous tests** to feature-branches, +**caches** dependencies installed with Bundler, and +**continuously deploys** every push to the `master` branch. + +## Related topics + +For more information, see the following blog posts. + +- [Use GitLab CI/CD `environments` to deploy your + web app to staging and production](https://about.gitlab.com/blog/2016/08/26/ci-deployment-and-environments/). +- Learn [how to run jobs sequentially, + in parallel, or build a custom pipeline](https://about.gitlab.com/blog/2016/07/29/the-basics-of-gitlab-ci/). +- Learn [how to pull specific directories from different projects](https://about.gitlab.com/blog/2016/12/07/building-a-new-gitlab-docs-site-with-nanoc-gitlab-ci-and-gitlab-pages/) + to deploy this website, <https://docs.gitlab.com>. +- Learn [how to use GitLab Pages to produce a code coverage report](https://about.gitlab.com/blog/2016/11/03/publish-code-coverage-report-with-gitlab-pages/). diff --git a/doc/user/project/pages/getting_started/pages_new_project_template.md b/doc/user/project/pages/getting_started/pages_new_project_template.md new file mode 100644 index 00000000000..cfa4e0910db --- /dev/null +++ b/doc/user/project/pages/getting_started/pages_new_project_template.md @@ -0,0 +1,34 @@ +--- +type: reference, howto +stage: Release +group: Release Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Create a Pages website from a new project template + +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/47857) in GitLab 11.8. + +GitLab provides templates for the most popular Static Site Generators (SSGs). +You can create a new project from a template and run the CI/CD pipeline to generate a Pages website. + +Use a template when you want to test GitLab Pages or start a new project that's already +configured to generate a Pages site. + +1. From the top navigation, click the **+** button and select **New project**. +1. Select **Create from Template**. +1. Next to one of the templates starting with **Pages**, click **Use template**. + +  + +1. Complete the form and click **Create project**. +1. From the left sidebar, navigate to your project's **CI/CD > Pipelines** + and click **Run pipeline** to trigger GitLab CI/CD to build and deploy your + site. + +The site can take approximately 30 minutes to deploy. +When the pipeline is finished, go to **Settings > Pages** to find the link to +your Pages website. + +For every change pushed to your repository, GitLab CI/CD runs a new pipeline +that immediately publishes your changes to the Pages site. diff --git a/doc/user/project/pages/getting_started_part_four.md b/doc/user/project/pages/getting_started_part_four.md index 8cf58280b88..e45befe004e 100644 --- a/doc/user/project/pages/getting_started_part_four.md +++ b/doc/user/project/pages/getting_started_part_four.md @@ -1,402 +1,5 @@ --- -last_updated: 2020-01-06 -type: reference, howto -stage: Release -group: Release Management -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +redirect_to: 'getting_started/pages_from_scratch.md' --- -# Create a GitLab Pages website from scratch - -This tutorial shows you how to create a Pages site from scratch. You will start with -a blank project and create your own CI file, which gives instruction to the -[GitLab Runner](https://docs.gitlab.com/runner/). When your CI/CD -[pipeline](../../../ci/pipelines/index.md) runs, the Pages site is created. - -This example uses the [Jekyll](https://jekyllrb.com/) Static Site Generator (SSG). -Other SSGs follow similar steps. You do not need to be familiar with Jekyll or SSGs -to complete this tutorial. - -## Prerequisites - -To follow along with this example, start with a blank project in GitLab. -Create three files in the root (top-level) directory. - -- `.gitlab-ci.yml` A YAML file that contains the commands you want to run. - For now, leave the file's contents blank. - -- `index.html` An HTML file you can populate with whatever HTML content you'd like, - for example: - - ```html - <html> - <head> - <title>Home</title> - </head> - <body> - <h1>Hello World!</h1> - </body> - </html> - ``` - -- [`Gemfile`](https://bundler.io/gemfile.html) A file that describes dependencies for Ruby programs. - Populate it with this content: - - ```ruby - source "https://rubygems.org" - - gem "jekyll" - ``` - -## Choose a Docker image - -In this example, the Runner uses a [Docker image](../../../ci/docker/using_docker_images.md) -to run scripts and deploy the site. - -This specific Ruby image is maintained on [DockerHub](https://hub.docker.com/_/ruby). - -Edit your `.gitlab-ci.yml` and add this text as the first line. - -```yaml -image: ruby:2.7 -``` - -If your SSG needs [NodeJS](https://nodejs.org/) to build, you must specify an -image that contains NodeJS as part of its file system. For example, for a -[Hexo](https://gitlab.com/pages/hexo) site, you can use `image: node:12.17.0`. - -## Install Jekyll - -To run [Jekyll](https://jekyllrb.com/) locally, you would open your terminal and: - -- Install [Bundler](https://bundler.io/) by running `gem install bundler`. -- Create `Gemfile.lock` by running `bundle install`. -- Install Jekyll by running `bundle exec jekyll build`. - -In the `.gitlab-ci.yml` file, this is written as: - -```yaml -script: - - gem install bundler - - bundle install - - bundle exec jekyll build -``` - -In addition, in the `.gitlab-ci.yml` file, each `script` is organized by a `job`. -A `job` includes the scripts and settings you want to apply to that specific -task. - -```yaml -job: - script: - - gem install bundler - - bundle install - - bundle exec jekyll build -``` - -For GitLab Pages, this `job` has a specific name, called `pages`. -This setting tells the Runner you want the job to deploy your website -with GitLab Pages: - -```yaml -pages: - script: - - gem install bundler - - bundle install - - bundle exec jekyll build -``` - -## Specify the `public` directory for output - -Jekyll needs to know where to generate its output. -GitLab Pages only considers files in a directory called `public`. - -Jekyll uses destination (`-d`) to specify an output directory for the built website: - -```yaml -pages: - script: - - gem install bundler - - bundle install - - bundle exec jekyll build -d public -``` - -## Specify the `public` directory for artifacts - -Now that Jekyll has output the files to the `public` directory, -the Runner needs to know where to get them. The artifacts are stored -in the `public` directory: - -```yaml -pages: - script: - - gem install bundler - - bundle install - - bundle exec jekyll build -d public - artifacts: - paths: - - public -``` - -Paste this into `.gitlab-ci.yml` file, so it now looks like this: - -```yaml -image: ruby:2.7 - -pages: - script: - - gem install bundler - - bundle install - - bundle exec jekyll build -d public - artifacts: - paths: - - public -``` - -Now save and commit the `.gitlab-ci.yml` file. You can watch the pipeline run -by going to **CI / CD > Pipelines**. - -When it succeeds, go to **Settings > Pages** to view the URL where your site -is now available. - -If you want to do more advanced tasks, you can update your `.gitlab-ci.yml` file -with [any of the available settings](../../../ci/yaml/README.md). You can check -your CI syntax with the [GitLab CI/CD Lint Tool](https://gitlab.com/ci/lint). - -The following topics show other examples of other options you can add to your CI/CD file. - -## Deploy specific branches to a Pages site - -You may want to deploy to a Pages site only from specific branches. - -First, add a `workflow` section to force the pipeline to run only when changes are -pushed to branches: - -```yaml -image: ruby:2.7 - -workflow: - rules: - - if: '$CI_COMMIT_BRANCH' - -pages: - script: - - gem install bundler - - bundle install - - bundle exec jekyll build -d public - artifacts: - paths: - - public -``` - -Then configure the pipeline to run the job for the master branch only. - -```yaml -image: ruby:2.7 - -workflow: - rules: - - if: '$CI_COMMIT_BRANCH' - -pages: - script: - - gem install bundler - - bundle install - - bundle exec jekyll build -d public - artifacts: - paths: - - public - rules: - - if: '$CI_COMMIT_BRANCH == "master"' -``` - -## Specify a stage to deploy - -There are three default stages for GitLab CI/CD: build, test, -and deploy. - -If you want to test your script and check the built site before deploying -to production, you can run the test exactly as it will run when you -push to `master`. - -To specify a stage for your job to run in, -add a `stage` line to your CI file: - -```yaml -image: ruby:2.7 - -workflow: - rules: - - if: '$CI_COMMIT_BRANCH' - -pages: - stage: deploy - script: - - gem install bundler - - bundle install - - bundle exec jekyll build -d public - artifacts: - paths: - - public - rules: - - if: '$CI_COMMIT_BRANCH == "master"' -``` - -Now add another job to the CI file, telling it to -test every push to every branch **except** the `master` branch: - -```yaml -image: ruby:2.7 - -workflow: - rules: - - if: '$CI_COMMIT_BRANCH' - -pages: - stage: deploy - script: - - gem install bundler - - bundle install - - bundle exec jekyll build -d public - artifacts: - paths: - - public - rules: - - if: '$CI_COMMIT_BRANCH == "master"' - -test: - stage: test - script: - - gem install bundler - - bundle install - - bundle exec jekyll build -d test - artifacts: - paths: - - test - rules: - - if: '$CI_COMMIT_BRANCH != "master"' -``` - -When the `test` job runs in the `test` stage, Jekyll -builds the site in a directory called `test`. The job affects -all branches except `master`. - -When you apply stages to different jobs, every job in the same -stage builds in parallel. If your web application needs more than -one test before being deployed, you can run all your tests at the -same time. - -## Remove duplicate commands - -To avoid duplicating the same scripts in every job, you can add them -to a `before_script` section. - -In the example, `gem install bundler` and `bundle install` were running -for both jobs, `pages` and `test`. - -Move these commands to a `before_script` section: - -```yaml -image: ruby:2.7 - -workflow: - rules: - - if: '$CI_COMMIT_BRANCH' - -before_script: - - gem install bundler - - bundle install - -pages: - stage: deploy - script: - - bundle exec jekyll build -d public - artifacts: - paths: - - public - rules: - - if: '$CI_COMMIT_BRANCH == "master"' - -test: - stage: test - script: - - bundle exec jekyll build -d test - artifacts: - paths: - - test - rules: - - if: '$CI_COMMIT_BRANCH != "master"' -``` - -## Build faster with cached dependencies - -To build faster, you can cache the installation files for your -project's dependencies by using the `cache` parameter. - -This example caches Jekyll dependencies in a `vendor` directory -when you run `bundle install`: - -```yaml -image: ruby:2.7 - -workflow: - rules: - - if: '$CI_COMMIT_BRANCH' - -cache: - paths: - - vendor/ - -before_script: - - gem install bundler - - bundle install --path vendor - -pages: - stage: deploy - script: - - bundle exec jekyll build -d public - artifacts: - paths: - - public - rules: - - if: '$CI_COMMIT_BRANCH == "master"' - -test: - stage: test - script: - - bundle exec jekyll build -d test - artifacts: - paths: - - test - rules: - - if: '$CI_COMMIT_BRANCH != "master"' -``` - -In this case, you need to exclude the `/vendor` -directory from the list of folders Jekyll builds. Otherwise, Jekyll -will try to build the directory contents along with the site. - -In the root directory, create a file called `_config.yml` -and add this content: - -```yaml -exclude: - - vendor -``` - -Now GitLab CI/CD not only builds the website, -but also pushes with **continuous tests** to feature-branches, -**caches** dependencies installed with Bundler, and -**continuously deploys** every push to the `master` branch. - -## Related topics - -For more information, see the following blog posts. - -- [Use GitLab CI/CD `environments` to deploy your - web app to staging and production](https://about.gitlab.com/blog/2016/08/26/ci-deployment-and-environments/). -- Learn [how to run jobs sequentially, - in parallel, or build a custom pipeline](https://about.gitlab.com/blog/2016/07/29/the-basics-of-gitlab-ci/). -- Learn [how to pull specific directories from different projects](https://about.gitlab.com/blog/2016/12/07/building-a-new-gitlab-docs-site-with-nanoc-gitlab-ci-and-gitlab-pages/) - to deploy this website, <https://docs.gitlab.com>. -- Learn [how to use GitLab Pages to produce a code coverage report](https://about.gitlab.com/blog/2016/11/03/publish-code-coverage-report-with-gitlab-pages/). +This document was moved to [getting_started/pages_from_scratch.md](getting_started/pages_from_scratch.md). diff --git a/doc/user/project/pages/index.md b/doc/user/project/pages/index.md index 5861282ca6f..b116c28d94b 100644 --- a/doc/user/project/pages/index.md +++ b/doc/user/project/pages/index.md @@ -46,10 +46,10 @@ To create a GitLab Pages website: | Document | Description | | -------- | ----------- | -| [Use a `.gitlab-ci.yml` template](getting_started/new_or_existing_website.md) | Add a Pages site to an existing project. Use a pre-populated CI template file. | -| [Create a `gitlab-ci.yml` file from scratch](getting_started_part_four.md) | Add a Pages site to an existing project. Learn how to create and configure your own CI file. | -| [Use a new project template](getting_started/pages_bundled_template.md) | Create a new project with Pages already configured by using a new project template. | -| [Fork a sample project](getting_started/fork_sample_project.md) | Create a new project with Pages already configured by forking a sample project. | +| [Fork a sample project](getting_started/pages_forked_sample_project.md) | Create a new project with Pages already configured by forking a sample project. | +| [Use a new project template](getting_started/pages_new_project_template.md) | Create a new project with Pages already configured by using a new project template. | +| [Use a `.gitlab-ci.yml` template](getting_started/pages_ci_cd_template.md) | Add a Pages site to an existing project. Use a pre-populated CI template file. | +| [Create a `gitlab-ci.yml` file from scratch](getting_started/pages_from_scratch.md) | Add a Pages site to an existing project. Learn how to create and configure your own CI file. | To update a GitLab Pages website: @@ -81,7 +81,7 @@ becomes available automatically. To deploy your site, GitLab uses its built-in tool called [GitLab CI/CD](../../../ci/README.md) to build your site and publish it to the GitLab Pages server. The sequence of scripts that GitLab CI/CD runs to accomplish this task is created from a file named -`.gitlab-ci.yml`, which you can [create and modify](getting_started_part_four.md) at will. A specific `job` called `pages` in the configuration file will make GitLab aware that you are deploying a GitLab Pages website. +`.gitlab-ci.yml`, which you can [create and modify](getting_started/pages_from_scratch.md) at will. A specific `job` called `pages` in the configuration file will make GitLab aware that you are deploying a GitLab Pages website. You can either use GitLab's [default domain for GitLab Pages websites](getting_started_part_one.md#gitlab-pages-default-domain-names), `*.gitlab.io`, or your own domain (`example.com`). In that case, you'll diff --git a/doc/user/project/pages/introduction.md b/doc/user/project/pages/introduction.md index 614a0d0dd19..a6923779f24 100644 --- a/doc/user/project/pages/introduction.md +++ b/doc/user/project/pages/introduction.md @@ -118,19 +118,19 @@ is so `cp` doesn't also copy `public/` to itself in an infinite loop: ```yaml pages: script: - - mkdir .public - - cp -r * .public - - mv .public public + - mkdir .public + - cp -r * .public + - mv .public public artifacts: paths: - - public + - public only: - - master + - master ``` ### `.gitlab-ci.yml` for a static site generator -See this document for a [step-by-step guide](getting_started_part_four.md). +See this document for a [step-by-step guide](getting_started/pages_from_scratch.md). ### `.gitlab-ci.yml` for a repository where there's also actual code @@ -161,13 +161,13 @@ image: ruby:2.6 pages: script: - - gem install jekyll - - jekyll build -d public/ + - gem install jekyll + - jekyll build -d public/ artifacts: paths: - - public + - public only: - - pages + - pages ``` See an example that has different files in the [`master` branch](https://gitlab.com/pages/jekyll-branched/tree/master) diff --git a/doc/user/project/protected_tags.md b/doc/user/project/protected_tags.md index e80b8098bec..bb5bca39398 100644 --- a/doc/user/project/protected_tags.md +++ b/doc/user/project/protected_tags.md @@ -12,7 +12,7 @@ This feature evolved out of [protected branches](protected_branches.md) ## Overview -Protected tags will prevent anyone from updating or deleting the tag, as and will prevent creation of matching tags based on the permissions you have selected. By default, anyone without Maintainer permission will be prevented from creating tags. +Protected tags will prevent anyone from updating or deleting the tag, and will prevent creation of matching tags based on the permissions you have selected. By default, anyone without Maintainer permission will be prevented from creating tags. ## Configuring protected tags diff --git a/doc/user/project/quick_actions.md b/doc/user/project/quick_actions.md index a3df173bd9d..def05bf94e4 100644 --- a/doc/user/project/quick_actions.md +++ b/doc/user/project/quick_actions.md @@ -7,14 +7,15 @@ info: To determine the technical writer assigned to the Stage/Group associated w # GitLab Quick Actions +> - Introduced in [GitLab 12.1](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/26672): +> once an action is executed, an alert appears when a quick action is successfully applied. +> - In [GitLab 13.2](https://gitlab.com/gitlab-org/gitlab/-/issues/16877) and later, you can use +> quick actions when updating the description of issues, epics, and merge requests. + Quick actions are textual shortcuts for common actions on issues, epics, merge requests, and commits that are usually done by clicking buttons or dropdowns in GitLab's UI. -You can enter these commands while creating a new issue or merge request, or -in comments of issues, epics, merge requests, and commits. Each command should be -on a separate line in order to be properly detected and executed. - -> From [GitLab 12.1](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/26672), once an -> action is executed, an alert appears when a quick action is successfully applied. +You can enter these commands in the description or in comments of issues, epics, merge requests, and commits. +Each command should be on a separate line in order to be properly detected and executed. ## Quick Actions for issues, merge requests and epics @@ -40,7 +41,7 @@ The following quick actions are applicable to descriptions, discussions and thre | `/create_merge_request <branch name>` | ✓ | | | Create a new merge request starting from the current issue. | | `/done` | ✓ | ✓ | ✓ | Mark To-Do as done. | | `/due <date>` | ✓ | | | Set due date. Examples of valid `<date>` include `in 2 days`, `this Friday` and `December 31st`. | -| `/duplicate <#issue>` | ✓ | | | Mark this issue as a duplicate of another issue and mark them as related. **(STARTER)** | +| `/duplicate <#issue>` | ✓ | | | Close this issue and mark as a duplicate of another issue. **(CORE)** Also, mark both as related. **(STARTER)** | | `/epic <epic>` | ✓ | | | Add to epic `<epic>`. The `<epic>` value should be in the format of `&epic`, `group&epic`, or a URL to an epic. **(PREMIUM)** | | `/estimate <<W>w <DD>d <hh>h <mm>m>` | ✓ | ✓ | | Set time estimate. For example, `/estimate 1w 3d 2h 14m`. | | `/iteration *iteration:iteration` | ✓ | | | Set iteration ([Introduced in GitLab 13.1](https://gitlab.com/gitlab-org/gitlab/-/issues/196795)) **(STARTER)** | diff --git a/doc/user/project/releases/img/custom_notifications_dropdown_v12_5.png b/doc/user/project/releases/img/custom_notifications_dropdown_v12_5.png Binary files differdeleted file mode 100644 index 879599a71f5..00000000000 --- a/doc/user/project/releases/img/custom_notifications_dropdown_v12_5.png +++ /dev/null diff --git a/doc/user/project/releases/img/custom_notifications_new_release_v12_5.png b/doc/user/project/releases/img/custom_notifications_new_release_v12_5.png Binary files differdeleted file mode 100644 index d136aa710b2..00000000000 --- a/doc/user/project/releases/img/custom_notifications_new_release_v12_5.png +++ /dev/null diff --git a/doc/user/project/releases/img/edit_release_page_v13_0.png b/doc/user/project/releases/img/edit_release_page_v13_0.png Binary files differdeleted file mode 100644 index 1b4343019af..00000000000 --- a/doc/user/project/releases/img/edit_release_page_v13_0.png +++ /dev/null diff --git a/doc/user/project/releases/img/milestone_list_with_releases_v12_5.png b/doc/user/project/releases/img/milestone_list_with_releases_v12_5.png Binary files differindex 2e3ec08ba87..efe48058d9a 100644 --- a/doc/user/project/releases/img/milestone_list_with_releases_v12_5.png +++ b/doc/user/project/releases/img/milestone_list_with_releases_v12_5.png diff --git a/doc/user/project/releases/img/milestone_with_releases_v12_5.png b/doc/user/project/releases/img/milestone_with_releases_v12_5.png Binary files differdeleted file mode 100644 index 8719a58ce4e..00000000000 --- a/doc/user/project/releases/img/milestone_with_releases_v12_5.png +++ /dev/null diff --git a/doc/user/project/releases/img/new_tag_12_5.png b/doc/user/project/releases/img/new_tag_12_5.png Binary files differdeleted file mode 100644 index 9a6145d71c7..00000000000 --- a/doc/user/project/releases/img/new_tag_12_5.png +++ /dev/null diff --git a/doc/user/project/releases/img/release_edit_button_v12_6.png b/doc/user/project/releases/img/release_edit_button_v12_6.png Binary files differdeleted file mode 100644 index 8cc080621cf..00000000000 --- a/doc/user/project/releases/img/release_edit_button_v12_6.png +++ /dev/null diff --git a/doc/user/project/releases/img/release_milestone_dropdown_v13_0.png b/doc/user/project/releases/img/release_milestone_dropdown_v13_0.png Binary files differdeleted file mode 100644 index 453c7ca93cc..00000000000 --- a/doc/user/project/releases/img/release_milestone_dropdown_v13_0.png +++ /dev/null diff --git a/doc/user/project/releases/img/release_with_milestone_v12_9.png b/doc/user/project/releases/img/release_with_milestone_v12_9.png Binary files differindex 53100e33955..c828e36114a 100644 --- a/doc/user/project/releases/img/release_with_milestone_v12_9.png +++ b/doc/user/project/releases/img/release_with_milestone_v12_9.png diff --git a/doc/user/project/releases/img/releases_count_v13_2.png b/doc/user/project/releases/img/releases_count_v13_2.png Binary files differnew file mode 100644 index 00000000000..1c0493326d1 --- /dev/null +++ b/doc/user/project/releases/img/releases_count_v13_2.png diff --git a/doc/user/project/releases/img/releases_v12_9.png b/doc/user/project/releases/img/releases_v12_9.png Binary files differdeleted file mode 100644 index bd23cf76651..00000000000 --- a/doc/user/project/releases/img/releases_v12_9.png +++ /dev/null diff --git a/doc/user/project/releases/img/tags_12_5.png b/doc/user/project/releases/img/tags_12_5.png Binary files differdeleted file mode 100644 index c9673a5232d..00000000000 --- a/doc/user/project/releases/img/tags_12_5.png +++ /dev/null diff --git a/doc/user/project/releases/index.md b/doc/user/project/releases/index.md index 58d143fb32b..258601574ca 100644 --- a/doc/user/project/releases/index.md +++ b/doc/user/project/releases/index.md @@ -9,261 +9,302 @@ info: To determine the technical writer assigned to the Stage/Group associated w > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/41766) in GitLab 11.7. -It is typical to create a [Git tag](../../../university/training/topics/tags.md) at -the moment of release to introduce a checkpoint in your source code -history, but in most cases your users will need compiled objects or other -assets output by your CI system to use them, not just the raw source -code. - -GitLab's **Releases** are a way to track deliverables in your project. Consider them -a snapshot in time of the source, build output, artifacts, and other metadata +To introduce a checkpoint in your source code history, you can assign a +[Git tag](https://git-scm.com/book/en/v2/Git-Basics-Tagging) at the moment of release. +However, in most cases, your users need more than just the raw source code. +They need compiled objects or other assets output by your CI/CD system. + +A GitLab *Release* is a snapshot of the source, build output, artifacts, and other metadata associated with a released version of your code. -## Getting started with Releases +You can create a GitLab release on any branch. When you create a release: -Start by giving a [description](#release-description) to the Release and -including its [assets](#release-assets), as follows. +- GitLab automatically archives source code and associates it with the release. +- GitLab automatically creates a JSON file that lists everything in the release, + so you can compare and audit releases. This file is called [release evidence](#release-evidence). +- You can add release notes and a message for the tag associated with the release. -## Release versioning +After you create a release, you can [associate milestones with it](#associate-milestones-with-a-release), +and attach [release assets](#release-assets), like runbooks or packages. -Release versions are manually assigned by the user in the Release title. GitLab uses [Semantic Versioning](https://semver.org/) for our releases, and we recommend you do too. Use `(Major).(Minor).(Patch)`, as detailed in the [GitLab Policy for Versioning](../../../policy/maintenance.md#versioning). +## View releases -For example, for GitLab version `10.5.7`: +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/36667) in GitLab 12.8. -- `10` represents the major version. The major release was `10.0.0`, but often referred to as `10.0`. -- `5` represents the minor version. The minor release was `10.5.0`, but often referred to as `10.5`. -- `7` represents the patch number. +To view a list of releases: -Any part of the version number can be multiple digits, for example, `13.10.11`. +- Go to **Project overview > Releases**, or + +- On the project's overview page, if at least one release exists, click the number of releases. + +  -### Release description + - On public projects, this number is visible to all users. + - On private projects, this number is visible to users with Reporter + [permissions](../../permissions.md#project-members-permissions) or higher. -Every Release has a description. You can add any text you like, but we recommend -including a changelog to describe the content of your release. This will allow -your users to quickly scan the differences between each one you publish. +## Create a release + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/32812) in GitLab 12.9. Releases can be created directly in the GitLab UI. NOTE: **Note:** -[Git's tagging messages](https://git-scm.com/book/en/v2/Git-Basics-Tagging) and -Release descriptions are unrelated. Description supports [Markdown](../../markdown.md). +Only users with Developer permissions or higher can create releases. +Read more about [Release permissions](../../../user/permissions.md#project-members-permissions). -### Release assets +You can create a release in the user interface, or by using the +[Releases API](../../../api/releases/index.md#create-a-release). +We recommend using the API to add release notes as one of the last steps in your CI/CD release pipeline. -You can currently add the following types of assets to each Release: +To create a new release through the GitLab UI: -- [Source code](#source-code): state of the repository at the time of the Release -- [Links](#links): to content such as built binaries or documentation +1. Navigate to **Project overview > Releases** and click the **New release** button. +1. In the [**Tag name**](#tag-name) box, enter a name. +1. In the **Create from** list, select the branch or enter a tag or commit SHA. +1. In the **Message** box, enter a message associated with the tag. +1. Optionally, in the [**Release notes**](#release-notes-description) + field, enter the release's description. You can use Markdown and drag and drop files to this field. + - If you leave this field empty, only a tag will be created. + - If you populate it, both a tag and a release will be created. +1. Click **Create tag**. -GitLab will support more asset types in the future, including objects such -as pre-built packages, compliance/security evidence, or container images. +If you created a release, you can view it at **Project overview > Releases**. +If you created a tag, you can view it at **Repository > Tags**. -#### Source code +You can now edit the release to [add milestones](#associate-milestones-with-a-release) +and [release assets](#release-assets). -GitLab automatically generate `zip`, `tar.gz`, `tar.bz2` and `tar` -archived source code from the given Git tag. These are read-only assets. +### Schedule a future release -#### Links +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/38105) in GitLab 12.1. -A link is any URL which can point to whatever you like; documentation, built -binaries, or other related materials. These can be both internal or external -links from your GitLab instance. +You can create a release ahead of time by using the [Releases API](../../../api/releases/index.md#upcoming-releases). +When you set a future `released_at` date, an **Upcoming Release** badge is displayed next to the +release tag. When the `released_at` date and time has passed, the badge is automatically removed. -The four types of links are "Runbook," "Package," "Image," and "Other." + -#### Permanent links to Release assets +## Edit a release -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/27300) in GitLab 12.9. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/26016) in GitLab 12.6. Asset link editing was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9427) in GitLab 12.10. -The assets associated with a Release are accessible through a permanent URL. -GitLab will always redirect this URL to the actual asset -location, so even if the assets move to a different location, you can continue -to use the same URL. This is defined during [link creation](../../../api/releases/links.md#create-a-link) or [updating](../../../api/releases/links.md#update-a-link). +NOTE: **Note:** +Only users with Developer permissions or higher can edit releases. +Read more about [Release permissions](../../../user/permissions.md#project-members-permissions). -Each asset has a name, a URL of the *actual* asset location, and optionally, a -`filepath` parameter, which, if you specify it, will create a URL pointing -to the asset for the Release. The format of the URL is: +To edit the details of a release: -```html -https://host/namespace/project/releases/:release/downloads/:filepath -``` +1. Navigate to **Project overview > Releases**. +1. In the top-right corner of the release you want to modify, click **Edit this release** (the pencil icon). +1. On the **Edit Release** page, change the release's details. +1. Click **Save changes**. -If you have an asset for the `v11.9.0-rc2` release in the `gitlab-org` -namespace and `gitlab-runner` project on `gitlab.com`, for example: +You can edit the release title, notes, associated milestones, and asset links. +To change other release information, such as the tag or release date, use the +[Releases API](../../../api/releases/index.md#update-a-release). -```json -{ - "name": "linux amd64", - "filepath": "/binaries/gitlab-runner-linux-amd64", - "url": "https://gitlab-runner-downloads.s3.amazonaws.com/v11.9.0-rc2/binaries/gitlab-runner-linux-amd64" -} -``` +## Add release notes to Git tags -This asset has a direct link of: +If you have an existing Git tag, you can add release notes to it. -```html -https://gitlab.com/gitlab-org/gitlab-runner/releases/v11.9.0-rc2/downloads/binaries/gitlab-runner-linux-amd64 -``` +You can do this in the user interface, or by using the [Releases API](../../../api/releases/index.md). +We recommend using the API to add release notes as one of the last steps in your CI/CD release pipeline. -The physical location of the asset can change at any time and the direct link will remain unchanged. +In the interface, to add release notes to a new Git tag: + +1. Navigate to your project's **Repository > Tags**. +1. Click **New tag**. +1. In the **Release notes** field, enter the release's description. + You can use Markdown and drag and drop files to this field. +1. Click **Create tag**. + +In the interface, to add release notes to an existing Git tag: + +1. Navigate to your project's **Repository > Tags**. +1. Click **Edit release notes** (the pencil icon). +1. In the **Release notes** field, enter the release's description. + You can use Markdown in this field, and drag and drop files to it. +1. Click **Save changes**. -### Releases associated with milestones +## Associate milestones with a release > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/29020) in GitLab 12.5. > - [Updated](https://gitlab.com/gitlab-org/gitlab/-/issues/39467) to edit milestones in the UI in GitLab 13.0. -Releases can optionally be associated with one or more -[project milestones](../milestones/index.md#project-milestones-and-group-milestones) -by including a `milestones` array in your requests to the -[Releases API](../../../api/releases/index.md#create-a-release) or by using the dropdown in the [Edit Release](#editing-a-release) page. +You can associate a release with one or more [project milestones](../milestones/index.md#project-milestones-and-group-milestones). + +You can do this in the user interface, or by including a `milestones` array in your request to +the [Releases API](../../../api/releases/index.md#create-a-release). + +In the user interface, to associate milestones to a release: - +1. Navigate to **Project overview > Releases**. +1. In the top-right corner of the release you want to modify, click **Edit this release** (the pencil icon). +1. From the **Milestones** list, select each milestone you want to associate. You can select multiple milestones. +1. Click **Save changes**. -Releases display this association with the **Milestone** indicator in the top -section of the Release block on the **Project overview > Releases** page, along -with some statistics about the issues in the milestone(s). +On the **Project overview > Releases** page, the **Milestone** is listed in the top +section, along with statistics about the issues in the milestones.  -Below is an example of milestones with no Releases, one Release, and two -Releases, respectively. +Releases are also visible on the **Issues > Milestones** page, and when you click a milestone +on this page. + +Here is an example of milestones with no releases, one release, and two releases, respectively.  -This relationship is also visible in the **Releases** section of the sidebar -when viewing a specific milestone. Below is an example of a milestone -associated with a large number of Releases. +## Get notified when a release is created - +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/26001) in GitLab 12.4. -## Releases list +You can be notified by email when a new release is created for your project. -Navigate to **Project > Releases** in order to see the list of releases for a given -project. +To subscribe to notifications for releases: - +1. Navigate to **Project overview**. +1. Click **Notification setting** (the bell icon). +1. In the list, click **Custom**. +1. Select the **New release** check box. +1. Close the dialog box to save. -### Number of Releases +## Prevent unintentional releases by setting a deploy freeze -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/36667) in GitLab 12.8. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/29382) in GitLab 13.0. -The incremental number of Releases is displayed on the project's details page. When clicked, -it takes you to the list of Releases. +Prevent unintended production releases during a period of time you specify by +setting a [*deploy freeze* period](../../../ci/environments/deployment_safety.md). +Deploy freezes help reduce uncertainty and risk when automating deployments. - +Use the [Freeze Periods API](../../../api/freeze_periods.md) to set a `freeze_start` and a `freeze_end`, which +are defined as [crontab](https://crontab.guru/) entries. -For private projects, the number of Releases is displayed to users with Reporter -[permissions](../../permissions.md#project-members-permissions) or higher. For public projects, -it is displayed to every user regardless of their permission level. +If the job that's executing is within a freeze period, GitLab CI/CD creates an environment +variable named `$CI_DEPLOY_FREEZE`. -### Upcoming Releases +To prevent the deployment job from executing, create a `rules` entry in your +`gitlab-ci.yaml`, for example: -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/38105) in GitLab 12.1. +```yaml +deploy_to_production: + stage: deploy + script: deploy_to_prod.sh + rules: + - if: $CI_DEPLOY_FREEZE == null +``` -A Release may be created ahead of time by specifying a future `released_at` date. Until -the `released_at` date and time is reached, an **Upcoming Release** badge will appear next to the -Release tag. Once the `released_at` date and time has passed, the badge is automatically removed. +If a project contains multiple freeze periods, all periods apply. If they overlap, the freeze covers the +complete overlapping period. - +For more information, see [Deployment safety](../../../ci/environments/deployment_safety.md). -## Creating a Release +## Release fields -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/32812) in GitLab 12.9, Releases can be created directly through the GitLab Releases UI. +The following fields are available when you create or edit a release. -NOTE: **Note:** -Only users with Developer permissions or higher can create Releases. -Read more about [Release permissions](../../../user/permissions.md#project-members-permissions). +### Tag name -To create a new Release through the GitLab UI: +The release tag name should include the release version. GitLab uses [Semantic Versioning](https://semver.org/) +for our releases, and we recommend you do too. Use `(Major).(Minor).(Patch)`, as detailed in the +[GitLab Policy for Versioning](../../../policy/maintenance.md#versioning). -1. Navigate to **Project overview > Releases** and click the **New release** button. -1. On the **New Tag** page, fill out the tag details. -1. Optionally, in the **Release notes** field, enter the Release's description. - If you leave this field empty, only a tag will be created. - If you populate it, both a tag and a Release will be created. -1. Click **Create tag**. +For example, for GitLab version `10.5.7`: -If you created a release, you can view it at **Project overview > Releases**. +- `10` represents the major version. The major release was `10.0.0`, but often referred to as `10.0`. +- `5` represents the minor version. The minor release was `10.5.0`, but often referred to as `10.5`. +- `7` represents the patch number. -You can also create a Release using the [Releases API](../../../api/releases/index.md#create-a-release): -we recommend doing this as one of the last steps in your CI/CD release pipeline. +Any part of the version number can be multiple digits, for example, `13.10.11`. -## Editing a release +### Release notes description -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/26016) in GitLab 12.6. Asset link editing was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9427) in GitLab 12.10. +Every release has a description. You can add any text you like, but we recommend +including a changelog to describe the content of your release. This helps users +quickly scan the differences between each release you publish. -To edit the details of a release, navigate to **Project overview > Releases** and click -the edit button (pencil icon) in the top-right corner of the release you want to modify. +NOTE: **Note:** +[Git's tagging messages](https://git-scm.com/book/en/v2/Git-Basics-Tagging) and +Release note descriptions are unrelated. Description supports [Markdown](../../markdown.md). - +### Release assets -This will bring you to the **Edit Release** page, from which you can -change some of the release's details. +You can currently add the following types of assets to each release: - +- [Source code](#source-code) +- [Links](#links) -Currently, it is only possible to edit the release title, notes, associated milestones, and asset -links. To change other release information, such as its tag, or release date, use the [Releases -API](../../../api/releases/index.md#update-a-release). Editing this information -through the **Edit Release** page is planned for a future version of GitLab. +GitLab will support more asset types in the future, including objects such +as pre-built packages, compliance/security evidence, or container images. -## Notification for Releases +#### Permanent links to release assets -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/26001) in GitLab 12.4. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/27300) in GitLab 12.9. -You can be notified by email when a new Release is created for your project. +The assets associated with a release are accessible through a permanent URL. +GitLab will always redirect this URL to the actual asset +location, so even if the assets move to a different location, you can continue +to use the same URL. This is defined during [link creation](../../../api/releases/links.md#create-a-link) or [updating](../../../api/releases/links.md#update-a-link). -To subscribe to Release notifications: +Each asset has a name, a URL of the *actual* asset location, and optionally, a +`filepath` parameter, which, if you specify it, will create a URL pointing +to the asset for the Release. The format of the URL is: -1. Navigate to your **Project**'s landing page. -1. Click the bell icon (**Notification setting**). -1. Select **Custom** from the dropdown menu. -  -1. Select **New release**. -  +```plaintext +https://host/namespace/project/releases/:release/downloads/:filepath +``` -## Add release notes to Git tags +If you have an asset for the `v11.9.0-rc2` release in the `gitlab-org` +namespace and `gitlab-runner` project on `gitlab.com`, for example: -You can add release notes to any Git tag using the notes feature. Release notes -behave like any other Markdown form in GitLab so you can write text and -drag and drop files to it. Release notes are stored in GitLab's database. +```json +{ + "name": "linux amd64", + "filepath": "/binaries/gitlab-runner-linux-amd64", + "url": "https://gitlab-runner-downloads.s3.amazonaws.com/v11.9.0-rc2/binaries/gitlab-runner-linux-amd64" +} +``` + +This asset has a direct link of: -There are several ways to add release notes: +```plaintext +https://gitlab.com/gitlab-org/gitlab-runner/releases/v11.9.0-rc2/downloads/binaries/gitlab-runner-linux-amd64 +``` + +The physical location of the asset can change at any time and the direct link will remain unchanged. -- In the interface, when you create a new Git tag. -- In the interface, by adding a release note to an existing Git tag. -- Using the [Releases API](../../../api/releases/index.md): (we recommend doing this as one of the last - steps in your CI/CD release pipeline). +### Source code -To create a new tag, navigate to your project's **Repository > Tags** and -click **New tag**. From there, you can fill the form with all the information -about the release: +GitLab automatically generates `zip`, `tar.gz`, `tar.bz2` and `tar` +archived source code from the given Git tag. These are read-only assets. - +### Links -You can also edit an existing tag to add release notes: +A link is any URL which can point to whatever you like: documentation, built +binaries, or other related materials. These can be both internal or external +links from your GitLab instance. - +The four types of links are "Runbook," "Package," "Image," and "Other." -## Release Evidence +## Release evidence > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/26019) in GitLab 12.6. Each time a release is created, GitLab takes a snapshot of data that's related to it. -This data is called Release Evidence. It includes linked milestones and issues, and -it is taken at moment the release is created. It provides a chain of custody and can -facilitate processes like external audits, for example. +This data is saved in a JSON file and called *release evidence*. It includes linked milestones +and issues and can facilitate internal processes like external audits. + +To access the release evidence, on the Releases page, click the link to the JSON file that's listed +under the **Evidence collection** heading. You can also [use the API](../../../api/releases/index.md#collect-release-evidence-premium-only) to -generate Release Evidence for an existing release. Because of this, [each release](#releases-list) -can have multiple Release Evidence snapshots. You can view the Release Evidence and -its details on the Release page. +generate release evidence for an existing release. Because of this, each release +can have multiple release evidence snapshots. You can view the release evidence and +its details on the Releases page. NOTE: **Note:** When the issue tracker is disabled, release evidence [cannot be downloaded](https://gitlab.com/gitlab-org/gitlab/-/issues/208397). -Release Evidence is stored as a JSON object, so you can compare evidence by using -commonly-available tools. - -Here is an example of a Release Evidence object: +Here is an example of a release evidence object: ```json { @@ -313,94 +354,95 @@ Here is an example of a Release Evidence object: "created_at": "2019-04-17 15:45:12 UTC", "issues": [] } + ], + "report_artifacts": [ + { + "url":"https://gitlab.example.com/root/project-name/-/jobs/111/artifacts/download" + } ] } } ``` -### Diabling Release Evidence display **(CORE ONLY)** +### Collect release evidence **(PREMIUM ONLY)** -This feature comes with the `:release_evidence_collection` feature flag -enabled by default in GitLab self-managed instances. To turn it off, -ask a GitLab administrator with Rails console access to run the following -command: +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/199065) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.10. -```ruby -Feature.disable(:release_evidence_collection) -``` +When a release is created, release evidence is automatically collected. To initiate evidence collection any other time, use an [API call](../../../api/releases/index.md#collect-release-evidence-premium-only). You can collect release evidence multiple times for one release. -NOTE: **Note:** -Please note that Release Evidence's data is collected regardless of this -feature flag, which only enables or disables the display of the data on the -Releases page. +Evidence collection snapshots are visible on the Releases page, along with the timestamp the evidence was collected. -### Collect release evidence **(PREMIUM ONLY)** +### Include report artifacts as release evidence **(ULTIMATE ONLY)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/199065) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.10. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/32773) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.2. -Evidence collection can be initiated by using an [API call](../../../api/releases/index.md#collect-release-evidence-premium-only) at any time. Evidence snapshots are visible on -the Release page, along with the timestamp the Evidence was collected. +When you create a release, if [job artifacts](../../../ci/pipelines/job_artifacts.md#artifactsreports) are included in the last pipeline that ran, they are automatically included in the release as release evidence. -### Schedule release evidence collection +Although job artifacts normally expire, artifacts included in release evidence do not expire. -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/23697) in GitLab 12.8. +To enable job artifact collection you need to specify both: -When the `released_at` date and time is not provided, the date and time of Release -creation is used. The Evidence collection background job is immediately executed. +1. [`artifacts:paths`](../../../ci/yaml/README.md#artifactspaths) +1. [`artifacts:reports`](../../../ci/pipelines/job_artifacts.md#artifactsreports) -If a future `released_at` is specified, the Release becomes an **Upcoming Release**. In this -case, the Evidence is scheduled to be collected at the `released_at` date and time, via a -background job. +```yaml +ruby: + script: + - gem install bundler + - bundle install + - bundle exec rspec --format progress --format RspecJunitFormatter --out rspec.xml + artifacts: + paths: + - rspec.xml + reports: + junit: rspec.xml +``` -If a past `released_at` is used, no Evidence is collected for the Release. +If the pipeline ran successfully, when you create your release, the `rspec.xml` file is saved as release evidence. -## GitLab Releaser +NOTE: **Note:** +If you [schedule release evidence collection](#schedule-release-evidence-collection), some artifacts may already be expired by the time of evidence collection. To avoid this you can use the [`artifacts:expire_in`](../../../ci/yaml/README.md#artifactsexpire_in) keyword. Learn more in [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/222351). -> [Introduced](https://gitlab.com/gitlab-org/gitlab-releaser/-/merge_requests/6) in GitLab 12.10. +### Schedule release evidence collection -GitLab Releaser is a CLI tool for managing GitLab Releases from the command line or from -GitLab CI/CD's configuration file, `.gitlab-ci.yml`. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/23697) in GitLab 12.8. -With it, you can create, update, modify, and delete Releases right through the -terminal. +In the API: -Read the [GitLab Releaser documentation](https://gitlab.com/gitlab-org/gitlab-releaser/-/tree/master/docs/index.md) -for details. +- If you specify a future `released_at` date, the release becomes an **Upcoming Release** + and the evidence is collected on the date of the release. You cannot collect + release evidence before then. +- If you use a past `released_at` date, no evidence is collected. +- If you do not specify a `released_at` date, release evidence is collected on the + date the release is created. -## Set a deploy freeze +### Disable release evidence display **(CORE ONLY)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/29382) in GitLab 13.0. +The `:release_evidence_collection` feature flag is enabled by default in GitLab +self-managed instances. To turn it off, ask a GitLab administrator with Rails console +access to run the following command: -With a deploy freeze, you can prevent an unintended production release during a -period of time you specify, whether a company event or public holiday. You can -now rely on the enforcement of policies that are typically outside the scope of -GitLab to reduce uncertainty and risk when automating deployments. +```ruby +Feature.disable(:release_evidence_collection) +``` -Deploy freeze periods are set at the Project level, and may be created and -managed using the [Freeze Periods API](../../../api/freeze_periods.md). -Each Freeze Period has a `freeze_start` and a `freeze_end`, which are defined -as [crontab](https://crontab.guru/) entries. If a project contains multiple -freeze periods, all will apply, and should they overlap, the freeze covers the -complete overlapped period. +NOTE: **Note:** +Release evidence is collected regardless of this feature flag, +which only enables or disables the display of the data on the +Releases page. -During pipeline processing, GitLab CI creates an environment variable named -`$CI_DEPLOY_FREEZE` if the currently executing job is within a -Freeze Period. +## GitLab Releaser -To take advantage of this variable, create a `rules` entry in your -`gitlab-ci.yaml` to prevent the deployment job from executing. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-releaser/-/merge_requests/6) in GitLab 12.10. -For example: +GitLab Releaser is a CLI tool for managing GitLab Releases from the command line or from +GitLab's CI/CD configuration file, `.gitlab-ci.yml`. -```yaml -deploy_to_production: - stage: deploy - script: deploy_to_prod.sh - rules: - - if: $CI_DEPLOY_FREEZE == null -``` +With it, you can create, update, modify, and delete releases right through the +terminal. -For more information, see [Deployment safety](../../../ci/environments/deployment_safety.md). +Read the [GitLab Releaser documentation](https://gitlab.com/gitlab-org/gitlab-releaser/-/tree/master/docs/index.md) +for details. <!-- ## Troubleshooting diff --git a/doc/user/project/repository/branches/index.md b/doc/user/project/repository/branches/index.md index 5fc6aa184bd..f94ca7ac106 100644 --- a/doc/user/project/repository/branches/index.md +++ b/doc/user/project/repository/branches/index.md @@ -41,17 +41,51 @@ See also: ## Default branch When you create a new [project](../../index.md), GitLab sets `master` as the default -branch for your project. You can choose another branch to be your project's +branch of the repository. You can choose another branch to be your project's default under your project's **Settings > Repository**. -The default branch is the branch affected by the -[issue closing pattern](../../issues/managing_issues.md#closing-issues-automatically), -which means that _an issue will be closed when a merge request is merged to -the **default branch**_. +When closing issues directly from merge requests through the [issue closing pattern](../../issues/managing_issues.md#closing-issues-automatically), +the target is the project's **default branch**. -The default branch is also protected against accidental deletion. Read through -the documentation on [protected branches](../../protected_branches.md#protected-branches) -to learn more. +The default branch is also initially [protected](../../protected_branches.md#protected-branches) +against accidental deletion and forced pushes. + +### Custom initial branch name **(CORE ONLY)** + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/221013) in GitLab 13.2. +> - It's deployed behind a feature flag, enabled by default. +> - It's enabled on GitLab.com. +> - It cannot be enabled or disabled per-project. +> - It's recommended for production use. +> - For GitLab self-managed instances, GitLab administrators can opt to [disable it](#enable-or-disable-custom-initial-branch-name-core-only). **(CORE ONLY)** + +By default, when you create a new project in GitLab, the initial branch is called `master`. +For self-managed instances, a GitLab administrator can customize the initial branch name to something +else. This way, every new project created from then on will start from the custom branch name rather than `master`. To do so: + +1. Go to the **{admin}** **Admin Area > Settings > Repository** and expand **Default initial + branch name**. +1. Change the default initial branch to a custom name of your choice. +1. **Save Changes**. + +#### Enable or disable custom initial branch name **(CORE ONLY)** + +Setting the default initial branch name is under development but ready for production use. +It is deployed behind a feature flag that is **enabled by default**. +[GitLab administrators with access to the GitLab Rails console](../../../../administration/feature_flags.md) +can opt to disable it for your instance. + +To disable it: + +```ruby +Feature.disable(:global_default_branch_name) +``` + +To enable it: + +```ruby +Feature.enable(:global_default_branch_name) +``` ## Compare diff --git a/doc/user/project/repository/index.md b/doc/user/project/repository/index.md index 48975b7864e..0cf375009a0 100644 --- a/doc/user/project/repository/index.md +++ b/doc/user/project/repository/index.md @@ -186,7 +186,8 @@ updated every 15 minutes at most, so may not reflect recent activity. The displa The project size may differ slightly from one instance to another due to compression, housekeeping, and other factors. -[Repository size limit](../../admin_area/settings/account_and_limit_settings.md) may be set by admins. GitLab.com's repository size limit [is set by GitLab](../../gitlab_com/index.md#repository-size-limit). +[Repository size limit](../../admin_area/settings/account_and_limit_settings.md) may be set by admins. +GitLab.com's repository size limit [is set by GitLab](../../gitlab_com/index.md#repository-size-limit). ## Contributors diff --git a/doc/user/project/repository/reducing_the_repo_size_using_git.md b/doc/user/project/repository/reducing_the_repo_size_using_git.md index 124150c441a..baad5027703 100644 --- a/doc/user/project/repository/reducing_the_repo_size_using_git.md +++ b/doc/user/project/repository/reducing_the_repo_size_using_git.md @@ -25,11 +25,16 @@ Rewriting repository history is a destructive operation. Make sure to backup you you begin. The best way back up a repository is to [export the project](../settings/import_export.md#exporting-a-project-and-its-data). +NOTE: **Note:** +Git LFS files can only be removed by an Administrator using a +[Rake task](../../../raketasks/cleanup.md). Removal of this limitation +[is planned](https://gitlab.com/gitlab-org/gitlab/-/issues/223621). + ## Purge files from repository history To make cloning your project faster, rewrite branches and tags to remove unwanted files. -1. [Install `git filter-repo`](https://github.com/newren/git-filter-repo/blob/master/INSTALL.md) +1. [Install `git filter-repo`](https://github.com/newren/git-filter-repo/blob/main/INSTALL.md) using a supported package manager or from source. 1. Clone a fresh copy of the repository using `--bare`: @@ -40,12 +45,25 @@ To make cloning your project faster, rewrite branches and tags to remove unwante 1. Using `git filter-repo`, purge any files from the history of your repository. - To purge all large files, the `--strip-blobs-bigger-than` option can be used: + To purge large files, the `--strip-blobs-bigger-than` option can be used: ```shell git filter-repo --strip-blobs-bigger-than 10M ``` + To purge large files stored using Git LFS, the `--blob--callback` option can + be used. The example below, uses the callback to read the file size from the + Git LFS pointer, and removes files larger than 10MB. + + ```shell + git filter-repo --blob-callback ' + if blob.data.startswith(b"version https://git-lfs.github.com/spec/v1"): + size_in_bytes = int.from_bytes(blob.data[124:], byteorder="big") + if size_in_bytes > 10*1000: + blob.skip() + ' + ``` + To purge specific large files by path, the `--path` and `--invert-paths` options can be combined: ```shell @@ -80,6 +98,12 @@ To make cloning your project faster, rewrite branches and tags to remove unwante [Protected tags](../protected_tags.md) will cause this to fail. To proceed, you must remove tag protection, push, and then re-enable protected tags. +1. Manually run [project housekeeping](../../../administration/housekeeping.md#manual-housekeeping) + +NOTE: **Note:** +Project statistics are cached for performance. You may need to wait 5-10 minutes +to see a reduction in storage utilization. + ## Purge files from GitLab storage To reduce the size of your repository in GitLab, you must remove GitLab internal references to @@ -103,7 +127,7 @@ cannot be fetched at all. However, these refs can be accessed from the Git bundle inside a project export. -1. [Install `git filter-repo`](https://github.com/newren/git-filter-repo/blob/master/INSTALL.md) +1. [Install `git filter-repo`](https://github.com/newren/git-filter-repo/blob/main/INSTALL.md) using a supported package manager or from source. 1. Generate a fresh [export from the @@ -128,7 +152,7 @@ However, these refs can be accessed from the Git bundle inside a project export. trying to remove internal refs, we will rely on the `commit-map` produced by each run to tell us which internal refs to remove. - NOTE:**Note:** + NOTE: **Note:** `git filter-repo` creates a new `commit-map` file every run, and overwrite the `commit-map` from the previous run. You will need this file from **every** run. Do the next step every time you run `git filter-repo`. @@ -176,6 +200,7 @@ You will receive an email once it has completed. When using repository cleanup, note: +- Project statistics are cached. You may need to wait 5-10 minutes to see a reduction in storage utilization. - Housekeeping prunes loose objects older than 2 weeks. This means objects added in the last 2 weeks will not be removed immediately. If you have access to the [Gitaly](../../../administration/gitaly/index.md) server, you may run `git gc --prune=now` to diff --git a/doc/user/project/repository/repository_mirroring.md b/doc/user/project/repository/repository_mirroring.md index f75b083e6dc..bdf13100a6a 100644 --- a/doc/user/project/repository/repository_mirroring.md +++ b/doc/user/project/repository/repository_mirroring.md @@ -114,7 +114,7 @@ After the mirror is created, this option can currently only be modified via the To set up a mirror from GitLab to GitHub, you need to follow these steps: -1. Create a [GitHub personal access token](https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line) with the `public_repo` box checked. +1. Create a [GitHub personal access token](https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token) with the `public_repo` box checked. 1. Fill in the **Git repository URL** field using this format: `https://<your_github_username>@github.com/<your_github_group>/<your_github_project>.git`. 1. Fill in **Password** field with your GitHub personal access token. 1. Click the **Mirror repository** button. @@ -125,10 +125,10 @@ The repository will push soon. To force a push, click the appropriate button. ## Setting up a push mirror to another GitLab instance with 2FA activated -1. On the destination GitLab instance, create a [personal access token](../../profile/personal_access_tokens.md) with `API` scope. +1. On the destination GitLab instance, create a [personal access token](../../profile/personal_access_tokens.md) with `write_repository` scope. 1. On the source GitLab instance: 1. Fill in the **Git repository URL** field using this format: `https://oauth2@<destination host>/<your_gitlab_group_or_name>/<your_gitlab_project>.git`. - 1. Fill in **Password** field with the GitLab personal access token created on the destination GitLab instance. + 1. Fill in the **Password** field with the GitLab personal access token created on the destination GitLab instance. 1. Click the **Mirror repository** button. ## Pulling from a remote repository **(STARTER)** @@ -231,7 +231,7 @@ those you expect. GitLab.com and other code hosting sites publish their fingerprints in the open for you to check: - [AWS CodeCommit](https://docs.aws.amazon.com/codecommit/latest/userguide/regions.html#regions-fingerprints) -- [Bitbucket](https://confluence.atlassian.com/bitbucket/ssh-keys-935365775.html) +- [Bitbucket](https://support.atlassian.com/bitbucket-cloud/docs/configure-ssh-and-two-step-verification/) - [GitHub](https://help.github.com/en/github/authenticating-to-github/githubs-ssh-key-fingerprints) - [GitLab.com](../../gitlab_com/index.md#ssh-host-keys-fingerprints) - [Launchpad](https://help.launchpad.net/SSHFingerprints) diff --git a/doc/user/project/repository/x509_signed_commits/index.md b/doc/user/project/repository/x509_signed_commits/index.md index d55d5c5c2d8..ffbad4e0989 100644 --- a/doc/user/project/repository/x509_signed_commits/index.md +++ b/doc/user/project/repository/x509_signed_commits/index.md @@ -25,9 +25,11 @@ For a commit or tag to be *verified* by GitLab: which is usually up to three years. - The signing time is equal or later then commit time. -NOTE: **Note:** Certificate revocation lists are checked on a daily basis via background worker. +NOTE: **Note:** +Certificate revocation lists are checked on a daily basis via background worker. -NOTE: **Note:** Self signed certificates without `authorityKeyIdentifier`, +NOTE: **Note:** +Self signed certificates without `authorityKeyIdentifier`, `subjectKeyIdentifier`, and `crlDistributionPoints` are not supported. We recommend using certificates from a PKI that are in line with [RFC 5280](https://tools.ietf.org/html/rfc5280). diff --git a/doc/user/project/requirements/img/requirement_archive_view_v12_10.png b/doc/user/project/requirements/img/requirement_archive_view_v12_10.png Binary files differdeleted file mode 100644 index b3a52caba6c..00000000000 --- a/doc/user/project/requirements/img/requirement_archive_view_v12_10.png +++ /dev/null diff --git a/doc/user/project/requirements/img/requirement_create_view_v12_10.png b/doc/user/project/requirements/img/requirement_create_view_v12_10.png Binary files differdeleted file mode 100644 index ecb08fe8a8b..00000000000 --- a/doc/user/project/requirements/img/requirement_create_view_v12_10.png +++ /dev/null diff --git a/doc/user/project/requirements/img/requirement_edit_view_v12_10.png b/doc/user/project/requirements/img/requirement_edit_view_v12_10.png Binary files differdeleted file mode 100644 index 5251e7eae1e..00000000000 --- a/doc/user/project/requirements/img/requirement_edit_view_v12_10.png +++ /dev/null diff --git a/doc/user/project/requirements/img/requirements_archived_list_view_v12_10.png b/doc/user/project/requirements/img/requirements_archived_list_view_v12_10.png Binary files differdeleted file mode 100644 index a5487b46894..00000000000 --- a/doc/user/project/requirements/img/requirements_archived_list_view_v12_10.png +++ /dev/null diff --git a/doc/user/project/requirements/img/requirements_archived_list_view_v13_1.png b/doc/user/project/requirements/img/requirements_archived_list_view_v13_1.png Binary files differnew file mode 100644 index 00000000000..aafc8543bae --- /dev/null +++ b/doc/user/project/requirements/img/requirements_archived_list_view_v13_1.png diff --git a/doc/user/project/requirements/img/requirements_list_v13_1.png b/doc/user/project/requirements/img/requirements_list_v13_1.png Binary files differnew file mode 100644 index 00000000000..3d2adfac074 --- /dev/null +++ b/doc/user/project/requirements/img/requirements_list_v13_1.png diff --git a/doc/user/project/requirements/img/requirements_list_view_v12_10.png b/doc/user/project/requirements/img/requirements_list_view_v12_10.png Binary files differdeleted file mode 100644 index cee1f3781f6..00000000000 --- a/doc/user/project/requirements/img/requirements_list_view_v12_10.png +++ /dev/null diff --git a/doc/user/project/requirements/index.md b/doc/user/project/requirements/index.md index d9bd02518a4..ae22dbc7e72 100644 --- a/doc/user/project/requirements/index.md +++ b/doc/user/project/requirements/index.md @@ -22,7 +22,7 @@ When a feature is no longer necessary, you can [archive the related requirement] <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> For an overview, see [GitLab 12.10 Introduces Requirements Management](https://www.youtube.com/watch?v=uSS7oUNSEoU). - + ## Create a requirement @@ -38,8 +38,6 @@ To create a requirement: You will see the newly created requirement on the top of the list, as the requirements list is sorted by creation date in descending order. - - ## Edit a requirement You can edit a requirement (if you have the necessary privileges) from the requirements @@ -51,16 +49,12 @@ To edit a requirement: 1. Update the title in text input field. 1. Click **Save changes**. - - ## Archive a requirement You can archive an open requirement (if you have the necessary privileges) while you're in the **Open** tab. -From the requirements list page, click **Archive** (**{archive}**). - - +To archive a requirement, click **Archive** (**{archive}**). As soon as a requirement is archived, it no longer appears in the **Open** tab. @@ -68,35 +62,37 @@ As soon as a requirement is archived, it no longer appears in the **Open** tab. You can view the list of archived requirements in the **Archived** tab. - + To reopen an archived requirement, click **Reopen**. As soon as a requirement is reopened, it no longer appears in the **Archived** tab. -## Search for a requirement from the requirements list page +## Search for a requirement -> - Introduced in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.1. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/212543) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.1. -You can search for a requirement from the list of requirements using filtered search bar (similar to -that of Issues and Merge Requests) based on following parameters: +You can search for a requirement from the requirements list page based on the following criteria: -- Title -- Author username +- Requirement title +- Author's username -To search, go to the list of requirements and click the field **Search or filter results**. -It will display a dropdown menu, from which you can add an author. You can also enter plain -text to search by epic title or description. When done, press <kbd>Enter</kbd> on your -keyboard to filter the list. +To search for a requirement: -You can also sort requirements list by: +1. In a project, go to **{requirements}** **Requirements > List**. +1. Click the **Search or filter results** field. A dropdown menu appears. +1. Select the requirement author from the dropdown or enter plain text to search by requirement title. +1. Press <kbd>Enter</kbd> on your keyboard to filter the list. + +You can also sort the requirements list by: - Created date - Last updated ## Allow requirements to be satisfied from a CI job -> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/2859) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.1. +> - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/2859) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.1. +> - [Added](https://gitlab.com/gitlab-org/gitlab/-/issues/215514) ability to specify individual requirements and their statuses in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.2. GitLab supports [requirements test reports](../../../ci/pipelines/job_artifacts.md#artifactsreportsrequirements-ultimate) now. @@ -132,6 +128,32 @@ the requirement report is checked for the "all passed" record (`{"*":"passed"}`), and on success, it marks all existing open requirements as Satisfied. +#### Specifying individual requirements + +It is possible to specify individual requirements and their statuses. + +If the following requirements exist: + +- `REQ-1` (with IID `1`) +- `REQ-2` (with IID `2`) +- `REQ-3` (with IID `3`) + +It is possible to specify that the first requirement passed, and the second failed. +Valid values are "passed" and "failed". +By omitting a requirement IID (in this case `REQ-3`'s IID `3`), no result is noted. + +```yaml +requirements_confirmation: + when: manual + allow_failure: false + script: + - mkdir tmp + - echo "{\"1\":\"passed\", \"2\":\"failed\"}" > tmp/requirements.json + artifacts: + reports: + requirements: tmp/requirements.json +``` + ### Add the manual job to CI conditionally To configure your CI to include the manual job only when there are some open @@ -140,9 +162,9 @@ requirements, add a rule which checks `CI_HAS_OPEN_REQUIREMENTS` CI variable. ```yaml requirements_confirmation: rules: - - if: "$CI_HAS_OPEN_REQUIREMENTS" == "true" - when: manual - - when: never + - if: "$CI_HAS_OPEN_REQUIREMENTS" == "true" + when: manual + - when: never allow_failure: false script: - mkdir tmp diff --git a/doc/user/project/service_desk.md b/doc/user/project/service_desk.md index ffb1f6a1407..cb6f8e2221d 100644 --- a/doc/user/project/service_desk.md +++ b/doc/user/project/service_desk.md @@ -4,10 +4,11 @@ group: Certify info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# Service Desk **(STARTER)** +# Service Desk > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/149) in [GitLab Premium](https://about.gitlab.com/pricing/) 9.1. > - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/214839) to [GitLab Starter](https://about.gitlab.com/pricing/) in 13.0. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/215364) to [GitLab Core](https://about.gitlab.com/pricing/) in 13.2. ## Overview @@ -61,10 +62,10 @@ users will only see the thread through email. ## Configuring Service Desk NOTE: **Note:** -Service Desk is enabled on GitLab.com. If you're a [Silver subscriber](https://about.gitlab.com/pricing/#gitlab-com), -you can skip step 1 below; you only need to enable it per project. +Service Desk is enabled on GitLab.com. +You can skip step 1 below; you only need to enable it per project. -If you have the correct access and a Premium license, you have the option to set up Service Desk. +If you have project maintainer access you have the option to set up Service Desk. Follow these steps to do so: 1. [Set up incoming email](../../administration/incoming_email.md#set-it-up) for the GitLab instance. @@ -149,7 +150,9 @@ It can contain only lowercase letters (`a-z`), numbers (`0-9`), or underscores (  -For example, suppose you add the following to your configuration: +You can add the following snippets to your configuration. + +Example for installations from source: ```yaml service_desk_email: @@ -167,6 +170,32 @@ service_desk_email: expunge_deleted: true ``` +Example for Omnibus GitLab installations: + +```ruby +gitlab_rails['service_desk_email_enabled'] = true + +gitlab_rails['service_desk_email_address'] = "project_contact+%{key}@gmail.com" + +gitlab_rails['service_desk_email_email'] = "project_support@gmail.com" + +gitlab_rails['service_desk_email_password'] = "[REDACTED]" + +gitlab_rails['service_desk_email_mailbox_name'] = "inbox" + +gitlab_rails['service_desk_email_idle_timeout'] = 60 + +gitlab_rails['service_desk_email_log_file'] = "/var/log/gitlab/mailroom/mail_room_json.log" + +gitlab_rails['service_desk_email_host'] = "imap.gmail.com" + +gitlab_rails['service_desk_email_port'] = 993 + +gitlab_rails['service_desk_email_ssl'] = true + +gitlab_rails['service_desk_email_start_tls'] = false +``` + In this case, suppose the `mygroup/myproject` project Service Desk settings has the project name suffix set to `support`, and a user sends an email to `project_contact+mygroup-myproject-support@example.com`. As a result, a new Service Desk issue is created from this email in the `mygroup/myproject` project. diff --git a/doc/user/project/settings/import_export.md b/doc/user/project/settings/import_export.md index 5a364eb89aa..cb9f0491b44 100644 --- a/doc/user/project/settings/import_export.md +++ b/doc/user/project/settings/import_export.md @@ -44,6 +44,8 @@ Note the following: ## Version history +### 13.0+ + Starting with GitLab 13.0, GitLab can import bundles that were exported from a different GitLab deployment. This ability is limited to two previous GitLab [minor](../../../policy/maintenance.md#versioning) releases, which is similar to our process for [Security Releases](../../../policy/maintenance.md#security-releases). @@ -61,7 +63,7 @@ Prior to 13.0 this was a defined compatibility table: | Exporting GitLab version | Importing GitLab version | | -------------------------- | -------------------------- | -| 11.7 to 13.0 | 11.7 to 13.0 | +| 11.7 to 12.10 | 11.7 to 12.10 | | 11.1 to 11.6 | 11.1 to 11.6 | | 10.8 to 11.0 | 10.8 to 11.0 | | 10.4 to 10.7 | 10.4 to 10.7 | @@ -95,7 +97,7 @@ The following items will be exported: - Project and wiki repositories - Project uploads -- Project configuration, including services +- Project configuration, excluding integrations - Issues with comments, merge requests with diffs and comments, labels, milestones, snippets, time tracking, and other project entities - Design Management files and data @@ -162,6 +164,15 @@ NOTE: **Note:** The maximum import file size can be set by the Administrator, default is 50MB. As an administrator, you can modify the maximum import file size. To do so, use the `max_import_size` option in the [Application settings API](../../../api/settings.md#change-application-settings) or the [Admin UI](../../admin_area/settings/account_and_limit_settings.md). +### Project import status + +You can query an import through the [Project import/export API](../../../api/project_import_export.md#import-status). +As described in the API documentation, the query may return an import error or exceptions. + +### Import large projects **(CORE ONLY)** + +If you have a larger project, consider using a Rake task, as described in our [developer documentation](../../../development/import_project.md#importing-via-a-rake-task). + ## Rate limits To help avoid abuse, users are rate limited to: diff --git a/doc/user/project/settings/index.md b/doc/user/project/settings/index.md index 0798c39fff5..7fe6e702d1c 100644 --- a/doc/user/project/settings/index.md +++ b/doc/user/project/settings/index.md @@ -20,7 +20,7 @@ Adjust your project's name, description, avatar, [default branch](../repository/ The project description also partially supports [standard Markdown](../../markdown.md#standard-markdown-and-extensions-in-gitlab). You can use [emphasis](../../markdown.md#emphasis), [links](../../markdown.md#links), and [line-breaks](../../markdown.md#line-breaks) to add more context to the project description. -#### Compliance framework **(ULTIMATE)** +#### Compliance framework **(PREMIUM)** You can select a framework label to identify that your project has certain compliance requirements or needs additional oversight. Available labels include: @@ -68,7 +68,7 @@ Some features depend on others: - If you disable the **Issues** option, GitLab also removes the following features: - **Issue Boards** - - [**Service Desk**](#service-desk-starter) **(STARTER)** + - [**Service Desk**](#service-desk-starter) NOTE: **Note:** When the **Issues** option is disabled, you can still access **Milestones** @@ -223,13 +223,18 @@ To remove a project: 1. In the Remove project section, click the **Remove project** button. 1. Confirm the action when asked to. -This action either: +This action: - Removes a project including all associated resources (issues, merge requests etc). -- Since [GitLab 12.6](https://gitlab.com/gitlab-org/gitlab/-/issues/32935), on - [GitLab Premium or GitLab.com Silver](https://about.gitlab.com/pricing/) or higher tiers, marks a project for - deletion. The deletion will happen 7 days later by default, but this can be changed in the - [instance settings](../../admin_area/settings/visibility_and_access_controls.md#default-deletion-adjourned-period-premium-only). +- From [GitLab 13.2](https://gitlab.com/gitlab-org/gitlab/-/issues/220382) on [Premium or Silver](https://about.gitlab.com/pricing/) or higher tiers, +group admins can [configure](../../group/index.md#enabling-delayed-project-removal-premium) projects within a group +to be deleted after a delayed period. +When enabled, actual deletion happens after number of days +specified in [instance settings](../../admin_area/settings/visibility_and_access_controls.md#default-deletion-adjourned-period-premium-only). + +CAUTION: **Warning:** +The default behavior of [Delayed Project deletion](https://gitlab.com/gitlab-org/gitlab/-/issues/32935) in GitLab 12.6 was changed to +[Immediate deletion](https://gitlab.com/gitlab-org/gitlab/-/issues/220382) in GitLab 13.2. #### Restore a project **(PREMIUM)** @@ -269,7 +274,7 @@ Configure Error Tracking to discover and view [Sentry errors within GitLab](../o ### Jaeger tracing **(ULTIMATE)** -Add the URL of a Jaeger server to allow your users to [easily access the Jaeger UI from within GitLab](../operations/tracing.md). +Add the URL of a Jaeger server to allow your users to [easily access the Jaeger UI from within GitLab](../../../operations/tracing.md). ### Status Page diff --git a/doc/user/project/static_site_editor/index.md b/doc/user/project/static_site_editor/index.md index 15c3bd5522e..8653bb5001a 100644 --- a/doc/user/project/static_site_editor/index.md +++ b/doc/user/project/static_site_editor/index.md @@ -9,6 +9,7 @@ description: "The static site editor enables users to edit content on static web > - WYSIWYG editor [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/214559) in GitLab 13.0. > - Support for adding images through the WYSIWYG editor [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/216640) in GitLab 13.1. > - Markdown front matter hidden on the WYSIWYG editor [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/216834) in GitLab 13.1. +> - Support for `*.md.erb` files [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/223171) in GitLab 13.2. DANGER: **Danger:** In GitLab 13.0, we [introduced breaking changes](https://gitlab.com/gitlab-org/gitlab/-/issues/213282) @@ -100,5 +101,4 @@ company and a new feature has been added to the company product. ## Limitations -- Currently, the Static Site Editor only works for files ending in `.md`. For example, it will not work for a file `index.html.md.erb` while it works for `index.html.md`. - The Static Site Editor still cannot be quickly added to existing Middleman sites. Follow this [epic](https://gitlab.com/groups/gitlab-org/-/epics/2784) for updates. diff --git a/doc/user/project/status_page/index.md b/doc/user/project/status_page/index.md index ec0a79583d5..1c885ae043f 100644 --- a/doc/user/project/status_page/index.md +++ b/doc/user/project/status_page/index.md @@ -34,10 +34,12 @@ Setting up a Status Page is pretty painless but there are a few things you need To use GitLab Status Page you first need to set up your account details for your cloud provider in the operations settings page. Today, only AWS is supported. -1. Within your AWS account, create an AWS access key. -1. Add the following permissions policies: +#### AWS Setup + +1. Within your AWS acccout, create two new IAM policies. - [Create bucket](https://gitlab.com/gitlab-org/status-page/-/blob/master/deploy/etc/s3_create_policy.json). - [Update bucket contents](https://gitlab.com/gitlab-org/status-page/-/blob/master/deploy/etc/s3_update_bucket_policy.json) (Remember replace `S3_BUCKET_NAME` with your bucket name). +1. Create a new AWS access key with the permissions policies created in the first step. ### Status Page project diff --git a/doc/user/project/web_ide/index.md b/doc/user/project/web_ide/index.md index ce20f2308e6..81f36317b0c 100644 --- a/doc/user/project/web_ide/index.md +++ b/doc/user/project/web_ide/index.md @@ -24,6 +24,8 @@ file path fragments to start seeing results. ## Syntax highlighting +> Support for `.gitlab.ci.yml` validation [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/218472) in GitLab 13.2. + As expected from an IDE, syntax highlighting for many languages within the Web IDE will make your direct editing even easier. @@ -35,10 +37,21 @@ The Web IDE currently provides: - IntelliSense and validation support (displaying errors and warnings, providing smart completions, formatting, and outlining) for some languages. For example: TypeScript, JavaScript, CSS, LESS, SCSS, JSON, and HTML. +- Validation support for certain JSON and YAML files using schemas based on the + [JSON Schema Store](https://www.schemastore.org/json/). This feature + is only supported for the `.gitlab-ci.yml` file. + + NOTE: **Note:** + Validation support based on schemas is hidden behind + the feature flag `:schema_linting` on self-managed installations. To enable the + feature, you can [turn on the feature flag in Rails console](../../../administration/feature_flags.md#how-to-enable-and-disable-features-behind-flags). Because the Web IDE is based on the [Monaco Editor](https://microsoft.github.io/monaco-editor/), you can find a more complete list of supported languages in the -[Monaco languages](https://github.com/Microsoft/monaco-languages) repository. +[Monaco languages](https://github.com/Microsoft/monaco-languages) repository. Under the hood, +Monaco uses the [Monarch](https://microsoft.github.io/monaco-editor/monarch.html) library for syntax highlighting. + +If you are missing Syntax Highlighting support for any language, we prepared a short guide on how to [add support for a missing language Syntax Highlighting.](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/assets/javascripts/ide/lib/languages/README.md) NOTE: **Note:** Single file editing is based on the [Ace Editor](https://ace.c9.io). @@ -210,16 +223,20 @@ to work: - The Runner needs to have [`[session_server]` configured properly](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-session_server-section). + This section requires at least a `session_timeout` value (which defaults to 1800 + seconds) and a `listen_address` value. If `advertise_address` is not defined, `listen_address` is used. - If you are using a reverse proxy with your GitLab instance, web terminals need to be [enabled](../../../administration/integration/terminal.md#enabling-and-disabling-terminal-support). **(ULTIMATE ONLY)** If you have the terminal open and the job has finished with its tasks, the terminal will block the job from finishing for the duration configured in -[`[session_server].terminal_max_retention_time`](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-session_server-section) +[`[session_server].session_timeout`](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-session_server-section) until you close the terminal window. -NOTE: **Note:** Not all executors are -[supported](https://docs.gitlab.com/runner/executors/#compatibility-chart) +NOTE: **Note:** +Not all executors are +[supported](https://docs.gitlab.com/runner/executors/#compatibility-chart). +The [File Sync](#file-syncing-to-web-terminal) feature is supported on Kubernetes runners only. ### Web IDE configuration file @@ -243,6 +260,8 @@ In the code below there is an example of this configuration file: ```yaml terminal: + # This can be any image that has the necessary runtime environment for your project. + image: node:10-alpine before_script: - apt-get update script: sleep 60 diff --git a/doc/user/project/wiki/img/wiki_page_diffs_v13_2.png b/doc/user/project/wiki/img/wiki_page_diffs_v13_2.png Binary files differnew file mode 100644 index 00000000000..261e6e66c97 --- /dev/null +++ b/doc/user/project/wiki/img/wiki_page_diffs_v13_2.png diff --git a/doc/user/project/wiki/img/wiki_page_history.png b/doc/user/project/wiki/img/wiki_page_history.png Binary files differindex 5a1ae295ed2..0cef2345e27 100644 --- a/doc/user/project/wiki/img/wiki_page_history.png +++ b/doc/user/project/wiki/img/wiki_page_history.png diff --git a/doc/user/project/wiki/index.md b/doc/user/project/wiki/index.md index 82dbeb0ff7e..9044ee0765f 100644 --- a/doc/user/project/wiki/index.md +++ b/doc/user/project/wiki/index.md @@ -134,47 +134,68 @@ The changes of a wiki page over time are recorded in the wiki's Git repository, and you can view them by clicking the **Page history** button. From the history page you can see the revision of the page (Git commit SHA), its -author, the commit message, when it was last updated, and the page markup format. +author, the commit message, and when it was last updated. To see how a previous version of the page looked like, click on a revision -number. +number in the **Page version** column.  +### Viewing the changes between page versions + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/15242) in GitLab 13.2. + +Similar to versioned diff file views, you can see the changes made in a given Wiki page version: + +1. Navigate to the Wiki page you're interested in. +1. Click on **Page history** to see all page versions. +1. Click on the commit message in the **Changes** column for the version you're interested in: + +  + ## Wiki activity records -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/14902) in GitLab 12.10. -> - It's deployed behind a feature flag, disabled by default. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/14902) in **GitLab 12.10.** +> - Git events were [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/216014) in **GitLab 13.0.** > - It's enabled on GitLab.com. -> - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enable-or-disable-wiki-events-core-only). **(CORE ONLY)** +> - Git access activity creation is managed by a feature flag. +> - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enable-or-disable-wiki-events-in-git-core-only). **(CORE ONLY)** Wiki events (creation, deletion, and updates) are tracked by GitLab and displayed on the [user profile](../../profile/index.md#user-profile), [group](../../group/index.md#view-group-activity), and [project](../index.md#project-activity) activity pages. -### Limitations - -Only edits made in the browser or through the API have their activity recorded. -Edits made and pushed through Git are not currently listed in the activity list. +### Enable or disable Wiki events in Git **(CORE ONLY)** -### Enable or disable Wiki Events **(CORE ONLY)** - -Wiki event activity is under development and not ready for production use. It is +Tracking wiki events through Git is under development and not ready for production use. It is deployed behind a feature flag that is **disabled by default**. -[GitLab administrators with access to the GitLab Rails console](../../../administration/troubleshooting/navigating_gitlab_via_rails_console.md#starting-a-rails-console-session) -can enable it for your instance. You're welcome to test it, but use it at your -own risk. +[GitLab administrators with access to the GitLab Rails console](../../../administration/feature_flags.md) +can enable it for your instance. To enable it: ```ruby -Feature.enable(:wiki_events) +Feature.enable(:wiki_events_on_git_push) +``` + +To enable for just a particular project: + +```ruby +project = Project.find_by_full_path('your-group/your-project') +Feature.enable(:wiki_events_on_git_push, project) ``` To disable it: ```ruby -Feature.disable(:wiki_events) +Feature.disable(:wiki_events_on_git_push) +``` + +To disable for just a particular project: + +```ruby +project = Project.find_by_full_path('your-group/your-project') +Feature.disable(:wiki_events_on_git_push, project) ``` ## Adding and editing wiki pages locally diff --git a/doc/user/search/advanced_global_search.md b/doc/user/search/advanced_global_search.md index eaa57395b8f..0613e9b8e34 100644 --- a/doc/user/search/advanced_global_search.md +++ b/doc/user/search/advanced_global_search.md @@ -1,11 +1,9 @@ -# Advanced Global Search **(STARTER ONLY)** +# Advanced Global Search **(STARTER)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/109) in GitLab [Starter](https://about.gitlab.com/pricing/) 8.4. NOTE: **GitLab.com availability:** -Advanced Global Search (powered by Elasticsearch) is not yet available on GitLab.com. -It will be progressively enabled for all paid groups in Q3 of 2020. -[Follow this epic](https://gitlab.com/groups/gitlab-com/-/epics/649) for the latest updates on the timeline. +Advanced Global Search (powered by Elasticsearch) is enabled for Bronze and above on GitLab.com since 2020-07-10. Leverage Elasticsearch for faster, more advanced code search across your entire GitLab instance. @@ -22,7 +20,6 @@ now search for code within other teams that can help your own project. GitLab leverages the search capabilities of [Elasticsearch](https://www.elastic.co/elasticsearch/) and enables it when searching in: -- GitLab application - Projects - Repositories - Commits diff --git a/doc/user/search/advanced_search_syntax.md b/doc/user/search/advanced_search_syntax.md index 1ca9649b581..5dc1f8fe779 100644 --- a/doc/user/search/advanced_search_syntax.md +++ b/doc/user/search/advanced_search_syntax.md @@ -1,13 +1,9 @@ -# Advanced Syntax Search **(STARTER ONLY)** +# Advanced Syntax Search **(STARTER)** -> **Notes:** -> > - Introduced in [GitLab Enterprise Starter](https://about.gitlab.com/pricing/) 9.2 NOTE: **GitLab.com availability:** -Advanced Global Search (powered by Elasticsearch) is not yet available on GitLab.com. -It will be progressively enabled for all paid groups in Q3 of 2020. -[Follow this epic](https://gitlab.com/groups/gitlab-com/-/epics/649) for the latest updates on the timeline. +Advanced Global Search (powered by Elasticsearch) is enabled for Bronze and above on GitLab.com since 2020-07-10. Use advanced queries for more targeted search results. diff --git a/doc/user/shortcuts.md b/doc/user/shortcuts.md index efa374cf1c3..314fe367ca6 100644 --- a/doc/user/shortcuts.md +++ b/doc/user/shortcuts.md @@ -78,10 +78,11 @@ These shortcuts are available when viewing issues and merge requests. | <kbd>m</kbd> | Change milestone. | | <kbd>l</kbd> | Change label. | | <kbd>r</kbd> | Start writing a comment. If any text is selected, it will be quoted in the comment. Can't be used to reply within a thread. | -| <kbd>n</kbd> | Move to next unresolved discussion (Merge requests only). | -| <kbd>p</kbd> | Move to previous unresolved discussion (Merge requests only). | -| <kbd>]</kbd> or <kbd>j</kbd> | Move to next file (Merge requests only). | -| <kbd>[</kbd> or <kbd>k</kbd> | Move to previous file (Merge requests only). | +| <kbd>n</kbd> | Move to next unresolved discussion (merge requests only). | +| <kbd>p</kbd> | Move to previous unresolved discussion (merge requests only). | +| <kbd>]</kbd> or <kbd>j</kbd> | Move to next file (merge requests only). | +| <kbd>[</kbd> or <kbd>k</kbd> | Move to previous file (merge requests only). | +| <kbd>b</kbd> | Copy source branch name (merge requests only). | ### Project Files diff --git a/doc/user/snippets.md b/doc/user/snippets.md index 68e5c5ac92c..4d8f47ee3be 100644 --- a/doc/user/snippets.md +++ b/doc/user/snippets.md @@ -96,6 +96,14 @@ This allows you to have a local copy of the snippet's repository and make changes as needed. You can commit those changes and push them to the remote master branch. +### Reduce snippets repository size + +Since versioned Snippets are considered as part of the [namespace storage size](../user/admin_area/settings/account_and_limit_settings.md), +it's recommended to keep snippets' repositories as compact as possible. + +For more information about tools to compact repositories, +see the documentation on [reducing repository size](../user/project/repository/reducing_the_repo_size_using_git.md). + ### Limitations - Binary files are not supported. diff --git a/doc/user/todos.md b/doc/user/todos.md index 5d3e3e62652..ef0e75bc197 100644 --- a/doc/user/todos.md +++ b/doc/user/todos.md @@ -42,8 +42,7 @@ A To Do appears on your To-Do List when: - You are `@mentioned` in a comment on a: - Commit - Design -- A job in the CI pipeline running for your merge request failed, but this - job is not allowed to fail +- The CI/CD pipeline for your merge request failed - An open merge request becomes unmergeable due to conflict, and you are either: - The author - Have set it to automatically merge once the pipeline succeeds |
