diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-10-21 07:08:36 +0000 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-10-21 07:08:36 +0000 |
| commit | 48aff82709769b098321c738f3444b9bdaa694c6 (patch) | |
| tree | e00c7c43e2d9b603a5a6af576b1685e400410dee /doc | |
| parent | 879f5329ee916a948223f8f43d77fba4da6cd028 (diff) | |
| download | gitlab-ce-48aff82709769b098321c738f3444b9bdaa694c6.tar.gz | |
Add latest changes from gitlab-org/gitlab@13-5-stable-eev13.5.0-rc42
Diffstat (limited to 'doc')
963 files changed, 25612 insertions, 11780 deletions
diff --git a/doc/.vale/gitlab/Acronyms.yml b/doc/.vale/gitlab/Acronyms.yml index d26ce9810d7..53690138300 100644 --- a/doc/.vale/gitlab/Acronyms.yml +++ b/doc/.vale/gitlab/Acronyms.yml @@ -44,6 +44,7 @@ exceptions: - IBM - IDE - IID + - IMAP - IRC - ISO - JSON @@ -65,6 +66,7 @@ exceptions: - POST - PUT - RAM + - REST - RPC - RSA - RSS diff --git a/doc/.vale/gitlab/Admin.yml b/doc/.vale/gitlab/Admin.yml new file mode 100644 index 00000000000..27a703c30c3 --- /dev/null +++ b/doc/.vale/gitlab/Admin.yml @@ -0,0 +1,13 @@ +--- +# Warning: gitlab.Admin +# +# You should not use "admin", but "Admin Area" is OK. +# +# For a list of all options, see https://errata-ai.gitbook.io/vale/getting-started/styles +extends: substitution +message: 'Use "administration", "administrator", "administer", or "Admin Area" instead of "admin" or "admin area".' +link: https://docs.gitlab.com/ee/development/documentation/styleguide.html +level: warning +ignorecase: true +swap: + 'admin ?\w*': '(?:Admin Area|[Aa]dminist(ration|rator|er))' diff --git a/doc/.vale/gitlab/InclusionAbleism.yml b/doc/.vale/gitlab/InclusionAbleism.yml new file mode 100644 index 00000000000..29b26547130 --- /dev/null +++ b/doc/.vale/gitlab/InclusionAbleism.yml @@ -0,0 +1,14 @@ +--- +# Suggestion: gitlab.InclusionAbleism +# +# Suggests alternatives for words that foster ableism. +# +# For a list of all options, see https://errata-ai.gitbook.io/vale/getting-started/styles +extends: substitution +message: 'Use inclusive language. Consider "%s" instead of "%s".' +link: https://docs.gitlab.com/ee/development/documentation/styleguide.html#inclusive-language +level: suggestion +ignorecase: true +swap: + sanity (?:check|test): check for completeness + dummy: placeholder, sample, fake diff --git a/doc/.vale/gitlab/InclusionCultural.yml b/doc/.vale/gitlab/InclusionCultural.yml new file mode 100644 index 00000000000..5bfad84f100 --- /dev/null +++ b/doc/.vale/gitlab/InclusionCultural.yml @@ -0,0 +1,16 @@ +--- +# Warning: gitlab.InclusionCultural +# +# Suggests alternatives for words that are culturally inappropriate. +# +# For a list of all options, see https://errata-ai.gitbook.io/vale/getting-started/styles +extends: substitution +message: 'Use inclusive language. Consider "%s" instead of "%s".' +link: https://docs.gitlab.com/ee/development/documentation/styleguide.html#inclusive-language +level: warning +ignorecase: true +swap: + blacklist(?:ed|ing|s)?: denylist + whitelist(?:ed|ing|s)?: allowlist + master: primary, main + slave: secondary diff --git a/doc/.vale/gitlab/InclusionGender.yml b/doc/.vale/gitlab/InclusionGender.yml new file mode 100644 index 00000000000..389d76d7a24 --- /dev/null +++ b/doc/.vale/gitlab/InclusionGender.yml @@ -0,0 +1,18 @@ +--- +# Suggestion: gitlab.InclusionGender +# +# Suggests alternatives for words that are gender-specific. +# +# For a list of all options, see https://errata-ai.gitbook.io/vale/getting-started/styles +extends: substitution +message: 'Use inclusive language. Consider "%s" instead of "%s".' +link: https://docs.gitlab.com/ee/development/documentation/styleguide.html#inclusive-language +level: suggestion +ignorecase: true +swap: + mankind: humanity, people + manpower: GitLab team members + he: they + his: their + she: they + hers: their diff --git a/doc/.vale/gitlab/OutdatedVersions.yml b/doc/.vale/gitlab/OutdatedVersions.yml index 1bc0bf58f90..2ce61ee5798 100644 --- a/doc/.vale/gitlab/OutdatedVersions.yml +++ b/doc/.vale/gitlab/OutdatedVersions.yml @@ -7,7 +7,7 @@ extends: existence message: 'Can this reference to "%s" be refactored?' link: https://docs.gitlab.com/ee/development/documentation/styleguide.html#importance-of-referencing-gitlab-versions-and-tiers -level: warning +level: suggestion nonword: true ignorecase: true tokens: diff --git a/doc/.vale/gitlab/SubstitutionWarning.yml b/doc/.vale/gitlab/SubstitutionWarning.yml index 5324a48e38c..68313a37e7d 100644 --- a/doc/.vale/gitlab/SubstitutionWarning.yml +++ b/doc/.vale/gitlab/SubstitutionWarning.yml @@ -11,8 +11,6 @@ link: https://about.gitlab.com/handbook/communication/#top-misused-terms level: warning ignorecase: true swap: - admin: administrator - blacklist(ed|ing)?: denylist code base: codebase config: configuration distro: distribution @@ -20,4 +18,3 @@ swap: filesystem: file system info: information repo: repository - whitelist(ed|ing)?: allowlist diff --git a/doc/.vale/gitlab/Substitutions.yml b/doc/.vale/gitlab/Substitutions.yml index 77536ea0b33..704c64f1fbd 100644 --- a/doc/.vale/gitlab/Substitutions.yml +++ b/doc/.vale/gitlab/Substitutions.yml @@ -13,6 +13,7 @@ ignorecase: true swap: frontmatter: front matter GitLabber: GitLab team member + GitLab-shell: GitLab Shell gitlab omnibus: Omnibus GitLab param: parameter params: parameters diff --git a/doc/.vale/gitlab/spelling-exceptions.txt b/doc/.vale/gitlab/spelling-exceptions.txt index bf816afdfab..c0a85fc6b70 100644 --- a/doc/.vale/gitlab/spelling-exceptions.txt +++ b/doc/.vale/gitlab/spelling-exceptions.txt @@ -61,6 +61,7 @@ buildpacks bundler bundlers burndown +burnup cacheable CAS CentOS @@ -142,8 +143,10 @@ Facebook failover failovers failsafe +Falco fastlane favicon +Figma Filebeat Fio firewalled @@ -166,6 +169,7 @@ Gitleaks Gitter globals Gmail +Gollum Google Gosec Gradle @@ -220,6 +224,7 @@ Kibana Kinesis Knative Kramdown +Kubecost kubectl Kubernetes Kubesec @@ -253,6 +258,7 @@ memoize memoized memoizing Memorystore +mergeability mergeable Microsoft middleware @@ -275,6 +281,7 @@ mockups ModSecurity monorepo monorepos +multiline mutex nameserver nameservers @@ -298,6 +305,7 @@ OmniAuth onboarding OpenID OpenShift +Opsgenie Packagist parallelization parallelizations @@ -305,6 +313,7 @@ passwordless Patroni performant PgBouncer +Phabricator phaser phasers Pipfile @@ -345,6 +354,7 @@ Python Qualys Rackspace Raspbian +Rdoc reachability rebase rebased @@ -365,6 +375,8 @@ reindex reindexed reindexes reindexing +reinitialize +reinitializing relicensing remediations repmgr @@ -382,6 +394,7 @@ reusability reverified reverifies reverify +RHEL rollout rollouts rsync @@ -423,7 +436,10 @@ smartcard smartcards SMTP Sobelow +Solarized Sourcegraph +sparkline +sparklines spidering Splunk SpotBugs @@ -439,6 +455,8 @@ subfolder subfolders subgraph subgraphs +subkey +subkeys sublicense sublicensed sublicenses @@ -455,8 +473,10 @@ substring substrings subtree subtrees +sudo syslog tcpdump +Thanos Tiller timecop todos @@ -479,6 +499,7 @@ unarchived unarchives unarchiving unassign +unassigning unassigns uncheck unchecked @@ -510,6 +531,9 @@ unprioritized unprotect unprotected unprotects +unprovision +unprovisioned +unprovisions unpublish unpublished unpublishes diff --git a/doc/README.md b/doc/README.md index efae2cdd3ff..09638bb4ce8 100644 --- a/doc/README.md +++ b/doc/README.md @@ -85,14 +85,14 @@ The following sections provide links to documentation for each DevOps stage: ### Manage -GitLab provides statistics and insight into ways you can maximize the value of GitLab in your organization. +GitLab provides statistics and insights into ways you can maximize the value of GitLab in your organization. The following documentation relates to the DevOps **Manage** stage: | Manage topics | Description | |:--------------------------------------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | [Authentication and<br/>Authorization](administration/auth/README.md) **(CORE ONLY)** | Supported authentication and authorization providers. | -| [GitLab Value Stream Analytics](user/project/cycle_analytics.md) | Measure the time it takes to go from an [idea to production](https://about.gitlab.com/blog/2016/08/05/continuous-integration-delivery-and-deployment-with-gitlab/#from-idea-to-production-with-gitlab) for each project you have. | +| [GitLab Value Stream Analytics](user/analytics/value_stream_analytics.md) | Measure the time it takes to go from an [idea to production](https://about.gitlab.com/blog/2016/08/05/continuous-integration-delivery-and-deployment-with-gitlab/#from-idea-to-production-with-gitlab) for each project you have. | | [Instance-level Analytics](user/admin_area/analytics/index.md) | Discover statistics on how many GitLab features you use and user activity. | <div align="right"> @@ -121,7 +121,7 @@ The following documentation relates to the DevOps **Plan** stage: | [Milestones](user/project/milestones/index.md) | Set milestones for delivery of issues and merge requests, with optional due date. | | [Project Issue Board](user/project/issue_board.md) | Display issues on a Scrum or Kanban board. | | [Quick Actions](user/project/quick_actions.md) | Shortcuts for common actions on issues or merge requests, replacing the need to click buttons or use dropdowns in GitLab's UI. | -| [Related Issues](user/project/issues/related_issues.md) **(STARTER)** | Create a relationship between issues. | +| [Related Issues](user/project/issues/related_issues.md) | Create a relationship between issues. | | [Requirements Management](user/project/requirements/index.md) **(ULTIMATE)** | Check your products against a set of criteria. | | [Roadmap](user/group/roadmap/index.md) **(ULTIMATE)** | Visualize epic timelines. | | [Service Desk](user/project/service_desk.md) | A simple way to allow people to create issues in your GitLab instance without needing their own user account. | @@ -218,7 +218,7 @@ The following documentation relates to the DevOps **Create** stage: | [GitLab GraphQL API](api/graphql/index.md) | Integrate with GitLab using our GraphQL API. | | [GitLab Integration](integration/README.md) | Integrate with multiple third-party services with GitLab to allow external issue trackers and external authentication. | | [GitLab Webhooks](user/project/integrations/webhooks.md) | Let GitLab notify you when new code has been pushed to your project. | -| [Jira Development Panel](integration/jira_development_panel.md) **(PREMIUM)** | See GitLab information in the Jira Development Panel. | +| [Jira Development Panel](integration/jira_development_panel.md) | See GitLab information in the Jira Development Panel. | | [Integrations](user/project/integrations/overview.md) | Integrate a project with external services, such as CI and chat. | | [Trello Power-Up](integration/trello_power_up.md) | Integrate with GitLab's Trello Power-Up. | @@ -267,9 +267,7 @@ The following documentation relates to the DevOps **Package** stage: |:----------------------------------------------------------------|:-------------------------------------------------------| | [Container Registry](user/packages/container_registry/index.md) | The GitLab Container Registry enables every project in GitLab to have its own space to store [Docker](https://www.docker.com/) images. | | [Dependency Proxy](user/packages/dependency_proxy/index.md) **(PREMIUM)** | The GitLab Dependency Proxy sets up a local proxy for frequently used upstream images/packages. | -| [Conan Repository](user/packages/conan_repository/index.md) | The GitLab Conan Repository enables every project in GitLab to have its own space to store [Conan](https://conan.io/) packages. | -| [Maven Repository](user/packages/maven_repository/index.md) | The GitLab Maven Repository enables every project in GitLab to have its own space to store [Maven](https://maven.apache.org/) packages. | -| [NPM Registry](user/packages/npm_registry/index.md) | The GitLab NPM Registry enables every project in GitLab to have its own space to store [NPM](https://www.npmjs.com/) packages. | +| [Package Registry](user/packages/package_registry/index.md) | Use GitLab as a private or public registry for a variety of common package managers, including [NPM](user/packages/npm_registry/index.md), [Maven](user/packages/maven_repository/index.md), [PyPI](user/packages/pypi_repository/index.md), and others. You can also store generic files. | <div align="right"> <a type="button" class="btn btn-default" href="#overview"> @@ -295,7 +293,7 @@ The following documentation relates to the DevOps **Secure** stage: | [Dependency Scanning](user/application_security/dependency_scanning/index.md) **(ULTIMATE)** | Analyze your dependencies for known vulnerabilities. | | [Dynamic Application Security Testing (DAST)](user/application_security/dast/index.md) **(ULTIMATE)** | Analyze running web applications for known vulnerabilities. | | [Group Security Dashboard](user/application_security/security_dashboard/index.md#group-security-dashboard) **(ULTIMATE)** | View vulnerabilities in all the projects in a group and its subgroups. | -| [Instance Security Dashboard](user/application_security/security_dashboard/index.md#instance-security-dashboard) **(ULTIMATE)** | View vulnerabilities in all the projects you're interested in. | +| [Instance Security Center](user/application_security/security_dashboard/index.md#instance-security-center) **(ULTIMATE)** | View vulnerabilities in all the projects you're interested in. | | [License Compliance](user/compliance/license_compliance/index.md) **(ULTIMATE)** | Search your project's dependencies for their licenses. | | [Pipeline Security](user/application_security/security_dashboard/index.md#pipeline-security) **(ULTIMATE)** | View the security reports for your project's pipelines. | | [Project Security Dashboard](user/application_security/security_dashboard/index.md#project-security-dashboard) **(ULTIMATE)** | View the latest security reports for your project. | diff --git a/doc/administration/audit_events.md b/doc/administration/audit_events.md index 099346b2b0b..ac972e2e33e 100644 --- a/doc/administration/audit_events.md +++ b/doc/administration/audit_events.md @@ -9,7 +9,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w GitLab offers a way to view the changes made within the GitLab server for owners and administrators on a [paid plan](https://about.gitlab.com/pricing/). GitLab system administrators can also take advantage of the logs located on the -filesystem. See [the logs system documentation](logs.md) for more details. +file system. See [the logs system documentation](logs.md) for more details. ## Overview @@ -108,7 +108,7 @@ Server-wide audit logging introduces the ability to observe user actions across the entire instance of your GitLab server, making it easy to understand who changed what and when for audit purposes. -To view the server-wide admin log, visit **Admin Area > Monitoring > Audit Log**. +To view the server-wide administrator log, visit **Admin Area > Monitoring > Audit Log**. In addition to the group and project events, the following user actions are also recorded: @@ -126,6 +126,7 @@ recorded: - User was added ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/251) in GitLab 12.8) - User was blocked via Admin Area ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/251) in GitLab 12.8) - User was blocked via API ([introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/25872) in GitLab 12.9) +- Failed second-factor authentication attempt ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/16826) in GitLab 13.5) It's possible to filter particular actions by choosing an audit data type from the filter dropdown box. You can further filter by specific group, project, or user @@ -151,7 +152,7 @@ on adding these events into GitLab: The current architecture of audit events is not prepared to receive a very high amount of records. It may make the user interface for your project or audit logs very busy, and the disk space consumed by the -`audit_events` PostgreSQL table will increase considerably. It's disabled by default +`audit_events` PostgreSQL table may increase considerably. It's disabled by default to prevent performance degradations on GitLab instances with very high Git write traffic. In an upcoming release, Audit Logs for Git push events will be enabled @@ -172,6 +173,7 @@ the steps bellow. ```ruby Feature.enable(:repository_push_audit_event) + ``` ## Export to CSV **(PREMIUM ONLY)** @@ -183,6 +185,7 @@ the steps bellow. CAUTION: **Warning:** This feature might not be available to you. Check the **version history** note above for details. +If available, you can enable it with a [feature flag](#enable-or-disable-audit-log-export-to-csv). Export to CSV allows customers to export the current filter view of your audit log as a CSV file, diff --git a/doc/administration/auditor_users.md b/doc/administration/auditor_users.md index ace210183b2..c41065abd17 100644 --- a/doc/administration/auditor_users.md +++ b/doc/administration/auditor_users.md @@ -1,34 +1,39 @@ -# Auditor users **(PREMIUM ONLY)** +--- +stage: none +group: unassigned +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- ->[Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/998) in [GitLab Premium](https://about.gitlab.com/pricing/) 8.17. +# Auditor users **(PREMIUM ONLY)** Auditor users are given read-only access to all projects, groups, and other resources on the GitLab instance. ## Overview -Auditor users can have full access to their own resources (projects, groups, -snippets, etc.), and read-only access to **all** other resources, except the -Admin Area. To put another way, they are just regular users (who can be added -to projects, create personal snippets, create milestones on their groups, etc.) -who also happen to have read-only access to all projects on the system that -they haven't been explicitly [given access](../user/permissions.md) to. +Auditor users are able to have both full access to their own resources +(including projects, groups, and snippets) and read-only access to _all_ other +resources, except the [Admin Area](../user/admin_area/index.md). These user +accounts are regular users who can be added to projects, create personal +snippets, and create milestones on their groups, while also having read-only +access to all projects on the server to which they haven't been explicitly +[given access](../user/permissions.md). The Auditor role is _not_ a read-only version of the Admin role. Auditor users -will not be able to access the project/group settings pages, or the Admin Area. +can't access the project or group settings pages, or the Admin Area. -To sum up, assuming you have logged-in as an Auditor user: +Assuming you have signed in as an Auditor user: - For a project the Auditor is not member of, the Auditor should have - read-only access. If the project is public or internal, they would have the - same access as the users that are not members of that project/group. + read-only access. If the project is public or internal, they have the same + access as users that aren't members of that project or group. - For a project the Auditor owns, the Auditor should have full access to everything. -- For a project the Auditor has been added to as a member, the Auditor should - have the same access as the [permissions](../user/permissions.md) they were given to. For example, if - they were added as a Developer, they could then push commits or comment on - issues. -- The Auditor cannot view the Admin Area, or perform any admin actions. +- For a project to which the Auditor is added as a member, the Auditor should + have the same access as their given [permissions](../user/permissions.md). + For example, if they were added as a Developer, they can push commits or + comment on issues. +- The Auditor can't view the Admin Area, or perform any admin actions. For more information about what an Auditor can or can't do, see the [Permissions and restrictions of an Auditor user](#permissions-and-restrictions-of-an-auditor-user) @@ -36,33 +41,37 @@ section. ## Use cases -1. Your compliance department wants to run tests against the entire GitLab base - to ensure users are complying with password, credit card, and other sensitive - data policies. With Auditor users, this can be achieved very easily without - resulting to tactics like giving a user admin rights or having to use the API - to add them to all projects. -1. If particular users need visibility or access to most of all projects in - your GitLab instance, instead of manually adding the user to all projects, - you can simply create an Auditor user and share the credentials with those - that you want to grant access to. +The following use cases describe some situations where Auditor users could be +helpful: + +- Your compliance department wants to run tests against the entire GitLab base + to ensure users are complying with password, credit card, and other sensitive + data policies. With Auditor users, this can be achieved very without having + to give them user admin rights or using the API to add them to all projects. +- If particular users need visibility or access to most of all projects in + your GitLab instance, instead of manually adding the user to all projects, + you can create an Auditor user and then share the credentials with those users + to which you want to grant access. ## Adding an Auditor user +To create a new Auditor user: + 1. Create a new user or edit an existing one by navigating to - **Admin Area > Users**. You will find the option of the access level under + **Admin Area > Users**. You will find the option of the access level in the 'Access' section.  -1. Click **Save changes** or **Create user** for the changes to take effect. +1. Select **Save changes** or **Create user** for the changes to take effect. -To revoke the Auditor permissions from a user, simply make them a Regular user -following the same steps as above. +To revoke Auditor permissions from a user, make them a regular user by +following the previous steps. ## Permissions and restrictions of an Auditor user An Auditor user should be able to access all projects and groups of a GitLab -instance, with the following permissions/restrictions: +instance, with the following permissions and restrictions: - Has read-only access to the API - Can access projects that are: @@ -70,15 +79,15 @@ instance, with the following permissions/restrictions: - Public - Internal - Can read all files in a repository -- Can read issues / MRs +- Can read issues and MRs - Can read project snippets - Cannot be Admin and Auditor at the same time - Cannot access the Admin Area -- In a group / project they're not a member of: +- In a group or project they're not a member of: - Cannot access project settings - Cannot access group settings - Cannot commit to repository - - Cannot create / comment on issues / MRs - - Cannot create/modify files from the Web UI + - Cannot create or comment on issues and MRs + - Cannot create or modify files from the Web UI - Cannot merge a merge request - Cannot create project snippets diff --git a/doc/administration/auth/README.md b/doc/administration/auth/README.md index 926a4abab7d..cf82454cfd2 100644 --- a/doc/administration/auth/README.md +++ b/doc/administration/auth/README.md @@ -18,7 +18,7 @@ providers: - [Azure](../../integration/azure.md) - [Bitbucket Cloud](../../integration/bitbucket.md) - [CAS](../../integration/cas.md) -- [Crowd](../../integration/crowd.md) +- [Crowd](crowd.md) - [Facebook](../../integration/facebook.md) - [GitHub](../../integration/github.md) - [GitLab.com](../../integration/gitlab.md) diff --git a/doc/administration/auth/ldap/index.md b/doc/administration/auth/ldap/index.md index 1dac098ec0c..3df85babc94 100644 --- a/doc/administration/auth/ldap/index.md +++ b/doc/administration/auth/ldap/index.md @@ -12,6 +12,7 @@ GitLab integrates with LDAP to support user authentication. This integration works with most LDAP-compliant directory servers, including: - Microsoft Active Directory + - [Microsoft Active Directory Trusts](https://docs.microsoft.com/en-us/previous-versions/windows/it-pro/windows-server-2008-R2-and-2008/cc771568(v=ws.10)) are not supported. - Apple Open Directory - Open LDAP - 389 Server @@ -21,9 +22,6 @@ Users added through LDAP take a [licensed seat](../../../subscriptions/self_mana GitLab Enterprise Editions (EE) include enhanced integration, including group membership syncing as well as multiple LDAP servers support. -NOTE: **Note:** -[Microsoft Active Directory Trusts](https://docs.microsoft.com/en-us/previous-versions/windows/it-pro/windows-server-2008-R2-and-2008/cc771568(v=ws.10)) are not supported. - ## Overview [LDAP](https://en.wikipedia.org/wiki/Lightweight_Directory_Access_Protocol) @@ -55,9 +53,8 @@ are already logged in or are using Git over SSH will still be able to access GitLab for up to one hour. Manually block the user in the GitLab Admin Area to immediately block all access. -NOTE: **Note:** GitLab Enterprise Edition Starter supports a -[configurable sync time](#adjusting-ldap-user-sync-schedule). +[configurable sync time](#adjusting-ldap-user-sync-schedule). **(STARTER)** ## Git password authentication **(CORE ONLY)** @@ -100,7 +97,6 @@ library. `start_tls` corresponds to StartTLS, not to be confused with regular TL Normally, if you specify `simple_tls` it will be on port 636, while `start_tls` (StartTLS) would be on port 389. `plain` also operates on port 389. Removed values: `tls` was replaced with `start_tls` and `ssl` was replaced with `simple_tls`. -NOTE: **Note:** LDAP users must have an email address set, regardless of whether it is used to sign-in. ### Example Configurations **(CORE ONLY)** @@ -120,7 +116,6 @@ gitlab_rails['ldap_servers'] = { 'verify_certificates' => true, 'bind_dn' => '_the_full_dn_of_the_user_you_will_bind_with', 'password' => '_the_password_of_the_bind_user', - 'encryption' => 'plain', 'verify_certificates' => true, 'tls_options' => { 'ca_file' => '', @@ -430,8 +425,7 @@ gitlab_rails['ldap_servers'] = { } ``` -NOTE: **Note:** -Any number of LDAP servers can be configured. However, make sure to use a unique naming convention for the `label` section of each entry as this will be the display name of the tab shown on the sign-in page. +If you configure multiple LDAP servers, use a unique naming convention for the `label` section of each entry. That label is used as the display name of the tab shown on the sign-in page. ## User sync **(STARTER ONLY)** @@ -445,11 +439,10 @@ The process executes the following access checks: blocked/disabled state). This will only be checked if `active_directory: true` is set in the LDAP configuration. -NOTE: **Note:** In Active Directory, a user is marked as disabled/blocked if the user account control attribute (`userAccountControl:1.2.840.113556.1.4.803`) -has bit 2 set. See <https://ctovswild.com/2009/09/03/bitmask-searches-in-ldap/> -for more information. +has bit 2 set. +For more information, see <https://ctovswild.com/2009/09/03/bitmask-searches-in-ldap/> The user will be set to `ldap_blocked` state in GitLab if the above conditions fail. This means the user will not be able to sign-in or push/pull code. @@ -460,8 +453,10 @@ The process will also update the following user information: - If `sync_ssh_keys` is set, SSH public keys. - If Kerberos is enabled, Kerberos identity. -NOTE: **Note:** -The LDAP sync process updates existing users while new users are created on first sign in. +The LDAP sync process: + +- Updates existing users. +- Creates new users on first sign in. ### Adjusting LDAP user sync schedule **(STARTER ONLY)** @@ -469,11 +464,13 @@ NOTE: **Note:** These are cron formatted values. You can use a crontab generator to create these values, for example <http://www.crontabgenerator.com/>. -By default, GitLab will run a worker once per day at 01:30 a.m. server time to +By default, GitLab runs a worker once per day at 01:30 a.m. server time to check and update GitLab users against LDAP. You can manually configure LDAP user sync times by setting the -following configuration values. The example below shows how to set LDAP user +following configuration values, in cron format. If needed, you can +use a [crontab generator](http://crontabgenerator.com). +The example below shows how to set LDAP user sync to run once every 12 hours at the top of the hour. **Omnibus installations** @@ -512,7 +509,7 @@ GitLab group membership to be automatically updated based on LDAP group members. The `group_base` configuration should be a base LDAP 'container', such as an 'organization' or 'organizational unit', that contains LDAP groups that should be available to GitLab. For example, `group_base` could be -`ou=groups,dc=example,dc=com`. In the config file it will look like the +`ou=groups,dc=example,dc=com`. In the configuration file it will look like the following. **Omnibus configuration** @@ -617,14 +614,12 @@ To enable it you need to: ### Adjusting LDAP group sync schedule **(STARTER ONLY)** -NOTE: **Note:** -These are cron formatted values. You can use a crontab generator to create -these values, for example [Crontab Generator](http://www.crontabgenerator.com/). - By default, GitLab runs a group sync process every hour, on the hour. +The values shown are in cron format. If needed, you can use a +[Crontab Generator](http://www.crontabgenerator.com). CAUTION: **Important:** -It's recommended that you do not start the sync process too frequently as this +Do not start the sync process too frequently as this could lead to multiple syncs running concurrently. This is primarily a concern for installations with a large number of LDAP users. Please review the [LDAP group sync benchmark metrics](#benchmarks) to see how @@ -727,7 +722,8 @@ Other LDAP servers should work, too. Active Directory also supports nested groups. Group sync will recursively resolve membership if `active_directory: true` is set in the configuration file. -NOTE: **Note:** +##### Nested group memberships + Nested group memberships are resolved only if the nested group is found within the configured `group_base`. For example, if GitLab sees a nested group with DN `cn=nested_group,ou=special_groups,dc=example,dc=com` but diff --git a/doc/administration/auth/ldap/ldap-troubleshooting.md b/doc/administration/auth/ldap/ldap-troubleshooting.md index 3d3ac124ac4..c6558bf1791 100644 --- a/doc/administration/auth/ldap/ldap-troubleshooting.md +++ b/doc/administration/auth/ldap/ldap-troubleshooting.md @@ -682,7 +682,7 @@ The rails console is a valuable tool to help debug LDAP problems. It allows you directly interact with the application by running commands and seeing how GitLab responds to them. -Please refer to [this guide](../../troubleshooting/debug.md#starting-a-rails-console-session) +Please refer to [this guide](../../operations/rails_console.md#starting-a-rails-console-session) for instructions on how to use the rails console. #### Enable debug output diff --git a/doc/administration/auth/smartcard.md b/doc/administration/auth/smartcard.md index 0ecf3ca090d..a696d0499a4 100644 --- a/doc/administration/auth/smartcard.md +++ b/doc/administration/auth/smartcard.md @@ -1,4 +1,7 @@ --- +stage: none +group: unassigned +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference --- diff --git a/doc/administration/compliance.md b/doc/administration/compliance.md index 237261f2567..44270283308 100644 --- a/doc/administration/compliance.md +++ b/doc/administration/compliance.md @@ -1,3 +1,9 @@ +--- +stage: none +group: unassigned +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Compliance features You can configure the following GitLab features to help ensure that your GitLab instance meets common compliance standards. Click a feature name for further documentation. diff --git a/doc/administration/consul.md b/doc/administration/consul.md index ae22d8bd4d0..4eed020c284 100644 --- a/doc/administration/consul.md +++ b/doc/administration/consul.md @@ -1,4 +1,7 @@ --- +stage: none +group: unassigned +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference --- diff --git a/doc/administration/database_load_balancing.md b/doc/administration/database_load_balancing.md index 36ef905fd90..e88f88a0427 100644 --- a/doc/administration/database_load_balancing.md +++ b/doc/administration/database_load_balancing.md @@ -1,3 +1,9 @@ +--- +stage: none +group: unassigned +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Database Load Balancing **(PREMIUM ONLY)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/1283) in [GitLab Premium](https://about.gitlab.com/pricing/) 9.0. @@ -128,7 +134,7 @@ production: disconnect_timeout: 120 ``` -Here the `discover:` section specifies the configuration details to use for +Here, the `discover:` section specifies the configuration details to use for service discovery. ### Configuration @@ -139,7 +145,7 @@ The following options can be set: |----------------------|---------------------------------------------------------------------------------------------------|-----------| | `nameserver` | The nameserver to use for looking up the DNS record. | localhost | | `record` | The record to look up. This option is required for service discovery to work. | | -| `record_type` | Optional record type to look up, this can be either A or SRV (since GitLab 12.3) | A | +| `record_type` | Optional record type to look up, this can be either A or SRV (GitLab 12.3 and later) | A | | `port` | The port of the nameserver. | 8600 | | `interval` | The minimum time in seconds between checking the DNS record. | 60 | | `disconnect_timeout` | The time in seconds after which an old connection is closed, after the list of hosts was updated. | 120 | diff --git a/doc/administration/environment_variables.md b/doc/administration/environment_variables.md index d48a47e9645..25bfc3c132d 100644 --- a/doc/administration/environment_variables.md +++ b/doc/administration/environment_variables.md @@ -1,77 +1,75 @@ --- -stage: Verify -group: Continuous Integration +stage: Enablement +group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference --- -# Environment Variables +# Environment variables GitLab exposes certain environment variables which can be used to override their defaults values. -People usually configure GitLab via `/etc/gitlab/gitlab.rb` for Omnibus +People usually configure GitLab with `/etc/gitlab/gitlab.rb` for Omnibus installations, or `gitlab.yml` for installations from source. -Below you will find the supported environment variables which you can use to -override certain values. +You can use the following environment variables to override certain values: ## Supported environment variables -Variable | Type | Description --------- | ---- | ----------- -`ENABLE_BOOTSNAP` | string | Enables Bootsnap for speeding up initial Rails boot (`1` to enable) -`GITLAB_CDN_HOST` | string | Sets the base URL for a CDN to serve static assets (e.g. `//mycdnsubdomain.fictional-cdn.com`) -`GITLAB_ROOT_PASSWORD` | string | Sets the password for the `root` user on installation -`GITLAB_HOST` | string | The full URL of the GitLab server (including `http://` or `https://`) -`RAILS_ENV` | string | The Rails environment; can be one of `production`, `development`, `staging` or `test` -`DATABASE_URL` | string | The database URL; is of the form: `postgresql://localhost/blog_development` -`GITLAB_EMAIL_FROM` | string | The e-mail address used in the "From" field in e-mails sent by GitLab -`GITLAB_EMAIL_DISPLAY_NAME` | string | The name used in the "From" field in e-mails sent by GitLab -`GITLAB_EMAIL_REPLY_TO` | string | The e-mail address used in the "Reply-To" field in e-mails sent by GitLab -`GITLAB_EMAIL_SUBJECT_SUFFIX` | string | The e-mail subject suffix used in e-mails sent by GitLab -`GITLAB_UNICORN_MEMORY_MIN` | integer | The minimum memory threshold (in bytes) for the Unicorn worker killer -`GITLAB_UNICORN_MEMORY_MAX` | integer | The maximum memory threshold (in bytes) for the Unicorn worker killer -`GITLAB_SHARED_RUNNERS_REGISTRATION_TOKEN` | string | Sets the initial registration token used for runners -`UNSTRUCTURED_RAILS_LOG` | string | Enables the unstructured log in addition to JSON logs (defaults to `true`) +| Variable | Type | Description | +|--------------------------------------------|---------|---------------------------------------------------------------------------------------------------------| +| `DATABASE_URL` | string | The database URL; is of the form: `postgresql://localhost/blog_development`. | +| `ENABLE_BOOTSNAP` | string | Enables Bootsnap for speeding up initial Rails boot (`1` to enable). | +| `GITLAB_CDN_HOST` | string | Sets the base URL for a CDN to serve static assets (for example, `//mycdnsubdomain.fictional-cdn.com`). | +| `GITLAB_EMAIL_DISPLAY_NAME` | string | The name used in the **From** field in emails sent by GitLab. | +| `GITLAB_EMAIL_FROM` | string | The email address used in the **From** field in emails sent by GitLab. | +| `GITLAB_EMAIL_REPLY_TO` | string | The email address used in the **Reply-To** field in emails sent by GitLab. | +| `GITLAB_EMAIL_SUBJECT_SUFFIX` | string | The email subject suffix used in emails sent by GitLab. | +| `GITLAB_HOST` | string | The full URL of the GitLab server (including `http://` or `https://`). | +| `GITLAB_ROOT_PASSWORD` | string | Sets the password for the `root` user on installation. | +| `GITLAB_SHARED_RUNNERS_REGISTRATION_TOKEN` | string | Sets the initial registration token used for runners. | +| `GITLAB_UNICORN_MEMORY_MAX` | integer | The maximum memory threshold (in bytes) for the [unicorn-worker-killer](operations/unicorn.md#unicorn-worker-killer). | +| `GITLAB_UNICORN_MEMORY_MIN` | integer | The minimum memory threshold (in bytes) for the [unicorn-worker-killer](operations/unicorn.md#unicorn-worker-killer). | +| `RAILS_ENV` | string | The Rails environment; can be one of `production`, `development`, `staging`, or `test`. | +| `UNSTRUCTURED_RAILS_LOG` | string | Enables the unstructured log in addition to JSON logs (defaults to `true`). | ## Complete database variables -The recommended way of specifying your database connection information is to set -the `DATABASE_URL` environment variable. This variable only holds connection -information (`adapter`, `database`, `username`, `password`, `host` and `port`), -but not behavior information (`encoding`, `pool`). If you don't want to use -`DATABASE_URL` and/or want to set database behavior information, you will have -to either: +The recommended method for specifying your database connection information is +to set the `DATABASE_URL` environment variable. This variable contains +connection information (`adapter`, `database`, `username`, `password`, `host`, +and `port`), but no behavior information (`encoding` or `pool`). If you don't +want to use `DATABASE_URL`, or want to set database behavior information, +either: -- copy our template file: `cp config/database.yml.env config/database.yml`, or -- set a value for some `GITLAB_DATABASE_XXX` variables +- Copy the template file, `cp config/database.yml.env config/database.yml`. +- Set a value for some `GITLAB_DATABASE_XXX` variables. The list of `GITLAB_DATABASE_XXX` variables that you can set is: -Variable | Default value | Overridden by `DATABASE_URL`? --------- | ------------- | ----------------------------- -`GITLAB_DATABASE_ADAPTER` | `postgresql` | Yes -`GITLAB_DATABASE_DATABASE` | `gitlab_#{ENV['RAILS_ENV']` | Yes -`GITLAB_DATABASE_USERNAME` | `root` | Yes -`GITLAB_DATABASE_PASSWORD` | None | Yes -`GITLAB_DATABASE_HOST` | `localhost` | Yes -`GITLAB_DATABASE_PORT` | `5432` | Yes -`GITLAB_DATABASE_ENCODING` | `unicode` | No -`GITLAB_DATABASE_POOL` | `10` | No +| Variable | Default value | Overridden by `DATABASE_URL`? | +|-----------------------------|--------------------------------|-------------------------------| +| `GITLAB_DATABASE_ADAPTER` | `postgresql` | **{check-circle}** Yes | +| `GITLAB_DATABASE_DATABASE` | `gitlab_#{ENV['RAILS_ENV']` | **{check-circle}** Yes | +| `GITLAB_DATABASE_ENCODING` | `unicode` | **{dotted-circle}** No | +| `GITLAB_DATABASE_HOST` | `localhost` | **{check-circle}** Yes | +| `GITLAB_DATABASE_PASSWORD` | _none_ | **{check-circle}** Yes | +| `GITLAB_DATABASE_POOL` | `10` | **{dotted-circle}** No | +| `GITLAB_DATABASE_PORT` | `5432` | **{check-circle}** Yes | +| `GITLAB_DATABASE_USERNAME` | `root` | **{check-circle}** Yes | ## Adding more variables -We welcome merge requests to make more settings configurable via variables. -Please make changes in the `config/initializers/1_settings.rb` file and stick -to the naming scheme `GITLAB_#{name in 1_settings.rb in upper case}`. +We welcome merge requests to make more settings configurable by using variables. +Make changes to the `config/initializers/1_settings.rb` file, and use the +naming scheme `GITLAB_#{name in 1_settings.rb in upper case}`. ## Omnibus configuration -To set environment variables, follow [these -instructions](https://docs.gitlab.com/omnibus/settings/environment-variables.html). +To set environment variables, follow [these instructions](https://docs.gitlab.com/omnibus/settings/environment-variables.html). It's possible to preconfigure the GitLab Docker image by adding the environment variable `GITLAB_OMNIBUS_CONFIG` to the `docker run` command. -For more information see the [Pre-configure Docker container](https://docs.gitlab.com/omnibus/docker/#pre-configure-docker-container) -section in the Omnibus documentation. +For more information, see the [Pre-configure Docker container](https://docs.gitlab.com/omnibus/docker/#pre-configure-docker-container) +section of the Omnibus GitLab documentation. diff --git a/doc/administration/feature_flags.md b/doc/administration/feature_flags.md index a8a14063f26..4129677f134 100644 --- a/doc/administration/feature_flags.md +++ b/doc/administration/feature_flags.md @@ -65,7 +65,7 @@ For installations from the source: sudo -u git -H bundle exec rails console -e production ``` -For details, see [starting a Rails console session](troubleshooting/debug.md#starting-a-rails-console-session). +For details, see [starting a Rails console session](operations/rails_console.md#starting-a-rails-console-session). ### Enable or disable the feature @@ -79,10 +79,10 @@ To enable a feature, run: Feature.enable(:<feature flag>) ``` -Example, to enable Evidence Collection: +Example, to enable a fictional feature flag named `my_awesome_feature`: ```ruby -Feature.enable(:release_evidence_collection) +Feature.enable(:my_awesome_feature) ``` To disable a feature, run: @@ -91,10 +91,10 @@ To disable a feature, run: Feature.disable(:<feature flag>) ``` -Example, to disable Evidence Collection: +Example, to disable a fictional feature flag named `my_awesome_feature`: ```ruby -Feature.disable(:release_evidence_collection) +Feature.disable(:my_awesome_feature) ``` Some feature flags can be enabled or disabled on a per project basis: @@ -112,18 +112,18 @@ Feature.enable(:product_analytics, Project.find(1234)) `Feature.enable` and `Feature.disable` always return `nil`, this is not an indication that the command failed: ```ruby -irb(main):001:0> Feature.enable(:release_evidence_collection) +irb(main):001:0> Feature.enable(:my_awesome_feature) => nil ``` -To check if a flag is enabled or disabled you can use `Feature.enabled?` or `Feature.disabled?`: +To check if a flag is enabled or disabled you can use `Feature.enabled?` or `Feature.disabled?`. For example, for a fictional feature flag named `my_awesome_feature`: ```ruby -Feature.enable(:release_evidence_collection) +Feature.enable(:my_awesome_feature) => nil -Feature.enabled?(:release_evidence_collection) +Feature.enabled?(:my_awesome_feature) => true -Feature.disabled?(:release_evidence_collection) +Feature.disabled?(:my_awesome_feature) => false ``` diff --git a/doc/administration/geo/disaster_recovery/index.md b/doc/administration/geo/disaster_recovery/index.md index 8862776ee1b..32b3558a51f 100644 --- a/doc/administration/geo/disaster_recovery/index.md +++ b/doc/administration/geo/disaster_recovery/index.md @@ -11,11 +11,11 @@ Geo replicates your database, your Git repositories, and few other assets. We will support and replicate more data in the future, that will enable you to failover with minimal effort, in a disaster situation. -See [Geo current limitations](../index.md#current-limitations) for more information. +See [Geo limitations](../index.md#limitations) for more information. CAUTION: **Warning:** Disaster recovery for multi-secondary configurations is in **Alpha**. -For the latest updates, check the [Disaster Recovery epic for complete maturity](https://gitlab.com/groups/gitlab-org/-/epics/590). +For the latest updates, check the [Disaster Recovery epic for complete maturity](https://gitlab.com/groups/gitlab-org/-/epics/590). Multi-secondary configurations require the complete re-synchronization and re-configuration of all non-promoted secondaries and will cause downtime. @@ -84,8 +84,8 @@ must disable the **primary** node. single recommendation. You may need to: - Reconfigure the load balancers. - - Change DNS records (for example, point the primary DNS record to the **secondary** - node in order to stop usage of the **primary** node). + - Change DNS records (for example, point the primary DNS record to the + **secondary** node to stop usage of the **primary** node). - Stop the virtual servers. - Block traffic through a firewall. - Revoke object storage permissions from the **primary** node. @@ -98,16 +98,16 @@ must disable the **primary** node. Note the following when promoting a secondary: -- If replication was paused on the secondary node, for example as a part of upgrading, - while you were running a version of GitLab lower than 13.4, you _must_ - [enable the node via the database](../replication/troubleshooting.md#while-promoting-the-secondary-i-got-an-error-activerecordrecordinvalid) +- If replication was paused on the secondary node (for example as a part of + upgrading, while you were running a version of GitLab earlier than 13.4), you + _must_ [enable the node by using the database](../replication/troubleshooting.md#message-activerecordrecordinvalid-validation-failed-enabled-geo-primary-node-cannot-be-disabled) before proceeding. - A new **secondary** should not be added at this time. If you want to add a new **secondary**, do this after you have completed the entire process of promoting the **secondary** to the **primary**. - If you encounter an `ActiveRecord::RecordInvalid: Validation failed: Name has already been taken` - error during this process, please read - [the troubleshooting advice](../replication/troubleshooting.md#fixing-errors-during-a-failover-or-when-promoting-a-secondary-to-a-primary-node). + error message during this process, for more information, see this + [troubleshooting advice](../replication/troubleshooting.md#fixing-errors-during-a-failover-or-when-promoting-a-secondary-to-a-primary-node). #### Promoting a **secondary** node running on a single machine @@ -120,6 +120,10 @@ Note the following when promoting a secondary: 1. Edit `/etc/gitlab/gitlab.rb` to reflect its new status as **primary** by removing any lines that enabled the `geo_secondary_role`: + Users of GitLab 13.5 or later can skip this step, due to the appropriate + roles being enabled or disabled during the promotion in the following + step. + ```ruby ## In pre-11.5 documentation, the role was enabled as follows. Remove this line. geo_secondary_role['enable'] = true @@ -129,7 +133,10 @@ Note the following when promoting a secondary: ``` 1. Promote the **secondary** node to the **primary** node. - + +DANGER: **Danger:** +In GitLab 13.2 and later versions, promoting a secondary node to a primary while the secondary is paused fails. We are [investigating the issue](https://gitlab.com/gitlab-org/gitlab/-/issues/225173). Do not pause replication before promoting a secondary. If the node is paused, please resume before promoting. + To promote the secondary node to primary along with preflight checks: ```shell @@ -139,7 +146,7 @@ Note the following when promoting a secondary: If you have already run the [preflight checks](planned_failover.md#preflight-checks) separately or don't want to run them, you can skip preflight checks with: ```shell - gitlab-ctl promote-to-primary-node --skip-preflight-check + gitlab-ctl promote-to-primary-node --skip-preflight-checks ``` You can also promote the secondary node to primary **without any further confirmation**, even when preflight checks fail: @@ -159,6 +166,9 @@ conjunction with multiple servers, as it can only perform changes on a **secondary** with only a single machine. Instead, you must do this manually. +DANGER: **Danger:** +In GitLab 13.2 and later versions, promoting a secondary node to a primary while the secondary is paused fails. We are [investigating the issue](https://gitlab.com/gitlab-org/gitlab/-/issues/225173). Do not pause replication before promoting a secondary. If the node is paused, please resume before promoting. + 1. SSH in to the database node in the **secondary** and trigger PostgreSQL to promote to read-write: @@ -201,6 +211,9 @@ an external PostgreSQL database, as it can only perform changes on a **secondary node with GitLab and the database on the same machine. As a result, a manual process is required: +DANGER: **Danger:** +In GitLab 13.2 and later versions, promoting a secondary node to a primary while the secondary is paused fails. We are [investigating the issue](https://gitlab.com/gitlab-org/gitlab/-/issues/225173). Do not pause replication before promoting a secondary. If the node is paused, please resume before promoting. + 1. Promote the replica database associated with the **secondary** site. This will set the database to read-write: - Amazon RDS - [Promoting a Read Replica to Be a Standalone DB Instance](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_ReadRepl.html#USER_ReadRepl.Promote) diff --git a/doc/administration/geo/disaster_recovery/planned_failover.md b/doc/administration/geo/disaster_recovery/planned_failover.md index 9b9c386652c..1238c4d8e2a 100644 --- a/doc/administration/geo/disaster_recovery/planned_failover.md +++ b/doc/administration/geo/disaster_recovery/planned_failover.md @@ -27,7 +27,7 @@ have a high degree of confidence in being able to perform them accurately. ## Not all data is automatically replicated -If you are using any GitLab features that Geo [doesn't support](../index.md#current-limitations), +If you are using any GitLab features that Geo [doesn't support](../index.md#limitations), you must make separate provisions to ensure that the **secondary** node has an up-to-date copy of any data associated with that feature. This may extend the required scheduled maintenance period significantly. diff --git a/doc/administration/geo/disaster_recovery/promotion_runbook.md b/doc/administration/geo/disaster_recovery/promotion_runbook.md index fb2353513df..7eb6ef01aee 100644 --- a/doc/administration/geo/disaster_recovery/promotion_runbook.md +++ b/doc/administration/geo/disaster_recovery/promotion_runbook.md @@ -1,269 +1,5 @@ --- -stage: Enablement -group: Geo -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers -type: howto +redirect_to: runbooks/planned_failover_single_node.md --- -CAUTION: **Caution:** -This runbook is in **alpha**. For complete, production-ready documentation, see the -[disaster recovery documentation](index.md). - -# Disaster Recovery (Geo) promotion runbooks **(PREMIUM ONLY)** - -## Geo planned failover runbook 1 - -| Component | Configuration | -| ----------- | --------------- | -| PostgreSQL | Omnibus-managed | -| Geo site | Single-node | -| Secondaries | One | - -This runbook will guide you through a planned failover of a single-node Geo site -with one secondary. The following general architecture is assumed: - -```mermaid -graph TD - subgraph main[Geo deployment] - subgraph Primary[Primary site] - Node_1[(GitLab node)] - end - subgraph Secondary1[Secondary site] - Node_2[(GitLab node)] - end - end -``` - -This guide will result in the following: - -1. An offline primary. -1. A promoted secondary that is now the new primary. - -What is not covered: - -1. Re-adding the old **primary** as a secondary. -1. Adding a new secondary. - -### Preparation - -NOTE: **Note:** -Before following any of those steps, make sure you have `root` access to the -**secondary** to promote it, since there isn't provided an automated way to -promote a Geo replica and perform a failover. - -On the **secondary** node, navigate to the **Admin Area > Geo** dashboard to -review its status. Replicated objects (shown in green) should be close to 100%, -and there should be no failures (shown in red). If a large proportion of -objects aren't yet replicated (shown in gray), consider giving the node more -time to complete. - - - -If any objects are failing to replicate, this should be investigated before -scheduling the maintenance window. After a planned failover, anything that -failed to replicate will be **lost**. - -You can use the -[Geo status API](../../../api/geo_nodes.md#retrieve-project-sync-or-verification-failures-that-occurred-on-the-current-node) -to review failed objects and the reasons for failure. -A common cause of replication failures is the data being missing on the -**primary** node - you can resolve these failures by restoring the data from backup, -or removing references to the missing data. - -The maintenance window won't end until Geo replication and verification is -completely finished. To keep the window as short as possible, you should -ensure these processes are close to 100% as possible during active use. - -If the **secondary** node is still replicating data from the **primary** node, -follow these steps to avoid unnecessary data loss: - -1. Until a [read-only mode](https://gitlab.com/gitlab-org/gitlab/-/issues/14609) - is implemented, updates must be prevented from happening manually to the - **primary**. Note that your **secondary** node still needs read-only - access to the **primary** node during the maintenance window: - - 1. At the scheduled time, using your cloud provider or your node's firewall, block - all HTTP, HTTPS and SSH traffic to/from the **primary** node, **except** for your IP and - the **secondary** node's IP. - - For instance, you can run the following commands on the **primary** node: - - ```shell - sudo iptables -A INPUT -p tcp -s <secondary_node_ip> --destination-port 22 -j ACCEPT - sudo iptables -A INPUT -p tcp -s <your_ip> --destination-port 22 -j ACCEPT - sudo iptables -A INPUT --destination-port 22 -j REJECT - - sudo iptables -A INPUT -p tcp -s <secondary_node_ip> --destination-port 80 -j ACCEPT - sudo iptables -A INPUT -p tcp -s <your_ip> --destination-port 80 -j ACCEPT - sudo iptables -A INPUT --tcp-dport 80 -j REJECT - - sudo iptables -A INPUT -p tcp -s <secondary_node_ip> --destination-port 443 -j ACCEPT - sudo iptables -A INPUT -p tcp -s <your_ip> --destination-port 443 -j ACCEPT - sudo iptables -A INPUT --tcp-dport 443 -j REJECT - ``` - - From this point, users will be unable to view their data or make changes on the - **primary** node. They will also be unable to log in to the **secondary** node. - However, existing sessions will work for the remainder of the maintenance period, and - public data will be accessible throughout. - - 1. Verify the **primary** node is blocked to HTTP traffic by visiting it in browser via - another IP. The server should refuse connection. - - 1. Verify the **primary** node is blocked to Git over SSH traffic by attempting to pull an - existing Git repository with an SSH remote URL. The server should refuse - connection. - - 1. On the **primary** node, disable non-Geo periodic background jobs by navigating - to **Admin Area > Monitoring > Background Jobs > Cron**, clicking `Disable All`, - and then clicking `Enable` for the `geo_sidekiq_cron_config_worker` cron job. - This job will re-enable several other cron jobs that are essential for planned - failover to complete successfully. - -1. Finish replicating and verifying all data: - - CAUTION: **Caution:** - Not all data is automatically replicated. Read more about - [what is excluded](planned_failover.md#not-all-data-is-automatically-replicated). - - 1. If you are manually replicating any - [data not managed by Geo](../replication/datatypes.md#limitations-on-replicationverification), - trigger the final replication process now. - 1. On the **primary** node, navigate to **Admin Area > Monitoring > Background Jobs > Queues** - and wait for all queues except those with `geo` in the name to drop to 0. - These queues contain work that has been submitted by your users; failing over - before it is completed will cause the work to be lost. - 1. On the **primary** node, navigate to **Admin Area > Geo** and wait for the - following conditions to be true of the **secondary** node you are failing over to: - - All replication meters to each 100% replicated, 0% failures. - - All verification meters reach 100% verified, 0% failures. - - Database replication lag is 0ms. - - The Geo log cursor is up to date (0 events behind). - - 1. On the **secondary** node, navigate to **Admin Area > Monitoring > Background Jobs > Queues** - and wait for all the `geo` queues to drop to 0 queued and 0 running jobs. - 1. On the **secondary** node, use [these instructions](../../raketasks/check.md) - to verify the integrity of CI artifacts, LFS objects, and uploads in file - storage. - - At this point, your **secondary** node will contain an up-to-date copy of everything the - **primary** node has, meaning nothing will be lost when you fail over. - -1. In this final step, you need to permanently disable the **primary** node. - - CAUTION: **Caution:** - When the **primary** node goes offline, there may be data saved on the **primary** node - that has not been replicated to the **secondary** node. This data should be treated - as lost if you proceed. - - TIP: **Tip:** - If you plan to [update the **primary** domain DNS record](index.md#step-4-optional-updating-the-primary-domain-dns-record), - you may wish to lower the TTL now to speed up propagation. - - When performing a failover, we want to avoid a split-brain situation where - writes can occur in two different GitLab instances. So to prepare for the - failover, you must disable the **primary** node: - - - If you have SSH access to the **primary** node, stop and disable GitLab: - - ```shell - sudo gitlab-ctl stop - ``` - - Prevent GitLab from starting up again if the server unexpectedly reboots: - - ```shell - sudo systemctl disable gitlab-runsvdir - ``` - - NOTE: **Note:** - (**CentOS only**) In CentOS 6 or older, there is no easy way to prevent GitLab from being - started if the machine reboots isn't available (see [Omnibus GitLab issue #3058](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/3058)). - It may be safest to uninstall the GitLab package completely with `sudo yum remove gitlab-ee`. - - NOTE: **Note:** - (**Ubuntu 14.04 LTS**) If you are using an older version of Ubuntu - or any other distribution based on the Upstart init system, you can prevent GitLab - from starting if the machine reboots as `root` with - `initctl stop gitlab-runsvvdir && echo 'manual' > /etc/init/gitlab-runsvdir.override && initctl reload-configuration`. - - - If you do not have SSH access to the **primary** node, take the machine offline and - prevent it from rebooting. Since there are many ways you may prefer to accomplish - this, we will avoid a single recommendation. You may need to: - - - Reconfigure the load balancers. - - Change DNS records (for example, point the **primary** DNS record to the **secondary** - node in order to stop usage of the **primary** node). - - Stop the virtual servers. - - Block traffic through a firewall. - - Revoke object storage permissions from the **primary** node. - - Physically disconnect a machine. - -### Promoting the **secondary** node - -Note the following when promoting a secondary: - -- A new **secondary** should not be added at this time. If you want to add a new - **secondary**, do this after you have completed the entire process of promoting - the **secondary** to the **primary**. -- If you encounter an `ActiveRecord::RecordInvalid: Validation failed: Name has already been taken` - error during this process, read - [the troubleshooting advice](../replication/troubleshooting.md#fixing-errors-during-a-failover-or-when-promoting-a-secondary-to-a-primary-node). - -To promote the secondary node: - -1. SSH in to your **secondary** node and login as root: - - ```shell - sudo -i - ``` - -1. Edit `/etc/gitlab/gitlab.rb` to reflect its new status as **primary** by - removing any lines that enabled the `geo_secondary_role`: - - ```ruby - ## In pre-11.5 documentation, the role was enabled as follows. Remove this line. - geo_secondary_role['enable'] = true - - ## In 11.5+ documentation, the role was enabled as follows. Remove this line. - roles ['geo_secondary_role'] - ``` - -1. Run the following command to list out all preflight checks and automatically - check if replication and verification are complete before scheduling a planned - failover to ensure the process will go smoothly: - - ```shell - gitlab-ctl promotion-preflight-checks - ``` - -1. Promote the **secondary**: - - ```shell - gitlab-ctl promote-to-primary-node - ``` - - If you have already run the [preflight checks](planned_failover.md#preflight-checks) - or don't want to run them, you can skip them: - - ```shell - gitlab-ctl promote-to-primary-node --skip-preflight-check - ``` - - You can also promote the secondary node to primary **without any further confirmation**, even when preflight checks fail: - - ```shell - sudo gitlab-ctl promote-to-primary-node --force - ``` - -1. Verify you can connect to the newly promoted **primary** node using the URL used - previously for the **secondary** node. - - If successful, the **secondary** node has now been promoted to the **primary** node. - -### Next steps - -To regain geographic redundancy as quickly as possible, you should -[add a new **secondary** node](../setup/index.md). To -do that, you can re-add the old **primary** as a new secondary and bring it back -online. +This document was moved to [another location](runbooks/planned_failover_single_node.md). diff --git a/doc/administration/geo/disaster_recovery/runbooks/planned_failover_multi_node.md b/doc/administration/geo/disaster_recovery/runbooks/planned_failover_multi_node.md new file mode 100644 index 00000000000..1e3bac0b354 --- /dev/null +++ b/doc/administration/geo/disaster_recovery/runbooks/planned_failover_multi_node.md @@ -0,0 +1,274 @@ +--- +stage: Enablement +group: Geo +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: howto +--- + +CAUTION: **Caution:** +This runbook is in **alpha**. For complete, production-ready documentation, see the +[disaster recovery documentation](../index.md). + +# Disaster Recovery (Geo) promotion runbooks **(PREMIUM ONLY)** + +## Geo planned failover for a multi-node configuration + +| Component | Configuration | +|-------------|-----------------| +| PostgreSQL | Omnibus-managed | +| Geo site | Multi-node | +| Secondaries | One | + +This runbook will guide you through a planned failover of a multi-node Geo site +with one secondary. The following [2000 user reference architecture](../../../../administration/reference_architectures/2k_users.md) is assumed: + +```mermaid +graph TD + subgraph main[Geo deployment] + subgraph Primary[Primary site, multi-node] + Node_1[Rails node 1] + Node_2[Rails node 2] + Node_3[PostgreSQL node] + Node_4[Gitaly node] + Node_5[Redis node] + Node_6[Monitoring node] + end + subgraph Secondary[Secondary site, multi-node] + Node_7[Rails node 1] + Node_8[Rails node 2] + Node_9[PostgreSQL node] + Node_10[Gitaly node] + Node_11[Redis node] + Node_12[Monitoring node] + end + end +``` + +The load balancer node and optional NFS server are omitted for clarity. + +This guide will result in the following: + +1. An offline primary. +1. A promoted secondary that is now the new primary. + +What is not covered: + +1. Re-adding the old **primary** as a secondary. +1. Adding a new secondary. + +### Preparation + +NOTE: **Note:** +Before following any of those steps, make sure you have `root` access to the +**secondary** to promote it, since there isn't provided an automated way to +promote a Geo replica and perform a failover. + +On the **secondary** node, navigate to the **Admin Area > Geo** dashboard to +review its status. Replicated objects (shown in green) should be close to 100%, +and there should be no failures (shown in red). If a large proportion of +objects aren't yet replicated (shown in gray), consider giving the node more +time to complete. + + + +If any objects are failing to replicate, this should be investigated before +scheduling the maintenance window. After a planned failover, anything that +failed to replicate will be **lost**. + +You can use the +[Geo status API](../../../../api/geo_nodes.md#retrieve-project-sync-or-verification-failures-that-occurred-on-the-current-node) +to review failed objects and the reasons for failure. +A common cause of replication failures is the data being missing on the +**primary** node - you can resolve these failures by restoring the data from backup, +or removing references to the missing data. + +The maintenance window won't end until Geo replication and verification is +completely finished. To keep the window as short as possible, you should +ensure these processes are close to 100% as possible during active use. + +If the **secondary** node is still replicating data from the **primary** node, +follow these steps to avoid unnecessary data loss: + +1. Until a [read-only mode](https://gitlab.com/gitlab-org/gitlab/-/issues/14609) + is implemented, updates must be prevented from happening manually to the + **primary**. Note that your **secondary** node still needs read-only + access to the **primary** node during the maintenance window: + + 1. At the scheduled time, using your cloud provider or your node's firewall, block + all HTTP, HTTPS and SSH traffic to/from the **primary** node, **except** for your IP and + the **secondary** node's IP. + + For instance, you can run the following commands on the **primary** node: + + ```shell + sudo iptables -A INPUT -p tcp -s <secondary_node_ip> --destination-port 22 -j ACCEPT + sudo iptables -A INPUT -p tcp -s <your_ip> --destination-port 22 -j ACCEPT + sudo iptables -A INPUT --destination-port 22 -j REJECT + + sudo iptables -A INPUT -p tcp -s <secondary_node_ip> --destination-port 80 -j ACCEPT + sudo iptables -A INPUT -p tcp -s <your_ip> --destination-port 80 -j ACCEPT + sudo iptables -A INPUT --tcp-dport 80 -j REJECT + + sudo iptables -A INPUT -p tcp -s <secondary_node_ip> --destination-port 443 -j ACCEPT + sudo iptables -A INPUT -p tcp -s <your_ip> --destination-port 443 -j ACCEPT + sudo iptables -A INPUT --tcp-dport 443 -j REJECT + ``` + + From this point, users will be unable to view their data or make changes on the + **primary** node. They will also be unable to log in to the **secondary** node. + However, existing sessions will work for the remainder of the maintenance period, and + public data will be accessible throughout. + + 1. Verify the **primary** node is blocked to HTTP traffic by visiting it in browser via + another IP. The server should refuse connection. + + 1. Verify the **primary** node is blocked to Git over SSH traffic by attempting to pull an + existing Git repository with an SSH remote URL. The server should refuse + connection. + + 1. On the **primary** node, disable non-Geo periodic background jobs by navigating + to **Admin Area > Monitoring > Background Jobs > Cron**, clicking `Disable All`, + and then clicking `Enable` for the `geo_sidekiq_cron_config_worker` cron job. + This job will re-enable several other cron jobs that are essential for planned + failover to complete successfully. + +1. Finish replicating and verifying all data: + + CAUTION: **Caution:** + Not all data is automatically replicated. Read more about + [what is excluded](../planned_failover.md#not-all-data-is-automatically-replicated). + + 1. If you are manually replicating any + [data not managed by Geo](../../replication/datatypes.md#limitations-on-replicationverification), + trigger the final replication process now. + 1. On the **primary** node, navigate to **Admin Area > Monitoring > Background Jobs > Queues** + and wait for all queues except those with `geo` in the name to drop to 0. + These queues contain work that has been submitted by your users; failing over + before it is completed will cause the work to be lost. + 1. On the **primary** node, navigate to **Admin Area > Geo** and wait for the + following conditions to be true of the **secondary** node you are failing over to: + - All replication meters to each 100% replicated, 0% failures. + - All verification meters reach 100% verified, 0% failures. + - Database replication lag is 0ms. + - The Geo log cursor is up to date (0 events behind). + + 1. On the **secondary** node, navigate to **Admin Area > Monitoring > Background Jobs > Queues** + and wait for all the `geo` queues to drop to 0 queued and 0 running jobs. + 1. On the **secondary** node, use [these instructions](../../../raketasks/check.md) + to verify the integrity of CI artifacts, LFS objects, and uploads in file + storage. + + At this point, your **secondary** node will contain an up-to-date copy of everything the + **primary** node has, meaning nothing will be lost when you fail over. + +1. In this final step, you need to permanently disable the **primary** node. + + CAUTION: **Caution:** + When the **primary** node goes offline, there may be data saved on the **primary** node + that has not been replicated to the **secondary** node. This data should be treated + as lost if you proceed. + + TIP: **Tip:** + If you plan to [update the **primary** domain DNS record](../index.md#step-4-optional-updating-the-primary-domain-dns-record), + you may wish to lower the TTL now to speed up propagation. + + When performing a failover, we want to avoid a split-brain situation where + writes can occur in two different GitLab instances. So to prepare for the + failover, you must disable the **primary** node: + + - If you have SSH access to the **primary** node, stop and disable GitLab: + + ```shell + sudo gitlab-ctl stop + ``` + + Prevent GitLab from starting up again if the server unexpectedly reboots: + + ```shell + sudo systemctl disable gitlab-runsvdir + ``` + + NOTE: **Note:** + (**CentOS only**) In CentOS 6 or older, there is no easy way to prevent GitLab from being + started if the machine reboots isn't available (see [Omnibus GitLab issue #3058](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/3058)). + It may be safest to uninstall the GitLab package completely with `sudo yum remove gitlab-ee`. + + NOTE: **Note:** + (**Ubuntu 14.04 LTS**) If you are using an older version of Ubuntu + or any other distribution based on the Upstart init system, you can prevent GitLab + from starting if the machine reboots as `root` with + `initctl stop gitlab-runsvvdir && echo 'manual' > /etc/init/gitlab-runsvdir.override && initctl reload-configuration`. + + - If you do not have SSH access to the **primary** node, take the machine offline and + prevent it from rebooting. Since there are many ways you may prefer to accomplish + this, we will avoid a single recommendation. You may need to: + + - Reconfigure the load balancers. + - Change DNS records (for example, point the **primary** DNS record to the + **secondary** node to stop using the **primary** node). + - Stop the virtual servers. + - Block traffic through a firewall. + - Revoke object storage permissions from the **primary** node. + - Physically disconnect a machine. + +### Promoting the **secondary** node + +NOTE: **Note:** +A new **secondary** should not be added at this time. If you want to add a new +**secondary**, do this after you have completed the entire process of promoting +the **secondary** to the **primary**. + +CAUTION: **Caution:** +If you encounter an `ActiveRecord::RecordInvalid: Validation failed: Name has already been taken` error during this process, read +[the troubleshooting advice](../../replication/troubleshooting.md#fixing-errors-during-a-failover-or-when-promoting-a-secondary-to-a-primary-node). + +The `gitlab-ctl promote-to-primary-node` command cannot be used yet in +conjunction with multiple servers, as it can only +perform changes on a **secondary** with only a single machine. Instead, you must +do this manually. + +DANGER: **Danger:** +In GitLab 13.2 and later versions, promoting a secondary node to a primary while the secondary is paused fails. We are [investigating the issue](https://gitlab.com/gitlab-org/gitlab/-/issues/225173). Do not pause replication before promoting a secondary. If the node is paused, please resume before promoting. + +1. SSH in to the PostgreSQL node in the **secondary** and trigger PostgreSQL to + promote to read-write: + + ```shell + sudo gitlab-pg-ctl promote + ``` + + In GitLab 12.8 and earlier, see [Message: `sudo: gitlab-pg-ctl: command not found`](../../replication/troubleshooting.md#message-sudo-gitlab-pg-ctl-command-not-found). + +1. Edit `/etc/gitlab/gitlab.rb` on every machine in the **secondary** to + reflect its new status as **primary** by removing any lines that enabled the + `geo_secondary_role`: + + ```ruby + ## In pre-11.5 documentation, the role was enabled as follows. Remove this line. + geo_secondary_role['enable'] = true + + ## In 11.5+ documentation, the role was enabled as follows. Remove this line. + roles ['geo_secondary_role'] + ``` + + After making these changes [Reconfigure GitLab](../../../restart_gitlab.md#omnibus-gitlab-reconfigure) each + machine so the changes take effect. + +1. Promote the **secondary** to **primary**. SSH into a single Rails node + server and execute: + + ```shell + sudo gitlab-rake geo:set_secondary_as_primary + ``` + +1. Verify you can connect to the newly promoted **primary** using the URL used + previously for the **secondary**. + +1. Success! The **secondary** has now been promoted to **primary**. + +### Next steps + +To regain geographic redundancy as quickly as possible, you should +[add a new **secondary** node](../../setup/index.md). To +do that, you can re-add the old **primary** as a new secondary and bring it back +online. diff --git a/doc/administration/geo/disaster_recovery/runbooks/planned_failover_single_node.md b/doc/administration/geo/disaster_recovery/runbooks/planned_failover_single_node.md new file mode 100644 index 00000000000..5e847030077 --- /dev/null +++ b/doc/administration/geo/disaster_recovery/runbooks/planned_failover_single_node.md @@ -0,0 +1,269 @@ +--- +stage: Enablement +group: Geo +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: howto +--- + +CAUTION: **Caution:** +This runbook is in **alpha**. For complete, production-ready documentation, see the +[disaster recovery documentation](../index.md). + +# Disaster Recovery (Geo) promotion runbooks **(PREMIUM ONLY)** + +## Geo planned failover for a single-node configuration + +| Component | Configuration | +|-------------|-----------------| +| PostgreSQL | Omnibus-managed | +| Geo site | Single-node | +| Secondaries | One | + +This runbook will guide you through a planned failover of a single-node Geo site +with one secondary. The following general architecture is assumed: + +```mermaid +graph TD + subgraph main[Geo deployment] + subgraph Primary[Primary site] + Node_1[(GitLab node)] + end + subgraph Secondary1[Secondary site] + Node_2[(GitLab node)] + end + end +``` + +This guide will result in the following: + +1. An offline primary. +1. A promoted secondary that is now the new primary. + +What is not covered: + +1. Re-adding the old **primary** as a secondary. +1. Adding a new secondary. + +### Preparation + +NOTE: **Note:** +Before following any of those steps, make sure you have `root` access to the +**secondary** to promote it, since there isn't provided an automated way to +promote a Geo replica and perform a failover. + +On the **secondary** node, navigate to the **Admin Area > Geo** dashboard to +review its status. Replicated objects (shown in green) should be close to 100%, +and there should be no failures (shown in red). If a large proportion of +objects aren't yet replicated (shown in gray), consider giving the node more +time to complete. + + + +If any objects are failing to replicate, this should be investigated before +scheduling the maintenance window. After a planned failover, anything that +failed to replicate will be **lost**. + +You can use the +[Geo status API](../../../../api/geo_nodes.md#retrieve-project-sync-or-verification-failures-that-occurred-on-the-current-node) +to review failed objects and the reasons for failure. +A common cause of replication failures is the data being missing on the +**primary** node - you can resolve these failures by restoring the data from backup, +or removing references to the missing data. + +The maintenance window won't end until Geo replication and verification is +completely finished. To keep the window as short as possible, you should +ensure these processes are close to 100% as possible during active use. + +If the **secondary** node is still replicating data from the **primary** node, +follow these steps to avoid unnecessary data loss: + +1. Until a [read-only mode](https://gitlab.com/gitlab-org/gitlab/-/issues/14609) + is implemented, updates must be prevented from happening manually to the + **primary**. Note that your **secondary** node still needs read-only + access to the **primary** node during the maintenance window: + + 1. At the scheduled time, using your cloud provider or your node's firewall, block + all HTTP, HTTPS and SSH traffic to/from the **primary** node, **except** for your IP and + the **secondary** node's IP. + + For instance, you can run the following commands on the **primary** node: + + ```shell + sudo iptables -A INPUT -p tcp -s <secondary_node_ip> --destination-port 22 -j ACCEPT + sudo iptables -A INPUT -p tcp -s <your_ip> --destination-port 22 -j ACCEPT + sudo iptables -A INPUT --destination-port 22 -j REJECT + + sudo iptables -A INPUT -p tcp -s <secondary_node_ip> --destination-port 80 -j ACCEPT + sudo iptables -A INPUT -p tcp -s <your_ip> --destination-port 80 -j ACCEPT + sudo iptables -A INPUT --tcp-dport 80 -j REJECT + + sudo iptables -A INPUT -p tcp -s <secondary_node_ip> --destination-port 443 -j ACCEPT + sudo iptables -A INPUT -p tcp -s <your_ip> --destination-port 443 -j ACCEPT + sudo iptables -A INPUT --tcp-dport 443 -j REJECT + ``` + + From this point, users will be unable to view their data or make changes on the + **primary** node. They will also be unable to log in to the **secondary** node. + However, existing sessions will work for the remainder of the maintenance period, and + public data will be accessible throughout. + + 1. Verify the **primary** node is blocked to HTTP traffic by visiting it in browser via + another IP. The server should refuse connection. + + 1. Verify the **primary** node is blocked to Git over SSH traffic by attempting to pull an + existing Git repository with an SSH remote URL. The server should refuse + connection. + + 1. On the **primary** node, disable non-Geo periodic background jobs by navigating + to **Admin Area > Monitoring > Background Jobs > Cron**, clicking `Disable All`, + and then clicking `Enable` for the `geo_sidekiq_cron_config_worker` cron job. + This job will re-enable several other cron jobs that are essential for planned + failover to complete successfully. + +1. Finish replicating and verifying all data: + + CAUTION: **Caution:** + Not all data is automatically replicated. Read more about + [what is excluded](../planned_failover.md#not-all-data-is-automatically-replicated). + + 1. If you are manually replicating any + [data not managed by Geo](../../replication/datatypes.md#limitations-on-replicationverification), + trigger the final replication process now. + 1. On the **primary** node, navigate to **Admin Area > Monitoring > Background Jobs > Queues** + and wait for all queues except those with `geo` in the name to drop to 0. + These queues contain work that has been submitted by your users; failing over + before it is completed will cause the work to be lost. + 1. On the **primary** node, navigate to **Admin Area > Geo** and wait for the + following conditions to be true of the **secondary** node you are failing over to: + - All replication meters to each 100% replicated, 0% failures. + - All verification meters reach 100% verified, 0% failures. + - Database replication lag is 0ms. + - The Geo log cursor is up to date (0 events behind). + + 1. On the **secondary** node, navigate to **Admin Area > Monitoring > Background Jobs > Queues** + and wait for all the `geo` queues to drop to 0 queued and 0 running jobs. + 1. On the **secondary** node, use [these instructions](../../../raketasks/check.md) + to verify the integrity of CI artifacts, LFS objects, and uploads in file + storage. + + At this point, your **secondary** node will contain an up-to-date copy of everything the + **primary** node has, meaning nothing will be lost when you fail over. + +1. In this final step, you need to permanently disable the **primary** node. + + CAUTION: **Caution:** + When the **primary** node goes offline, there may be data saved on the **primary** node + that has not been replicated to the **secondary** node. This data should be treated + as lost if you proceed. + + TIP: **Tip:** + If you plan to [update the **primary** domain DNS record](../index.md#step-4-optional-updating-the-primary-domain-dns-record), + you may wish to lower the TTL now to speed up propagation. + + When performing a failover, we want to avoid a split-brain situation where + writes can occur in two different GitLab instances. So to prepare for the + failover, you must disable the **primary** node: + + - If you have SSH access to the **primary** node, stop and disable GitLab: + + ```shell + sudo gitlab-ctl stop + ``` + + Prevent GitLab from starting up again if the server unexpectedly reboots: + + ```shell + sudo systemctl disable gitlab-runsvdir + ``` + + NOTE: **Note:** + (**CentOS only**) In CentOS 6 or older, there is no easy way to prevent GitLab from being + started if the machine reboots isn't available (see [Omnibus GitLab issue #3058](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/3058)). + It may be safest to uninstall the GitLab package completely with `sudo yum remove gitlab-ee`. + + NOTE: **Note:** + (**Ubuntu 14.04 LTS**) If you are using an older version of Ubuntu + or any other distribution based on the Upstart init system, you can prevent GitLab + from starting if the machine reboots as `root` with + `initctl stop gitlab-runsvvdir && echo 'manual' > /etc/init/gitlab-runsvdir.override && initctl reload-configuration`. + + - If you do not have SSH access to the **primary** node, take the machine offline and + prevent it from rebooting. Since there are many ways you may prefer to accomplish + this, we will avoid a single recommendation. You may need to: + + - Reconfigure the load balancers. + - Change DNS records (for example, point the **primary** DNS record to the + **secondary** node to stop using the **primary** node). + - Stop the virtual servers. + - Block traffic through a firewall. + - Revoke object storage permissions from the **primary** node. + - Physically disconnect a machine. + +### Promoting the **secondary** node + +Note the following when promoting a secondary: + +- A new **secondary** should not be added at this time. If you want to add a new + **secondary**, do this after you have completed the entire process of promoting + the **secondary** to the **primary**. +- If you encounter an `ActiveRecord::RecordInvalid: Validation failed: Name has already been taken` + error during this process, read + [the troubleshooting advice](../../replication/troubleshooting.md#fixing-errors-during-a-failover-or-when-promoting-a-secondary-to-a-primary-node). + +To promote the secondary node: + +1. SSH in to your **secondary** node and login as root: + + ```shell + sudo -i + ``` + +1. Edit `/etc/gitlab/gitlab.rb` to reflect its new status as **primary** by + removing any lines that enabled the `geo_secondary_role`: + + ```ruby + ## In pre-11.5 documentation, the role was enabled as follows. Remove this line. + geo_secondary_role['enable'] = true + + ## In 11.5+ documentation, the role was enabled as follows. Remove this line. + roles ['geo_secondary_role'] + ``` + +1. Run the following command to list out all preflight checks and automatically + check if replication and verification are complete before scheduling a planned + failover to ensure the process will go smoothly: + + ```shell + gitlab-ctl promotion-preflight-checks + ``` + +1. Promote the **secondary**: + + ```shell + gitlab-ctl promote-to-primary-node + ``` + + If you have already run the [preflight checks](../planned_failover.md#preflight-checks) + or don't want to run them, you can skip them: + + ```shell + gitlab-ctl promote-to-primary-node --skip-preflight-check + ``` + + You can also promote the secondary node to primary **without any further confirmation**, even when preflight checks fail: + + ```shell + sudo gitlab-ctl promote-to-primary-node --force + ``` + +1. Verify you can connect to the newly promoted **primary** node using the URL used + previously for the **secondary** node. + + If successful, the **secondary** node has now been promoted to the **primary** node. + +### Next steps + +To regain geographic redundancy as quickly as possible, you should +[add a new **secondary** node](../../setup/index.md). To +do that, you can re-add the old **primary** as a new secondary and bring it back +online. diff --git a/doc/administration/geo/index.md b/doc/administration/geo/index.md index 6fdf213ac78..8767940816b 100644 --- a/doc/administration/geo/index.md +++ b/doc/administration/geo/index.md @@ -39,7 +39,7 @@ Implementing Geo provides the following benefits: In addition, it: -- Can be used for cloning and fetching projects, in addition to reading any data available in the GitLab web interface (see [current limitations](#current-limitations)). +- Can be used for cloning and fetching projects, in addition to reading any data available in the GitLab web interface (see [limitations](#limitations)). - Overcomes slow connections between distant offices, saving time by improving speed for distributed teams. - Helps reducing the loading time for automated tasks, custom integrations, and internal workflows. - Can quickly fail over to a **secondary** node in a [disaster recovery](disaster_recovery/index.md) scenario. @@ -67,9 +67,9 @@ Keep in mind that: - **Secondary** nodes talk to the **primary** node to: - Get user data for logins (API). - Replicate repositories, LFS Objects, and Attachments (HTTPS + JWT). -- Since GitLab Premium 10.0, the **primary** node no longer talks to **secondary** nodes to notify for changes (API). +- In GitLab Premium 10.0 and later, the **primary** node no longer talks to **secondary** nodes to notify for changes (API). - Pushing directly to a **secondary** node (for both HTTP and SSH, including Git LFS) was [introduced](https://about.gitlab.com/releases/2018/09/22/gitlab-11-3-released/) in [GitLab Premium](https://about.gitlab.com/pricing/#self-managed) 11.3. -- There are [limitations](#current-limitations) in the current implementation. +- There are [limitations](#limitations) when using Geo. ### Architecture @@ -195,6 +195,9 @@ For information on how to update your Geo nodes to the latest GitLab version, se > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/35913) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.2. +DANGER: **Danger:** +In GitLab 13.2 and later versions, promoting a secondary node to a primary while the secondary is paused fails. We are [investigating the issue](https://gitlab.com/gitlab-org/gitlab/-/issues/225173). Do not pause replication before promoting a secondary. If the node is paused, please resume before promoting. + In some circumstances, like during [upgrades](replication/updating_the_geo_nodes.md) or a [planned failover](disaster_recovery/planned_failover.md), it is desirable to pause replication between the primary and secondary. Pausing and resuming replication is done via a command line tool from the secondary node. @@ -247,7 +250,7 @@ For more information on removing a Geo node, see [Removing **secondary** Geo nod To find out how to disable Geo, see [Disabling Geo](replication/disable_geo.md). -## Current limitations +## Limitations CAUTION: **Caution:** This list of limitations only reflects the latest version of GitLab. If you are using an older version, extra limitations may be in place. @@ -261,6 +264,7 @@ This list of limitations only reflects the latest version of GitLab. If you are - Object pools for forked project deduplication work only on the **primary** node, and are duplicated on the **secondary** node. - [External merge request diffs](../merge_request_diffs.md) will not be replicated if they are on-disk, and viewing merge requests will fail. However, external MR diffs in object storage **are** supported. The default configuration (in-database) does work. - GitLab Runners cannot register with a **secondary** node. Support for this is [planned for the future](https://gitlab.com/gitlab-org/gitlab/-/issues/3294). +- Geo **secondary** nodes can not be configured to [use high-availability configurations of PostgreSQL](https://gitlab.com/groups/gitlab-org/-/epics/2536). ### Limitations on replication/verification @@ -278,7 +282,7 @@ For answers to common questions, see the [Geo FAQ](replication/faq.md). ## Log files -Since GitLab 9.5, Geo stores structured log messages in a `geo.log` file. For Omnibus installations, this file is at `/var/log/gitlab/gitlab-rails/geo.log`. +In GitLab 9.5 and later, Geo stores structured log messages in a `geo.log` file. For Omnibus installations, this file is at `/var/log/gitlab/gitlab-rails/geo.log`. This file contains information about when Geo attempts to sync repositories and files. Each line in the file contains a separate JSON entry that can be ingested into. For example, Elasticsearch or Splunk. diff --git a/doc/administration/geo/replication/datatypes.md b/doc/administration/geo/replication/datatypes.md index 166a724f9c1..d1fa2d503be 100644 --- a/doc/administration/geo/replication/datatypes.md +++ b/doc/administration/geo/replication/datatypes.md @@ -23,30 +23,34 @@ We currently distinguish between three different data types: See the list below of each feature or component we replicate, its corresponding data type, replication, and verification methods: -| Type | Feature / component | Replication method | Verification method | -|:---------|:----------------------------------------------|:--------------------------------------|:-----------------------| -| Database | Application data in PostgreSQL | Native | Native | -| Database | Redis | _N/A_ (*1*) | _N/A_ | -| Database | Elasticsearch | Native | Native | -| Database | Personal snippets | PostgreSQL Replication | PostgreSQL Replication | -| Database | Project snippets | PostgreSQL Replication | PostgreSQL Replication | -| Database | SSH public keys | PostgreSQL Replication | PostgreSQL Replication | -| Git | Project repository | Geo with Gitaly | Gitaly Checksum | -| Git | Project wiki repository | Geo with Gitaly | Gitaly Checksum | -| Git | Project designs repository | Geo with Gitaly | Gitaly Checksum | -| Git | Object pools for forked project deduplication | Geo with Gitaly | _Not implemented_ | -| Blobs | User uploads _(filesystem)_ | Geo with API | _Not implemented_ | -| Blobs | User uploads _(object storage)_ | Geo with API/Managed (*2*) | _Not implemented_ | -| Blobs | LFS objects _(filesystem)_ | Geo with API | _Not implemented_ | -| Blobs | LFS objects _(object storage)_ | Geo with API/Managed (*2*) | _Not implemented_ | -| Blobs | CI job artifacts _(filesystem)_ | Geo with API | _Not implemented_ | -| Blobs | CI job artifacts _(object storage)_ | Geo with API/Managed (*2*) | _Not implemented_ | -| Blobs | Archived CI build traces _(filesystem)_ | Geo with API | _Not implemented_ | -| Blobs | Archived CI build traces _(object storage)_ | Geo with API/Managed (*2*) | _Not implemented_ | -| Blobs | Container registry _(filesystem)_ | Geo with API/Docker API | _Not implemented_ | -| Blobs | Container registry _(object storage)_ | Geo with API/Managed/Docker API (*2*) | _Not implemented_ | -| Blobs | Package registry _(filesystem)_ | Geo with API | _Not implemented_ | -| Blobs | Package registry _(object storage)_ | Geo with API/Managed (*2*) | _Not implemented_ | +| Type | Feature / component | Replication method | Verification method | +|:---------|:------------------------------------------------|:--------------------------------------|:-----------------------| +| Database | Application data in PostgreSQL | Native | Native | +| Database | Redis | _N/A_ (*1*) | _N/A_ | +| Database | Elasticsearch | Native | Native | +| Database | Personal snippets | PostgreSQL Replication | PostgreSQL Replication | +| Database | Project snippets | PostgreSQL Replication | PostgreSQL Replication | +| Database | SSH public keys | PostgreSQL Replication | PostgreSQL Replication | +| Git | Project repository | Geo with Gitaly | Gitaly Checksum | +| Git | Project wiki repository | Geo with Gitaly | Gitaly Checksum | +| Git | Project designs repository | Geo with Gitaly | Gitaly Checksum | +| Git | Object pools for forked project deduplication | Geo with Gitaly | _Not implemented_ | +| Blobs | User uploads _(filesystem)_ | Geo with API | _Not implemented_ | +| Blobs | User uploads _(object storage)_ | Geo with API/Managed (*2*) | _Not implemented_ | +| Blobs | LFS objects _(filesystem)_ | Geo with API | _Not implemented_ | +| Blobs | LFS objects _(object storage)_ | Geo with API/Managed (*2*) | _Not implemented_ | +| Blobs | CI job artifacts _(filesystem)_ | Geo with API | _Not implemented_ | +| Blobs | CI job artifacts _(object storage)_ | Geo with API/Managed (*2*) | _Not implemented_ | +| Blobs | Archived CI build traces _(filesystem)_ | Geo with API | _Not implemented_ | +| Blobs | Archived CI build traces _(object storage)_ | Geo with API/Managed (*2*) | _Not implemented_ | +| Blobs | Container registry _(filesystem)_ | Geo with API/Docker API | _Not implemented_ | +| Blobs | Container registry _(object storage)_ | Geo with API/Managed/Docker API (*2*) | _Not implemented_ | +| Blobs | Package registry _(filesystem)_ | Geo with API | _Not implemented_ | +| Blobs | Package registry _(object storage)_ | Geo with API/Managed (*2*) | _Not implemented_ | +| Blobs | Versioned Terraform State _(filesystem)_ | Geo with API | _Not implemented_ | +| Blobs | Versioned Terraform State _(object storage)_ | Geo with API/Managed (*2*) | _Not implemented_ | +| Blobs | External Merge Request Diffs _(filesystem)_ | Geo with API | _Not implemented_ | +| Blobs | External Merge Request Diffs _(object storage)_ | Geo with API/Managed (*2*) | _Not implemented_ | - (*1*): Redis replication can be used as part of HA with Redis sentinel. It's not used between Geo nodes. - (*2*): Object storage replication can be performed by Geo or by your object storage provider/appliance @@ -160,39 +164,34 @@ replicating data from those features will cause the data to be **lost**. If you wish to use those features on a **secondary** node, or to execute a failover successfully, you must replicate their data using some other means. -| Feature | Replicated (added in GitLab version) | Verified (added in GitLab version) | Notes | -|:------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------|:----------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------| -| Application data in PostgreSQL | **Yes** (10.2) | **Yes** (10.2) | | -| Project repository | **Yes** (10.2) | **Yes** (10.7) | | -| Project wiki repository | **Yes** (10.2) | **Yes** (10.7) | | -| Project designs repository | **Yes** (12.7) | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/32467) | | -| Uploads | **Yes** (10.2) | [No](https://gitlab.com/groups/gitlab-org/-/epics/1817) | Verified only on transfer, or manually (*1*) | -| LFS objects | **Yes** (10.2) | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/8922) | Verified only on transfer, or manually (*1*). Unavailable for new LFS objects in 11.11.x and 12.0.x (*2*). | -| CI job artifacts (other than traces) | **Yes** (10.4) | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/8923) | Verified only manually (*1*) | -| Archived traces | **Yes** (10.4) | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/8923) | Verified only on transfer, or manually (*1*) | -| Personal snippets | **Yes** (10.2) | **Yes** (10.2) | | -| [Versioned snippets](../../../user/snippets.md#versioned-snippets) | [No](https://gitlab.com/groups/gitlab-org/-/epics/2809) | [No](https://gitlab.com/groups/gitlab-org/-/epics/2810) | | -| Project snippets | **Yes** (10.2) | **Yes** (10.2) | | -| Object pools for forked project deduplication | **Yes** | No | | -| [Server-side Git hooks](../../server_hooks.md) | No | No | | -| [Elasticsearch integration](../../../integration/elasticsearch.md) | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/1186) | No | | -| [GitLab Pages](../../pages/index.md) | [No](https://gitlab.com/groups/gitlab-org/-/epics/589) | No | | -| [Container Registry](../../packages/container_registry.md) | **Yes** (12.3) | No | | -| [NPM Registry](../../../user/packages/npm_registry/index.md) | **Yes** (13.2) | No | Behind feature flag `geo_package_file_replication`, enabled by default | -| [Maven Repository](../../../user/packages/maven_repository/index.md) | **Yes** (13.2) | No | Behind feature flag `geo_package_file_replication`, enabled by default | -| [Conan Repository](../../../user/packages/conan_repository/index.md) | **Yes** (13.2) | No | Behind feature flag `geo_package_file_replication`, enabled by default | -| [NuGet Repository](../../../user/packages/nuget_repository/index.md) | **Yes** (13.2) | No | Behind feature flag `geo_package_file_replication`, enabled by default | -| [PyPi Repository](../../../user/packages/pypi_repository/index.md) | **Yes** (13.2) | No | Behind feature flag `geo_package_file_replication`, enabled by default | -| [Composer Repository](../../../user/packages/composer_repository/index.md) | **Yes** (13.2) | No | Behind feature flag `geo_package_file_replication`, enabled by default | -| [External merge request diffs](../../merge_request_diffs.md) | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/33817) | No | | -| [Terraform State](../../terraform_state.md) | [No](https://gitlab.com/groups/gitlab-org/-/epics/3112)(*3*) | No | | -| [Vulnerability Export](../../../user/application_security/security_dashboard/#export-vulnerabilities) | [No](https://gitlab.com/groups/gitlab-org/-/epics/3111)(*3*) | No | | -| Content in object storage | **Yes** (12.4) | No | | - -- (*1*): The integrity can be verified manually using - [Integrity Check Rake Task](../../raketasks/check.md) on both nodes and comparing - the output between them. -- (*2*): GitLab versions 11.11.x and 12.0.x are affected by [a bug that prevents any new - LFS objects from replicating](https://gitlab.com/gitlab-org/gitlab/-/issues/32696). -- (*3*): If you are using Object Storage, the replication can be performed by the - Object Storage provider if supported. Please see [Geo with Object Storage](object_storage.md) +| Feature | Replicated (added in GitLab version) | Verified (added in GitLab version) | Object Storage replication (please see [Geo with Object Storage](object_storage.md)) | Notes | +|:---------------------------------------------------------------------------------------------------------------|:-----------------------------------------------------------------------------------|:----------------------------------------------------------|:-------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| [Application data in PostgreSQL](../../postgresql/index.md) | **Yes** (10.2) | **Yes** (10.2) | No | | +| [Project repository](../../..//user/project/repository/) | **Yes** (10.2) | **Yes** (10.7) | No | | +| [Project wiki repository](../../../user/project/wiki/) | **Yes** (10.2) | **Yes** (10.7) | No | | +| [Uploads](../../uploads.md) | **Yes** (10.2) | [No](https://gitlab.com/groups/gitlab-org/-/epics/1817) | No | Verified only on transfer or manually using [Integrity Check Rake Task](../../raketasks/check.md) on both nodes and comparing the output between them. | +| [LFS objects](../../lfs/index.md) | **Yes** (10.2) | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/8922) | Via Object Storage provider if supported. Native Geo support (Beta). | Verified only on transfer or manually using [Integrity Check Rake Task](../../raketasks/check.md) on both nodes and comparing the output between them. GitLab versions 11.11.x and 12.0.x are affected by [a bug that prevents any new LFS objects from replicating](https://gitlab.com/gitlab-org/gitlab/-/issues/32696). | +| [Personal snippets](../../../user/snippets.md#personal-snippets) | **Yes** (10.2) | **Yes** (10.2) | No | | +| [Project snippets](../../../user/snippets.md#project-snippets) | **Yes** (10.2) | **Yes** (10.2) | No | | +| [CI job artifacts (other than Job Logs)](../../../ci/pipelines/job_artifacts.md) | **Yes** (10.4) | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/8923) | Via Object Storage provider if supported. Native Geo support (Beta) . | Verified only manually using [Integrity Check Rake Task](../../raketasks/check.md) on both nodes and comparing the output between them | +| [Job logs](../../job_logs.md) | **Yes** (10.4) | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/8923) | Via Object Storage provider if supported. Native Geo support (Beta). | Verified only on transfer or manually using [Integrity Check Rake Task](../../raketasks/check.md) on both nodes and comparing the output between them | +| [Object pools for forked project deduplication](../../../development/git_object_deduplication.md) | **Yes** | No | No | | +| [Container Registry](../../packages/container_registry.md) | **Yes** (12.3) | No | No | Disabled by default. See [instructions](./docker_registry.md) to enable. | +| [Content in object storage (beta)](object_storage.md) | **Yes** (12.4) | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/13845) | No | | +| [Project designs repository](../../../user/project/issues/design_management.md) | **Yes** (12.7) | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/32467) | Via Object Storage provider if supported. Native Geo support (Beta). | | +| [NPM Registry](../../../user/packages/npm_registry/index.md) | **Yes** (13.2) | [No](https://gitlab.com/groups/gitlab-org/-/epics/1817) | Via Object Storage provider if supported. Native Geo support (Beta). | Behind feature flag `geo_package_file_replication`, enabled by default | +| [Maven Repository](../../../user/packages/maven_repository/index.md) | **Yes** (13.2) | [No](https://gitlab.com/groups/gitlab-org/-/epics/1817) | Via Object Storage provider if supported. Native Geo support (Beta). | Behind feature flag `geo_package_file_replication`, enabled by default | +| [Conan Repository](../../../user/packages/conan_repository/index.md) | **Yes** (13.2) | [No](https://gitlab.com/groups/gitlab-org/-/epics/1817) | Via Object Storage provider if supported. Native Geo support (Beta). | Behind feature flag `geo_package_file_replication`, enabled by default | +| [NuGet Repository](../../../user/packages/nuget_repository/index.md) | **Yes** (13.2) | [No](https://gitlab.com/groups/gitlab-org/-/epics/1817) | Via Object Storage provider if supported. Native Geo support (Beta). | Behind feature flag `geo_package_file_replication`, enabled by default | +| [PyPI Repository](../../../user/packages/pypi_repository/index.md) | **Yes** (13.2) | [No](https://gitlab.com/groups/gitlab-org/-/epics/1817) | Via Object Storage provider if supported. Native Geo support (Beta). | Behind feature flag `geo_package_file_replication`, enabled by default | +| [Composer Repository](../../../user/packages/composer_repository/index.md) | **Yes** (13.2) | [No](https://gitlab.com/groups/gitlab-org/-/epics/1817) | Via Object Storage provider if supported. Native Geo support (Beta). | Behind feature flag `geo_package_file_replication`, enabled by default | +| [Generic packages](../../../user/packages/generic_packages/index.md) | **Yes** (13.5) | [No](https://gitlab.com/groups/gitlab-org/-/epics/1817) | Via Object Storage provider if supported. Native Geo support (Beta). | Behind feature flag `geo_package_file_replication`, enabled by default | +| [Versioned Terraform State](../../terraform_state.md) | **Yes** (13.5) | No | Via Object Storage provider if supported. Native Geo support (Beta). | Behind feature flag `geo_terraform_state_version_replication`, enabled by default | +| [External merge request diffs](../../merge_request_diffs.md) | **Yes** (13.5) | No | Behind feature flag `geo_merge_request_diff_replication`, enabled by default | | +| [Versioned snippets](../../../user/snippets.md#versioned-snippets) | [No](https://gitlab.com/groups/gitlab-org/-/epics/2809) | [No](https://gitlab.com/groups/gitlab-org/-/epics/2810) | No | | +| [Server-side Git hooks](../../server_hooks.md) | [No](https://gitlab.com/groups/gitlab-org/-/epics/1867) | No | No | | +| [Elasticsearch integration](../../../integration/elasticsearch.md) | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/1186) | No | No | | +| [GitLab Pages](../../pages/index.md) | [No](https://gitlab.com/groups/gitlab-org/-/epics/589) | No | No | | +| [CI Pipeline Artifacts](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/models/ci/pipeline_artifact.rb) | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/238464) | No | Via Object Storage provider if supported. Native Geo support (Beta). | Persists additional artifacts after a pipeline completes | +| [Dependency proxy images](../../../user/packages/dependency_proxy/index.md) | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/259694) | No | No | Blocked on [Geo: Secondary Mimicry](https://gitlab.com/groups/gitlab-org/-/epics/1528). Note that replication of this cache is not needed for Disaster Recovery purposes because it can be recreated from external sources. | +| [Vulnerability Export](../../../user/application_security/security_dashboard/#export-vulnerabilities) | [Not planned](https://gitlab.com/groups/gitlab-org/-/epics/3111) | No | Via Object Storage provider if supported. Native Geo support (Beta). | Not planned because they are ephemeral and sensitive. They can be regenerated on demand. | diff --git a/doc/administration/geo/replication/disable_geo.md b/doc/administration/geo/replication/disable_geo.md index aed8e5fc3bc..14a11d9c1e3 100644 --- a/doc/administration/geo/replication/disable_geo.md +++ b/doc/administration/geo/replication/disable_geo.md @@ -29,7 +29,7 @@ anymore on these nodes. You can follow our docs to [remove your secondary Geo no If the current node that you want to keep using is a secondary node, you need to first promote it to primary. You can use our steps on [how to promote a secondary node](../disaster_recovery/#step-3-promoting-a-secondary-node) -in order to do that. +to do that. ## Remove the primary node from the UI diff --git a/doc/administration/geo/replication/faq.md b/doc/administration/geo/replication/faq.md index 3892d73b465..f7f391b360e 100644 --- a/doc/administration/geo/replication/faq.md +++ b/doc/administration/geo/replication/faq.md @@ -67,3 +67,7 @@ That's totally fine. We use HTTP(s) to fetch repository changes from the **prima ## Is this possible to set up a Docker Registry for a **secondary** node that mirrors the one on the **primary** node? Yes. See [Docker Registry for a **secondary** node](docker_registry.md). + +## Can I login to a secondary node? + +Yes, but secondary nodes receive all authentication data (like user accounts and logins) from the primary instance. This means you will be re-directed to the primary for authentication and routed back afterwards. diff --git a/doc/administration/geo/replication/geo_validation_tests.md b/doc/administration/geo/replication/geo_validation_tests.md index 8247b8c6336..c28688930b5 100644 --- a/doc/administration/geo/replication/geo_validation_tests.md +++ b/doc/administration/geo/replication/geo_validation_tests.md @@ -114,6 +114,22 @@ The following are GitLab upgrade validation tests we performed. The following are PostgreSQL upgrade validation tests we performed. +### September 2020 + +[Verify PostgreSQL 12 upgrade for Geo installations](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5454): + +- Description: With PostgreSQL 12 available as an opt-in version in GitLab 13.3, we tested upgrading + existing Geo installations from PostgreSQL 11 to 12. We also re-tested fresh installations of GitLab + with Geo after fixes were made to support PostgreSQL 12. These tests were done using a + [nightly build](https://packages.gitlab.com/gitlab/nightly-builds/packages/ubuntu/bionic/gitlab-ee_13.3.6+rnightly.169516.d5209202-0_amd64.deb) + of GitLab 13.4. +- Outcome: Tests were successful for Geo deployments with a single database node on the primary and secondary. + We encountered known issues with repmgr and Patroni managed PostgreSQL clusters on the Geo primary. Using + PostgreSQL 12 with a database cluster on the primary is not recommended until the issues are resolved. +- Known issues for PostgreSQL clusters: + - [Ensure Patroni detects PostgreSQL update](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5423) + - [Allow configuring permanent replication slots in patroni](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5628) + ### August 2020 [Verify Geo installation with PostgreSQL 12](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5453): @@ -168,5 +184,4 @@ The following are additional validation tests we performed. [Test Gitaly Cluster on a Geo Deployment](https://gitlab.com/gitlab-org/gitlab/-/issues/223210): - Description: Tested a Geo deployment with Gitaly clusters configured on both the primary and secondary Geo sites. Triggered automatic Gitaly cluster failover on the primary Geo site, and ran end-to-end Geo tests. Then triggered Gitaly cluster failover on the secondary Geo site, and re-ran the end-to-end Geo tests. - - Outcome: Successful end-to-end tests before and after Gitaly cluster failover on the primary site, and before and after Gitaly cluster failover on the secondary site. diff --git a/doc/administration/high_availability/img/geo-ha-diagram.png b/doc/administration/geo/replication/img/geo-ha-diagram.png Binary files differindex da5d612827c..da5d612827c 100644 --- a/doc/administration/high_availability/img/geo-ha-diagram.png +++ b/doc/administration/geo/replication/img/geo-ha-diagram.png diff --git a/doc/administration/geo/replication/multiple_servers.md b/doc/administration/geo/replication/multiple_servers.md index cba41c375a3..7d65d2165c5 100644 --- a/doc/administration/geo/replication/multiple_servers.md +++ b/doc/administration/geo/replication/multiple_servers.md @@ -13,7 +13,7 @@ described, it is possible to adapt these instructions to your needs. ## Architecture overview - + _[diagram source - GitLab employees only](https://docs.google.com/drawings/d/1z0VlizKiLNXVVVaERFwgsIOuEgjcUqDTWPdQYsE7Z4c/edit)_ @@ -133,7 +133,7 @@ Configure the following services, again using the non-Geo multi-node documentation: - [Configuring Redis for GitLab](../../redis/replication_and_failover.md#example-configuration-for-the-gitlab-application) for multiple nodes. -- [Gitaly](../../high_availability/gitaly.md), which will store data that is +- [Gitaly](../../gitaly/index.md), which will store data that is synchronized from the **primary** node. NOTE: **Note:** @@ -422,7 +422,6 @@ application servers above, with some changes to run only the `sidekiq` service: ## alertmanager['enable'] = false consul['enable'] = false - geo_logcursor['enable'] = false gitaly['enable'] = false gitlab_exporter['enable'] = false gitlab_workhorse['enable'] = false diff --git a/doc/administration/geo/replication/troubleshooting.md b/doc/administration/geo/replication/troubleshooting.md index f6d6f39fb19..b62c5c6f460 100644 --- a/doc/administration/geo/replication/troubleshooting.md +++ b/doc/administration/geo/replication/troubleshooting.md @@ -386,6 +386,15 @@ This happens when you have added IP addresses without a subnet mask in `postgres To fix this, add the subnet mask in `/etc/gitlab/gitlab.rb` under `postgresql['md5_auth_cidr_addresses']` to respect the CIDR format (i.e. `1.2.3.4/32`). +### Message: `Found data in the gitlabhq_production database!` when running `gitlab-ctl replicate-geo-database` + +This happens if data is detected in the `projects` table. When one or more projects are detected, the operation +is aborted to prevent accidental data loss. To bypass this message, pass the `--force` option to the command. + +In GitLab 13.4, a seed project is added when GitLab is first installed. This makes it necessary to pass `--force` even +on a new Geo secondary node. There is an [issue to account for seed projects](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5618) +when checking the database. + ### Very large repositories never successfully synchronize on the **secondary** node GitLab places a timeout on all repository clones, including project imports @@ -483,8 +492,8 @@ to start again from scratch, there are a few steps that can help you: gitlab-ctl start geo-postgresql ``` - Reconfigure in order to recreate the folders and make sure permissions and ownership - are correctly + Reconfigure to recreate the folders and make sure permissions and ownership + are correct: ```shell gitlab-ctl reconfigure @@ -605,46 +614,18 @@ or `gitlab-ctl promote-to-primary-node`, either: ### Message: ActiveRecord::RecordInvalid: Validation failed: Enabled Geo primary node cannot be disabled -This error may occur if you have paused replication from the original primary node before attempting to promote this node. -To double check this, you can do the following: - -- Get the current secondary node's ID using: - - ```shell - sudo gitlab-rails runner 'puts GeoNode.current_node.id' - ``` - -- Double check that the node is active through the database by running the following - on the secondary node, replacing `ID_FROM_ABOVE`: - - ```shell - sudo gitlab-rails dbconsole - - SELECT enabled FROM geo_nodes WHERE id = ID_FROM_ABOVE; - ``` - -- If the above returned `f` it means that the replication was paused. - You can re-enable it through an `UPDATE` statement in the database: - - ```shell - sudo gitlab-rails dbconsole - - UPDATE geo_nodes SET enabled = 't' WHERE id = ID_FROM_ABOVE; - ``` - -### While Promoting the secondary, I got an error `ActiveRecord::RecordInvalid` - If you disabled a secondary node, either with the [replication pause task](../index.md#pausing-and-resuming-replication) -(13.2) or via the UI (13.1 and earlier), you must first re-enable the -node before you can continue. This is fixed in 13.4. +(13.2) or by using the user interface (13.1 and earlier), you must first +re-enable the node before you can continue. This is fixed in 13.4. -From `gitlab-psql`, execute the following, replacing `<your secondary url>` -with the URL for your secondary server starting with `http` or `https` and ending with a `/`. +Run the following command, replacing `https://<secondary url>/` with the URL +for your secondary server, using either `http` or `https`, and ensuring that you +end the URL with a slash (`/`): ```shell -SECONDARY_URL="https://<secondary url>/" -DATABASE_NAME="gitlabhq_production" -sudo gitlab-psql -d "$DATABASE_NAME" -c "UPDATE geo_nodes SET enabled = true WHERE url = '$SECONDARY_URL';" +sudo gitlab-rails dbconsole + +UPDATE geo_nodes SET enabled = true WHERE url = 'https://<secondary url>/' AND enabled = false;" ``` This should update 1 row. diff --git a/doc/administration/geo/replication/updating_the_geo_nodes.md b/doc/administration/geo/replication/updating_the_geo_nodes.md index b78aeb06ebf..1af2b8d0b88 100644 --- a/doc/administration/geo/replication/updating_the_geo_nodes.md +++ b/doc/administration/geo/replication/updating_the_geo_nodes.md @@ -21,14 +21,17 @@ Updating Geo nodes involves performing: NOTE: **Note:** These general update steps are not intended for [high-availability deployments](https://docs.gitlab.com/omnibus/update/README.html#multi-node--ha-deployment), and will cause downtime. If you want to avoid downtime, consider using [zero downtime updates](https://docs.gitlab.com/omnibus/update/README.html#zero-downtime-updates). +DANGER: **Danger:** +In GitLab 13.2 and later versions, promoting a secondary node to a primary while the secondary is paused fails. We are [investigating the issue](https://gitlab.com/gitlab-org/gitlab/-/issues/225173). Do not pause replication before promoting a secondary. If the node is paused, please resume before promoting. + To update the Geo nodes when a new GitLab version is released, update **primary** and all **secondary** nodes: 1. **Optional:** [Pause replication on each **secondary** node.](../index.md#pausing-and-resuming-replication) 1. Log into the **primary** node. -1. [Update GitLab on the **primary** node using Omnibus](https://docs.gitlab.com/omnibus/update/README.html). +1. [Update GitLab on the **primary** node using Omnibus's Geo-specific steps](https://docs.gitlab.com/omnibus/update/README.html#geo-deployment). 1. Log into each **secondary** node. -1. [Update GitLab on each **secondary** node using Omnibus](https://docs.gitlab.com/omnibus/update/README.html). +1. [Update GitLab on each **secondary** node using Omnibus's Geo-specific steps](https://docs.gitlab.com/omnibus/update/README.html#geo-deployment). 1. If you paused replication in step 1, [resume replication on each **secondary**](../index.md#pausing-and-resuming-replication) 1. [Test](#check-status-after-updating) **primary** and **secondary** nodes, and check version in each. diff --git a/doc/administration/geo/replication/version_specific_updates.md b/doc/administration/geo/replication/version_specific_updates.md index 1ae246e3e61..85c869eff92 100644 --- a/doc/administration/geo/replication/version_specific_updates.md +++ b/doc/administration/geo/replication/version_specific_updates.md @@ -397,7 +397,7 @@ existing repositories was added in GitLab 10.1. ## Updating to GitLab 10.0 -Since GitLab 10.0, we require all **Geo** systems to [use SSH key lookups via +In GitLab 10.0 and later, we require all **Geo** systems to [use SSH key lookups via the database](../../operations/fast_ssh_key_lookup.md) to avoid having to maintain consistency of the `authorized_keys` file for SSH access. Failing to do this will prevent users from being able to clone via SSH. @@ -447,8 +447,8 @@ Omnibus is the following: > **IMPORTANT**: With GitLab 9.0, the PostgreSQL version is updated to 9.6 and manual steps are -required in order to update the **secondary** nodes and keep the Streaming -Replication working. Downtime is required, so plan ahead. +required to update the **secondary** nodes and keep the Streaming Replication +working. Downtime is required, so plan ahead. The following steps apply only if you update from a 8.17 GitLab version to 9.0+. For previous versions, update to GitLab 8.17 first before attempting to @@ -611,9 +611,9 @@ is prepended with the relevant node for better clarity: ### Update tracking database on **secondary** node -After updating a **secondary** node, you might need to run migrations on -the tracking database. The tracking database was added in GitLab 9.1, -and it is required since 10.0. +After updating a **secondary** node, you might need to run migrations on the +tracking database. The tracking database was added in GitLab 9.1, and is +required in GitLab 10.0 and later. 1. Run database migrations on tracking database: diff --git a/doc/administration/geo/setup/database.md b/doc/administration/geo/setup/database.md index aefa8a0e399..09b9c71aeb7 100644 --- a/doc/administration/geo/setup/database.md +++ b/doc/administration/geo/setup/database.md @@ -17,9 +17,10 @@ NOTE: **Note:** The stages of the setup process must be completed in the documented order. Before attempting the steps in this stage, [complete all prior stages](../setup/index.md#using-omnibus-gitlab). -This document describes the minimal steps you have to take in order to -replicate your **primary** GitLab database to a **secondary** node's database. You may -have to change some values according to your database setup, how big it is, etc. +This document describes the minimal steps you have to take to replicate your +**primary** GitLab database to a **secondary** node's database. You may have to +change some values, based on attributes including your database's setup and +size. You are encouraged to first read through all the steps before executing them in your testing/production environment. @@ -433,6 +434,11 @@ data before running `pg_basebackup`. NOTE: **Note:** Replication slot names must only contain lowercase letters, numbers, and the underscore character. + NOTE: **Note:** + In GitLab 13.4, a seed project is added when GitLab is first installed. This makes it necessary to pass `--force` even + on a new Geo secondary node. There is an [issue to account for seed projects](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5618) + when checking the database. + When prompted, enter the _plaintext_ password you set up for the `gitlab_replicator` user in the first step. diff --git a/doc/administration/gitaly/index.md b/doc/administration/gitaly/index.md index e6b137bac29..59a6f2596da 100644 --- a/doc/administration/gitaly/index.md +++ b/doc/administration/gitaly/index.md @@ -90,10 +90,10 @@ When running Gitaly on its own server, note the following regarding GitLab versi leveraged for redundancy on block-level Git data, but only has to be mounted on the Gitaly servers. - From GitLab 11.8 to 12.2, it is possible to use Elasticsearch in a Gitaly setup that doesn't use - NFS. In order to use Elasticsearch in these versions, the + NFS. To use Elasticsearch in these versions, the [repository indexer](../../integration/elasticsearch.md#elasticsearch-repository-indexer) must be enabled in your GitLab configuration. -- [Since GitLab 12.3](https://gitlab.com/gitlab-org/gitlab/-/issues/6481), the new indexer is +- [In GitLab 12.3 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/6481), the new indexer is the default and no configuration is required. ### Network architecture @@ -317,7 +317,7 @@ disable enforcement. For more information, see the documentation on configuring ``` 1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). -1. Run `sudo /opt/gitlab/embedded/service/gitlab-shell/bin/check -config /opt/gitlab/embedded/service/gitlab-shell/config.yml` +1. Run `sudo /opt/gitlab/embedded/bin/gitaly-hooks check /var/opt/gitlab/gitaly/config.toml` to confirm that Gitaly can perform callbacks to the GitLab internal API. **For installations from source** @@ -364,7 +364,7 @@ disable enforcement. For more information, see the documentation on configuring ``` 1. Save the files and [restart GitLab](../restart_gitlab.md#installations-from-source). -1. Run `sudo -u git /home/git/gitlab-shell/bin/check -config /home/git/gitlab-shell/config.yml` +1. Run `sudo -u git /home/git/gitaly/gitaly-hooks check /home/git/gitaly/config.toml` to confirm that Gitaly can perform callbacks to the GitLab internal API. ### Configure Gitaly clients @@ -382,10 +382,10 @@ if previously enabled manually. Gitaly makes the following assumptions: - Your `gitaly1.internal` Gitaly server can be reached at `gitaly1.internal:8075` from your Gitaly - clients, and that Gitaly server can read and write to `/mnt/gitlab/default` and + clients, and that Gitaly server can read, write, and set permissions on `/mnt/gitlab/default` and `/mnt/gitlab/storage1`. - Your `gitaly2.internal` Gitaly server can be reached at `gitaly2.internal:8075` from your Gitaly - clients, and that Gitaly server can read and write to `/mnt/gitlab/storage2`. + clients, and that Gitaly server can read, write, and set permissions on `/mnt/gitlab/storage2`. - Your `gitaly1.internal` and `gitaly2.internal` Gitaly servers can reach each other. You can't define Gitaly servers with some as a local Gitaly server @@ -406,8 +406,8 @@ server (with `gitaly_address`) unless you setup with special ``` 1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). -1. Run `sudo gitlab-rake gitlab:gitaly:check` to confirm the Gitaly client can connect to Gitaly - servers. +1. Run `sudo gitlab-rake gitlab:gitaly:check` on the Gitaly client (for example, the + Rails application) to confirm it can connect to Gitaly servers. 1. Tail the logs to see the requests: ```shell @@ -424,17 +424,17 @@ server (with `gitaly_address`) unless you setup with special storages: default: gitaly_address: tcp://gitaly1.internal:8075 - path: /some/dummy/path + path: /some/local/path storage1: gitaly_address: tcp://gitaly1.internal:8075 - path: /some/dummy/path + path: /some/local/path storage2: gitaly_address: tcp://gitaly2.internal:8075 - path: /some/dummy/path + path: /some/local/path ``` NOTE: **Note:** - `/some/dummy/path` should be set to a local folder that exists, however no data will be stored in + `/some/local/path` should be set to a local folder that exists, however no data will be stored in this folder. This will no longer be necessary after [this issue](https://gitlab.com/gitlab-org/gitaly/-/issues/1282) is resolved. @@ -482,6 +482,14 @@ git_data_dirs({ 'storage1' => { 'gitaly_address' => 'tcp://gitlab.internal:8075', 'path' => '/mnt/gitlab/git-data' }, 'storage2' => { 'gitaly_address' => 'tcp://gitaly2.internal:8075' }, }) + +# Make Gitaly accept connections on all network interfaces +gitaly['listen_addr'] = "0.0.0.0:8075" + +# Or for TLS +gitaly['tls_listen_addr'] = "0.0.0.0:9999" +gitaly['certificate_path'] = "/etc/gitlab/ssl/cert.pem" +gitaly['key_path'] = "/etc/gitlab/ssl/key.pem" ``` `path` can only be included for storage shards on the local Gitaly server. @@ -532,20 +540,12 @@ corresponding to each Gitaly server must be installed on that Gitaly server. Additionally, the certificate (or its certificate authority) must be installed on all: -- Gitaly servers, including the Gitaly server using the certificate. +- Gitaly servers. - Gitaly clients that communicate with it. -The process is documented in the -[GitLab custom certificate configuration](https://docs.gitlab.com/omnibus/settings/ssl.html#install-custom-public-certificates) -and repeated below. - Note the following: -- The certificate must specify the address you use to access the Gitaly server. If you are: - - Addressing the Gitaly server by a hostname, you can either use the Common Name field for this, - or add it as a Subject Alternative Name. - - Addressing the Gitaly server by its IP address, you must add it as a Subject Alternative Name to - the certificate. [gRPC does not support using an IP address as Common Name in a certificate](https://github.com/grpc/grpc/issues/2691). +- The certificate must specify the address you use to access the Gitaly server. You must add the hostname or IP address as a Subject Alternative Name to the certificate. - You can configure Gitaly servers with both an unencrypted listening address `listen_addr` and an encrypted listening address `tls_listen_addr` at the same time. This allows you to gradually transition from unencrypted to encrypted traffic if necessary. @@ -631,17 +631,17 @@ To configure Gitaly with TLS: storages: default: gitaly_address: tls://gitaly1.internal:9999 - path: /some/dummy/path + path: /some/local/path storage1: gitaly_address: tls://gitaly1.internal:9999 - path: /some/dummy/path + path: /some/local/path storage2: gitaly_address: tls://gitaly2.internal:9999 - path: /some/dummy/path + path: /some/local/path ``` NOTE: **Note:** - `/some/dummy/path` should be set to a local folder that exists, however no data will be stored + `/some/local/path` should be set to a local folder that exists, however no data will be stored in this folder. This will no longer be necessary after [Gitaly issue #1282](https://gitlab.com/gitlab-org/gitaly/-/issues/1282) is resolved. @@ -711,6 +711,15 @@ Gitaly Go process. Some examples of things that are implemented in `gitaly-ruby` - RPCs that deal with wikis. - RPCs that create commits on behalf of a user, such as merge commits. +We recommend: + +- At least 300MB memory per worker. +- No more than one worker per core. + +NOTE: **Note:** +`gitaly-ruby` is planned to be eventually removed. To track progress, see the +[Remove the Gitaly-Ruby sidecar](https://gitlab.com/groups/gitlab-org/-/epics/2862) epic. + ### Configure number of `gitaly-ruby` workers `gitaly-ruby` has much less capacity than Gitaly implemented in Go. If your Gitaly server has to handle lots of @@ -1021,6 +1030,9 @@ The second facet presents the only real solution. For this, we developed ## Troubleshooting Gitaly +Check [Gitaly timeouts](../../user/admin_area/settings/gitaly_timeouts.md) when troubleshooting +Gitaly. + ### Checking versions when using standalone Gitaly servers When using standalone Gitaly servers, you must make sure they are the same version @@ -1242,13 +1254,6 @@ unset http_proxy unset https_proxy ``` -### Gitaly not listening on new address after reconfiguring - -When updating the `gitaly['listen_addr']` or `gitaly['prometheus_listen_addr']` -values, Gitaly may continue to listen on the old address after a `sudo gitlab-ctl reconfigure`. - -When this occurs, performing a `sudo gitlab-ctl restart` will resolve the issue. This will no longer be necessary after [this issue](https://gitlab.com/gitlab-org/gitaly/-/issues/2521) is resolved. - ### Permission denied errors appearing in Gitaly logs when accessing repositories from a standalone Gitaly server If this error occurs even though file permissions are correct, it's likely that diff --git a/doc/administration/gitaly/praefect.md b/doc/administration/gitaly/praefect.md index 876904a2093..d091ae5895a 100644 --- a/doc/administration/gitaly/praefect.md +++ b/doc/administration/gitaly/praefect.md @@ -182,7 +182,7 @@ failure. For greater fault tolerance, the following options are available: - For Geo instances, either: - Set up a separate [PostgreSQL instance](https://www.postgresql.org/docs/11/high-availability.html). - Use a cloud-managed PostgreSQL service. AWS - [Relational Database Service](https://aws.amazon.com/rds/)) is recommended. + [Relational Database Service](https://aws.amazon.com/rds/) is recommended. To complete this section you will need: @@ -356,7 +356,7 @@ application server, or a Gitaly node. If you want to use a TLS client certificate, the options below can be used: ```ruby - # Connect to PostreSQL using a TLS client certificate + # Connect to PostgreSQL using a TLS client certificate # praefect['database_sslcert'] = '/path/to/client-cert' # praefect['database_sslkey'] = '/path/to/client-key' @@ -547,14 +547,14 @@ To configure Praefect with TLS: storages: default: gitaly_address: tls://praefect1.internal:3305 - path: /some/dummy/path + path: /some/local/path storage1: gitaly_address: tls://praefect2.internal:3305 - path: /some/dummy/path + path: /some/local/path ``` NOTE: **Note:** - `/some/dummy/path` should be set to a local folder that exists, however no + `/some/local/path` should be set to a local folder that exists, however no data will be stored in this folder. This will no longer be necessary after [this issue](https://gitlab.com/gitlab-org/gitaly/-/issues/1282) is resolved. @@ -873,10 +873,10 @@ Particular attention should be shown to: gitlab-ctl reconfigure ``` -1. Verify each `gitlab-shell` on each Gitaly node can reach GitLab. On each Gitaly node run: +1. Verify on each Gitaly node the Git Hooks can reach GitLab. On each Gitaly node run: ```shell - /opt/gitlab/embedded/service/gitlab-shell/bin/check -config /opt/gitlab/embedded/service/gitlab-shell/config.yml + /opt/gitlab/embedded/bin/gitaly-hooks check /var/opt/gitlab/gitaly/config.toml ``` 1. Verify that GitLab can reach Praefect: @@ -943,16 +943,17 @@ cluster. ## Distributed reads > - Introduced in GitLab 13.1 in [beta](https://about.gitlab.com/handbook/product/gitlab-the-product/#alpha-beta-ga) with feature flag `gitaly_distributed_reads` set to disabled. -> - [Made generally available](https://gitlab.com/gitlab-org/gitaly/-/issues/2951) in GitLab 13.3. +> - [Made generally available and enabled by default](https://gitlab.com/gitlab-org/gitaly/-/issues/2951) in GitLab 13.3. +> - [Disabled by default](https://gitlab.com/gitlab-org/gitaly/-/issues/3178) in GitLab 13.5. Praefect supports distribution of read operations across Gitaly nodes that are configured for the virtual node. -The feature is enabled by default. To disable distributed reads, the `gitaly_distributed_reads` -[feature flag](../feature_flags.md) must be disabled in a Ruby console: +The feature is disabled by default. To enable distributed reads, the `gitaly_distributed_reads` +[feature flag](../feature_flags.md) must be enabled in a Ruby console: ```ruby -Feature.disable(:gitaly_distributed_reads) +Feature.enable(:gitaly_distributed_reads) ``` If enabled, all RPCs marked with `ACCESSOR` option like @@ -993,6 +994,8 @@ information, see the [strong consistency epic](https://gitlab.com/groups/gitlab- To enable strong consistency: +- In GitLab 13.5, you must use Git v2.28.0 or higher on Gitaly nodes to enable + strong consistency. - In GitLab 13.4 and later, the strong consistency voting strategy has been improved. Instead of requiring all nodes to agree, only the primary and half of the secondaries need to agree. This strategy is enabled by default. To diff --git a/doc/administration/gitaly/reference.md b/doc/administration/gitaly/reference.md index 0c211c220d7..53001b946d8 100644 --- a/doc/administration/gitaly/reference.md +++ b/doc/administration/gitaly/reference.md @@ -138,8 +138,8 @@ Most of the time we use `git cat-file --batch` processes for that. For better performance, Gitaly can re-use these `git cat-file` processes across RPC calls. Previously used processes are kept around in a ["Git cat-file cache"](https://about.gitlab.com/blog/2019/07/08/git-performance-on-nfs/#enter-cat-file-cache). -In order to control how much system resources this uses, we have a maximum number -of cat-file processes that can go into the cache. +To control how much system resources this uses, we have a maximum number of +cat-file processes that can go into the cache. The default limit is 100 `cat-file`s, which constitute a pair of `git cat-file --batch` and `git cat-file --batch-check` processes. If diff --git a/doc/administration/high_availability/README.md b/doc/administration/high_availability/README.md deleted file mode 100644 index d36b029cbb3..00000000000 --- a/doc/administration/high_availability/README.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -redirect_to: ../reference_architectures/index.md ---- - -# Reference Architectures - -This document was moved to [another location](../reference_architectures/index.md). diff --git a/doc/administration/high_availability/alpha_database.md b/doc/administration/high_availability/alpha_database.md deleted file mode 100644 index 99c28e5c7a6..00000000000 --- a/doc/administration/high_availability/alpha_database.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: 'database.md' ---- - -This document was moved to [another location](../postgresql/index.md). diff --git a/doc/administration/high_availability/consul.md b/doc/administration/high_availability/consul.md deleted file mode 100644 index 362d6ee8ba7..00000000000 --- a/doc/administration/high_availability/consul.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: ../consul.md ---- - -This document was moved to [another location](../consul.md). diff --git a/doc/administration/high_availability/database.md b/doc/administration/high_availability/database.md deleted file mode 100644 index 784e496d10e..00000000000 --- a/doc/administration/high_availability/database.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../postgresql/index.md' ---- - -This document was moved to [another location](../postgresql/index.md). diff --git a/doc/administration/high_availability/gitaly.md b/doc/administration/high_availability/gitaly.md deleted file mode 100644 index a1e8fe3b488..00000000000 --- a/doc/administration/high_availability/gitaly.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: ../gitaly/index.md ---- - -This document was moved to [another location](../gitaly/index.md). diff --git a/doc/administration/high_availability/gitlab.md b/doc/administration/high_availability/gitlab.md deleted file mode 100644 index 748373c8941..00000000000 --- a/doc/administration/high_availability/gitlab.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: ../reference_architectures/index.md ---- - -This document was moved to [another location](../reference_architectures/index.md). diff --git a/doc/administration/high_availability/img/fully-distributed.png b/doc/administration/high_availability/img/fully-distributed.png Binary files differdeleted file mode 100644 index c3cd2bf24f0..00000000000 --- a/doc/administration/high_availability/img/fully-distributed.png +++ /dev/null diff --git a/doc/administration/high_availability/img/horizontal.png b/doc/administration/high_availability/img/horizontal.png Binary files differdeleted file mode 100644 index 75d08e1097a..00000000000 --- a/doc/administration/high_availability/img/horizontal.png +++ /dev/null diff --git a/doc/administration/high_availability/img/hybrid.png b/doc/administration/high_availability/img/hybrid.png Binary files differdeleted file mode 100644 index 8dd9923e597..00000000000 --- a/doc/administration/high_availability/img/hybrid.png +++ /dev/null diff --git a/doc/administration/high_availability/load_balancer.md b/doc/administration/high_availability/load_balancer.md deleted file mode 100644 index 5cedc4e11ae..00000000000 --- a/doc/administration/high_availability/load_balancer.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: ../load_balancer.md ---- - -This document was moved to [another location](../load_balancer.md). diff --git a/doc/administration/high_availability/monitoring_node.md b/doc/administration/high_availability/monitoring_node.md deleted file mode 100644 index 76bcf6d0d40..00000000000 --- a/doc/administration/high_availability/monitoring_node.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: ../monitoring/prometheus/index.md ---- - -This document was moved to [another location](../monitoring/prometheus/index.md). diff --git a/doc/administration/high_availability/nfs.md b/doc/administration/high_availability/nfs.md deleted file mode 100644 index e3342fa0813..00000000000 --- a/doc/administration/high_availability/nfs.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: ../nfs.md ---- - -This document was moved to [another location](../nfs.md). diff --git a/doc/administration/high_availability/nfs_host_client_setup.md b/doc/administration/high_availability/nfs_host_client_setup.md deleted file mode 100644 index e3342fa0813..00000000000 --- a/doc/administration/high_availability/nfs_host_client_setup.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: ../nfs.md ---- - -This document was moved to [another location](../nfs.md). diff --git a/doc/administration/high_availability/object_storage.md b/doc/administration/high_availability/object_storage.md deleted file mode 100644 index eeb730d3cc7..00000000000 --- a/doc/administration/high_availability/object_storage.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../object_storage.md' ---- - -This document was moved to [another location](../object_storage.md). diff --git a/doc/administration/high_availability/pgbouncer.md b/doc/administration/high_availability/pgbouncer.md deleted file mode 100644 index 44f4aa37651..00000000000 --- a/doc/administration/high_availability/pgbouncer.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: ../postgresql/pgbouncer.md ---- - -This document was moved to [another location](../postgresql/pgbouncer.md). diff --git a/doc/administration/high_availability/redis.md b/doc/administration/high_availability/redis.md deleted file mode 100644 index 2b5771f49f2..00000000000 --- a/doc/administration/high_availability/redis.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: ../redis/index.md ---- - -This document was moved to [another location](../redis/index.md). diff --git a/doc/administration/high_availability/redis_source.md b/doc/administration/high_availability/redis_source.md deleted file mode 100644 index 75496638979..00000000000 --- a/doc/administration/high_availability/redis_source.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: ../redis/replication_and_failover_external.md ---- - -This document was moved to [another location](../redis/replication_and_failover_external.md). diff --git a/doc/administration/high_availability/sidekiq.md b/doc/administration/high_availability/sidekiq.md deleted file mode 100644 index ac92ae2eaaa..00000000000 --- a/doc/administration/high_availability/sidekiq.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: ../sidekiq.md ---- - -This document was moved to [another location](../sidekiq.md). diff --git a/doc/administration/housekeeping.md b/doc/administration/housekeeping.md index 4110f8b7646..c784c788b31 100644 --- a/doc/administration/housekeeping.md +++ b/doc/administration/housekeeping.md @@ -1,3 +1,9 @@ +--- +stage: none +group: unassigned +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Housekeeping > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/3041) in GitLab 8.4. @@ -28,6 +34,9 @@ the `pushes_since_gc` value is 200 a `git gc` will be run. `git add`. - `git repack` ([man page](https://mirrors.edge.kernel.org/pub/software/scm/git/docs/git-repack.html)) re-organize existing packs into a single, more efficient pack. +Housekeeping will also [remove unreferenced LFS files](../raketasks/cleanup.md#remove-unreferenced-lfs-files) +from your project on the same schedule as the `git gc` operation, freeing up storage space for your project. + You can find this option under your project's **Settings > General > Advanced**.  diff --git a/doc/administration/img/export_audit_log_v13_4.png b/doc/administration/img/export_audit_log_v13_4.png Binary files differindex 1b404b5742c..e4ba330b8a9 100644 --- a/doc/administration/img/export_audit_log_v13_4.png +++ b/doc/administration/img/export_audit_log_v13_4.png diff --git a/doc/administration/incoming_email.md b/doc/administration/incoming_email.md index c0c03044225..bd075e86a15 100644 --- a/doc/administration/incoming_email.md +++ b/doc/administration/incoming_email.md @@ -82,15 +82,15 @@ instead of the regular password for the mailbox. To set up a basic Postfix mail server with IMAP access on Ubuntu, follow the [Postfix setup documentation](reply_by_email_postfix_setup.md). -### Security Concerns +### Security concerns -WARNING: **WARNING:** +CAUTION: **Caution:** Be careful when choosing the domain used for receiving incoming email. -For the sake of example, suppose your top-level company domain is `hooli.com`. +For example, suppose your top-level company domain is `hooli.com`. All employees in your company have an email address at that domain via Google Apps, and your company's private Slack instance requires a valid `@hooli.com` -email address in order to sign up. +email address to sign up. If you also host a public-facing GitLab instance at `hooli.com` and set your incoming email domain to `hooli.com`, an attacker could abuse the "Create new @@ -112,7 +112,7 @@ See GitLab issue [#30366](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/303 for a real-world example of this exploit. CAUTION: **Caution:** -Be sure to use a mail server that has been configured to reduce +Use a mail server that has been configured to reduce spam. A Postfix mail server that is running on a default configuration, for example, can result in abuse. All messages received on the configured mailbox will be processed diff --git a/doc/administration/index.md b/doc/administration/index.md index a6448fcf64f..07aa3b50bc6 100644 --- a/doc/administration/index.md +++ b/doc/administration/index.md @@ -1,8 +1,11 @@ --- +stage: none +group: unassigned +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers description: 'Learn how to install, configure, update, and maintain your GitLab instance.' --- -# Administrator Docs **(CORE ONLY)** +# Administrator documentation **(CORE ONLY)** Learn how to administer your self-managed GitLab instance. @@ -12,18 +15,16 @@ GitLab has two product distributions available through [different subscriptions] - The open core [GitLab Enterprise Edition (EE)](https://gitlab.com/gitlab-org/gitlab). You can [install either GitLab CE or GitLab EE](https://about.gitlab.com/install/ce-or-ee/). -However, the features you'll have access to depend on the subscription you choose -(Core, Starter, Premium, or Ultimate). +However, the features you have access to depend on your chosen [subscription](https://about.gitlab.com/pricing/). -NOTE: **Note:** -GitLab Community Edition installations only have access to Core features. +GitLab Community Edition installations have access only to Core features. -GitLab.com is administered by GitLab, Inc., therefore, only GitLab team members have -access to its admin configurations. If you're a GitLab.com user, please check the -[user documentation](../user/index.md). +Non-administrator users can't access GitLab administration tools and settings. -NOTE: **Note:** -Non-administrator users don’t have access to GitLab administration tools and settings. +GitLab.com is administered by GitLab, Inc., and only GitLab team members have +access to its administration tools and settings. Users of GitLab.com should +instead refer to the [User documentation](../user/index.md) for GitLab +configuration and usage documentation. ## Installing and maintaining GitLab @@ -52,8 +53,10 @@ Learn how to install, configure, update, and maintain your GitLab instance. - [GitLab Pages configuration](pages/index.md): Enable and configure GitLab Pages. - [GitLab Pages configuration for GitLab source installations](pages/source.md): Enable and configure GitLab Pages on [source installations](../install/installation.md#installation-from-source). - [Uploads administration](uploads.md): Configure GitLab uploads storage. -- [Environment variables](environment_variables.md): Supported environment variables that can be used to override their defaults values in order to configure GitLab. -- [Plugins](plugins.md): With custom plugins, GitLab administrators can introduce custom integrations without modifying GitLab's source code. +- [Environment variables](environment_variables.md): Supported environment + variables that can be used to override their default values to configure + GitLab. +- [Plugins](file_hooks.md): With custom plugins, GitLab administrators can introduce custom integrations without modifying GitLab's source code. - [Enforcing Terms of Service](../user/admin_area/settings/terms.md) - [Third party offers](../user/admin_area/settings/third_party_offers.md) - [Compliance](compliance.md): A collection of features from across the application that you may configure to help ensure that your GitLab instance and DevOps workflow meet compliance standards. @@ -82,6 +85,7 @@ Learn how to install, configure, update, and maintain your GitLab instance. - [Operations](operations/index.md): Keeping GitLab up and running (clean up Redis sessions, moving repositories, Sidekiq MemoryKiller, Puma). - [Restart GitLab](restart_gitlab.md): Learn how to restart GitLab and its components. - [Invalidate Markdown cache](invalidate_markdown_cache.md): Invalidate any cached Markdown. +- [Instance review](instance_review.md): Request a free review of your GitLab instance. #### Updating GitLab @@ -113,7 +117,7 @@ Learn how to install, configure, update, and maintain your GitLab instance. - [Kerberos authentication](../integration/kerberos.md) **(STARTER ONLY)** - See also other [authentication](../topics/authentication/index.md#gitlab-administrators) topics (for example, enforcing 2FA). - [Email users](../tools/email.md): Email GitLab users from within GitLab. **(STARTER ONLY)** -- [User Cohorts](../user/admin_area/user_cohorts.md): Display the monthly cohorts of new users and their activities over time. +- [User Cohorts](../user/admin_area/analytics/user_cohorts.md): Display the monthly cohorts of new users and their activities over time. - [Audit logs and events](audit_events.md): View the changes made within the GitLab server for: - Groups and projects. **(STARTER)** - Instances. **(PREMIUM ONLY)** diff --git a/doc/administration/instance_limits.md b/doc/administration/instance_limits.md index abd98002934..cb37be8d9dd 100644 --- a/doc/administration/instance_limits.md +++ b/doc/administration/instance_limits.md @@ -1,4 +1,7 @@ --- +stage: none +group: unassigned +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference --- @@ -163,7 +166,7 @@ There is a limit when embedding metrics in GFM for performance reasons. On GitLab.com, the [maximum number of webhooks and their size](../user/gitlab_com/index.md#webhooks) per project, and per group, is limited. To set this limit on a self-managed installation, run the following in the -[GitLab Rails console](troubleshooting/debug.md#starting-a-rails-console-session): +[GitLab Rails console](operations/rails_console.md#starting-a-rails-console-session): ```ruby # If limits don't exist for the default plan, you can create one with: @@ -203,7 +206,7 @@ support keyset-based pagination. More information about pagination options can b found in the [API docs section on pagination](../api/README.md#pagination). To set this limit on a self-managed installation, run the following in the -[GitLab Rails console](troubleshooting/debug.md#starting-a-rails-console-session): +[GitLab Rails console](operations/rails_console.md#starting-a-rails-console-session): ```ruby # If limits don't exist for the default plan, you can create one with: @@ -238,7 +241,7 @@ will fail with a `job_activity_limit_exceeded` error. This limit is disabled by default. To set this limit on a self-managed installation, run the following in the -[GitLab Rails console](troubleshooting/debug.md#starting-a-rails-console-session): +[GitLab Rails console](operations/rails_console.md#starting-a-rails-console-session): ```ruby # If limits don't exist for the default plan, you can create one with: @@ -264,7 +267,7 @@ limit, the subscription will be considered invalid. - On [GitLab Starter](https://about.gitlab.com/pricing/#self-managed) tier or higher self-managed installations, this limit is defined for the `default` plan that affects all projects. To set this limit on a self-managed installation, run the following in the -[GitLab Rails console](troubleshooting/debug.md#starting-a-rails-console-session): +[GitLab Rails console](operations/rails_console.md#starting-a-rails-console-session): ```ruby Plan.default.actual_limits.update!(ci_project_subscriptions: 500) @@ -290,7 +293,7 @@ or higher tiers), this limit is defined for the `default` plan that affects all projects. By default, there is no limit. To set this limit on a self-managed installation, run the following in the -[GitLab Rails console](troubleshooting/debug.md#starting-a-rails-console-session): +[GitLab Rails console](operations/rails_console.md#starting-a-rails-console-session): ```ruby Plan.default.actual_limits.update!(ci_pipeline_schedules: 100) @@ -308,7 +311,7 @@ On self-managed instances this limit is defined for the `default` plan. By defau this limit is set to `25`. To update this limit to a new value on a self-managed installation, run the following in the -[GitLab Rails console](troubleshooting/debug.md#starting-a-rails-console-session): +[GitLab Rails console](operations/rails_console.md#starting-a-rails-console-session): ```ruby Plan.default.actual_limits.update!(ci_instance_level_variables: 30) @@ -333,6 +336,7 @@ setting is used: | Artifact limit name | Default value | |---------------------------------------------|---------------| | `ci_max_artifact_size_accessibility` | 0 | +| `ci_max_artifact_size_api_fuzzing` | 0 | | `ci_max_artifact_size_archive` | 0 | | `ci_max_artifact_size_browser_performance` | 0 | | `ci_max_artifact_size_cluster_applications` | 0 | @@ -360,7 +364,7 @@ setting is used: | `ci_max_artifact_size_trace` | 0 | For example, to set the `ci_max_artifact_size_junit` limit to 10MB on a self-managed -installation, run the following in the [GitLab Rails console](troubleshooting/debug.md#starting-a-rails-console-session): +installation, run the following in the [GitLab Rails console](operations/rails_console.md#starting-a-rails-console-session): ```ruby Plan.default.actual_limits.update!(ci_max_artifact_size_junit: 10) @@ -480,7 +484,7 @@ indexed](#maximum-file-size-indexed)). - For self-managed installations it is unlimited by default This limit can be configured for self-managed installations when [enabling -Elasticsearch](../integration/elasticsearch.md#enabling-elasticsearch). +Elasticsearch](../integration/elasticsearch.md#enabling-advanced-search). NOTE: **Note:** Set the limit to `0` to disable it. @@ -527,13 +531,17 @@ More information can be found in the [Push event activities limit and bulk push > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/218017) in GitLab 13.4. -On GitLab.com, the maximum file size for a package that's uploaded to the [GitLab Package Registry](../user/packages/package_registry/index.md) -is 5 gigabytes. +On GitLab.com, the maximum file size for a package that's uploaded to the [GitLab Package Registry](../user/packages/package_registry/index.md) varies by format: -Limits are set per package type. +- Conan: 3GB +- Generic: 5GB +- Maven: 3GB +- NPM: 500MB +- NuGet: 500MB +- PyPI: 3GB To set this limit on a self-managed installation, run the following in the -[GitLab Rails console](troubleshooting/debug.md#starting-a-rails-console-session): +[GitLab Rails console](operations/rails_console.md#starting-a-rails-console-session): ```ruby # File size limit is stored in bytes @@ -552,6 +560,12 @@ Plan.default.actual_limits.update!(maven_max_file_size: 100.megabytes) # For PyPI Packages Plan.default.actual_limits.update!(pypi_max_file_size: 100.megabytes) + +# For Debian Packages +Plan.default.actual_limits.update!(debian_max_file_size: 100.megabytes) + +# For Generic Packages +Plan.default.actual_limits.update!(generic_packages_max_file_size: 100.megabytes) ``` Set the limit to `0` to allow any file size. diff --git a/doc/administration/instance_review.md b/doc/administration/instance_review.md index 8ea4bff252e..7eadb54804b 100644 --- a/doc/administration/instance_review.md +++ b/doc/administration/instance_review.md @@ -1,13 +1,25 @@ +--- +stage: Growth +group: Conversion +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Instance Review **(CORE ONLY)** > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/6995) in [GitLab Core](https://about.gitlab.com/pricing/) 11.3. -If you are running a medium size instance of GitLab Core edition you are qualified for a free Instance Review. You can find the button in the User menu. +If you are running a medium size instance (50+ users) of +[GitLab Core](https://about.gitlab.com/pricing/) edition, you are qualified for a +free Instance Review. - +1. Sign in as a user with Admin [permissions](../user/permissions.md). +1. In the top menu, click your user icon, and select + **Get a free instance review**: -When you click the button you will be redirected to a form with prefilled data obtained from your instance. +  -Once you submit the data to GitLab Inc. you can see the initial report. +1. GitLab redirects you to a form with prefilled data obtained from your instance. +1. Click **Submit** to see the initial report. -Additionally you will be contacted by our team for further review which should help you to improve your usage of GitLab. +A GitLab team member will contact you for further review, to provide suggestions +that will help you improve your usage of GitLab. diff --git a/doc/administration/integration/plantuml.md b/doc/administration/integration/plantuml.md index 49ea59d239c..5bdea9d8843 100644 --- a/doc/administration/integration/plantuml.md +++ b/doc/administration/integration/plantuml.md @@ -132,7 +132,7 @@ stop; You need to enable PlantUML integration from Settings under Admin Area. To do that, login with an Admin account and do following: -- In GitLab, go to **Admin Area > Settings > Integrations**. +- In GitLab, go to **Admin Area > Settings > General**. - Expand the **PlantUML** section. - Check **Enable PlantUML** checkbox. - Set the PlantUML instance as `https://gitlab.example.com/-/plantuml/`. diff --git a/doc/administration/integration/terminal.md b/doc/administration/integration/terminal.md index c363bd30543..3c53535d856 100644 --- a/doc/administration/integration/terminal.md +++ b/doc/administration/integration/terminal.md @@ -1,3 +1,9 @@ +--- +stage: none +group: unassigned +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Web terminals > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/7690) in GitLab 8.15. diff --git a/doc/administration/job_artifacts.md b/doc/administration/job_artifacts.md index 2a79923b793..8069b12e0b9 100644 --- a/doc/administration/job_artifacts.md +++ b/doc/administration/job_artifacts.md @@ -99,8 +99,13 @@ artifacts, you can use an object storage like AWS S3 instead. This configuration relies on valid AWS credentials to be configured already. Use an object storage option like AWS S3 to store job artifacts. +If you configure GitLab to store artifacts on object storage, you may also want to +[eliminate local disk usage for job logs](job_logs.md#prevent-local-disk-usage). +In both cases, job logs are archived and moved to object storage when the job completes. + DANGER: **Danger:** -If you configure GitLab to store CI logs and artifacts on object storage, you must also enable [incremental logging](job_logs.md#new-incremental-logging-architecture). Otherwise, job logs will disappear or not be saved. +In a multi-server setup you must use one of the options to +[eliminate local disk usage for job logs](job_logs.md#prevent-local-disk-usage), or job logs could be lost. [Read more about using object storage with GitLab](object_storage.md). @@ -117,9 +122,9 @@ For source installations the following settings are nested under `artifacts:` an |---------|-------------|---------| | `enabled` | Enable/disable object storage | `false` | | `remote_directory` | The bucket name where Artifacts will be stored| | -| `direct_upload` | Set to true to enable direct upload of Artifacts without the need of local shared storage. Option may be removed once we decide to support only single storage for all files. | `false` | -| `background_upload` | Set to false to disable automatic upload. Option may be removed once upload is direct to S3 | `true` | -| `proxy_download` | Set to true to enable proxying all files served. Option allows to reduce egress traffic as this allows clients to download directly from remote storage instead of proxying all data | `false` | +| `direct_upload` | Set to `true` to enable direct upload of Artifacts without the need of local shared storage. Option may be removed once we decide to support only single storage for all files. | `false` | +| `background_upload` | Set to `false` to disable automatic upload. Option may be removed once upload is direct to S3 | `true` | +| `proxy_download` | Set to `true` to enable proxying all files served. Option allows to reduce egress traffic as this allows clients to download directly from remote storage instead of proxying all data | `false` | | `connection` | Various connection options described below | | #### Connection settings @@ -203,9 +208,9 @@ _The artifacts are stored by default in enabled: true object_store: enabled: true - remote_directory: "artifacts" # The bucket name + remote_directory: "artifacts" # The bucket name connection: - provider: AWS # Only AWS supported at the moment + provider: AWS # Only AWS supported at the moment aws_access_key_id: AWS_ACCESS_KEY_ID aws_secret_access_key: AWS_SECRET_ACCESS_KEY region: eu-central-1 @@ -316,9 +321,9 @@ _The uploads are stored by default in **In Omnibus installations:** -In order to migrate back to local storage: +To migrate back to local storage: -1. Set both `direct_upload` and `background_upload` to false in `gitlab.rb`, under the artifacts object storage settings. +1. Set both `direct_upload` and `background_upload` to `false` in `gitlab.rb`, under the artifacts object storage settings. 1. [Reconfigure GitLab](restart_gitlab.md#omnibus-gitlab-reconfigure). 1. Run `gitlab-rake gitlab:artifacts:migrate_to_local`. 1. Disable object_storage for artifacts in `gitlab.rb`: @@ -369,7 +374,7 @@ default artifacts expiration setting, which you can find in the [CI/CD Admin set > Introduced in GitLab 10.3. -To disable [the dependencies validation](../ci/yaml/README.md#when-a-dependent-job-will-fail), +To disable [the dependencies validation](../ci/yaml/README.md#when-a-dependent-job-fails), you can enable the `ci_disable_validates_dependencies` feature flag from a Rails console. **In Omnibus installations:** @@ -419,10 +424,10 @@ generated by [GitLab Workhorse](https://gitlab.com/gitlab-org/gitlab-workhorse). that are located in the artifacts archive itself. The metadata file is in a binary format, with additional Gzip compression. -GitLab does not extract the artifacts archive in order to save space, memory -and disk I/O. It instead inspects the metadata file which contains all the -relevant information. This is especially important when there is a lot of -artifacts, or an archive is a very large file. +GitLab doesn't extract the artifacts archive to save space, memory, and disk +I/O. It instead inspects the metadata file which contains all the relevant +information. This is especially important when there is a lot of artifacts, or +an archive is a very large file. When clicking on a specific file, [GitLab Workhorse](https://gitlab.com/gitlab-org/gitlab-workhorse) extracts it from the archive and the download begins. This implementation saves space, diff --git a/doc/administration/job_logs.md b/doc/administration/job_logs.md index c34035e3c0c..c89ffb8da8b 100644 --- a/doc/administration/job_logs.md +++ b/doc/administration/job_logs.md @@ -65,6 +65,15 @@ job logs are automatically migrated to it along with the other job artifacts. See "Phase 4: uploading" in [Data flow](#data-flow) to learn about the process. +## Prevent local disk usage + +If you want to avoid any local disk usage for job logs, +you can do so using one of the following options: + +- Enable the [beta incremental logging](#new-incremental-logging-architecture) feature. +- Set the [job logs location](#changing-the-job-logs-local-location) + to an NFS drive. + ## How to remove job logs There isn't a way to automatically expire old job logs, but it's safe to remove diff --git a/doc/administration/lfs/index.md b/doc/administration/lfs/index.md index 5c1a9519a35..a360542ac44 100644 --- a/doc/administration/lfs/index.md +++ b/doc/administration/lfs/index.md @@ -6,14 +6,16 @@ type: reference, howto disqus_identifier: 'https://docs.gitlab.com/ee/workflow/lfs/lfs_administration.html' --- -# GitLab Git Large File Storage (LFS) Administration +# GitLab Git Large File Storage (LFS) Administration **(CORE ONLY)** + +> - Git LFS is supported in GitLab starting with version 8.2. +> - Support for object storage, such as AWS S3, was introduced in 10.0. +> - LFS is enabled in GitLab self-managed instances by default. Documentation on how to use Git LFS are under [Managing large binary files with Git LFS doc](../../topics/git/lfs/index.md). ## Requirements -- Git LFS is supported in GitLab starting with version 8.2. -- Support for object storage, such as AWS S3, was introduced in 10.0. - Users need to install [Git LFS client](https://git-lfs.github.com) version 1.0.1 and up. ## Configuration @@ -41,6 +43,8 @@ gitlab_rails['lfs_enabled'] = false gitlab_rails['lfs_storage_path'] = "/mnt/storage/lfs-objects" ``` +After you update settings in `/etc/gitlab/gitlab.rb`, make sure to run [Omnibus GitLab reconfigure](../restart_gitlab.md#omnibus-gitlab-reconfigure). + ### Configuration for installations from source In `config/gitlab.yml`: diff --git a/doc/administration/libravatar.md b/doc/administration/libravatar.md index beecd9e4783..a92e6fade03 100644 --- a/doc/administration/libravatar.md +++ b/doc/administration/libravatar.md @@ -1,4 +1,7 @@ --- +stage: none +group: unassigned +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: howto --- @@ -72,10 +75,9 @@ Then run `sudo gitlab-ctl reconfigure` for the changes to take effect. missing images for user email addresses that are not found on the Libravatar service. -In order to use a set other than `identicon`, replace the `&d=identicon` -portion of the URL with another supported set. -For example, you can use the `retro` set, in which case the URL would look like: -`plain_url: "http://cdn.libravatar.org/avatar/%{hash}?s=%{size}&d=retro"` +To use a set other than `identicon`, replace the `&d=identicon` portion of the +URL with another supported set. For example, you can use the `retro` set, in +which case the URL would look like: `plain_url: "http://cdn.libravatar.org/avatar/%{hash}?s=%{size}&d=retro"` ## Usage examples for Microsoft Office 365 @@ -84,8 +86,8 @@ Note that this service requires a login, so this use case is most useful in a corporate installation where all users have access to Office 365. ```ruby -gitlab_rails['gravatar_plain_url'] = 'http://outlook.office365.com/owa/service.svc/s/GetPersonaPhoto?email=%{email}&size=HR120x120' -gitlab_rails['gravatar_ssl_url'] = 'https://outlook.office365.com/owa/service.svc/s/GetPersonaPhoto?email=%{email}&size=HR120x120' +gitlab_rails['gravatar_plain_url'] = 'http://outlook.office.com/owa/service.svc/s/GetPersonaPhoto?email=%{email}&size=HR120x120' +gitlab_rails['gravatar_ssl_url'] = 'https://outlook.office.com/owa/service.svc/s/GetPersonaPhoto?email=%{email}&size=HR120x120' ``` <!-- ## Troubleshooting diff --git a/doc/administration/load_balancer.md b/doc/administration/load_balancer.md index fe534f30f66..ae4fa83662a 100644 --- a/doc/administration/load_balancer.md +++ b/doc/administration/load_balancer.md @@ -1,4 +1,7 @@ --- +stage: none +group: unassigned +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference --- diff --git a/doc/administration/logs.md b/doc/administration/logs.md index fcd6264dafd..c9e0cca807e 100644 --- a/doc/administration/logs.md +++ b/doc/administration/logs.md @@ -1,6 +1,6 @@ --- stage: Monitor -group: APM +group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- @@ -100,9 +100,9 @@ The ActionCable connection or channel class is used as the `controller`. ```json { - "method":{}, - "path":{}, - "format":{}, + "method":null, + "path":null, + "format":null, "controller":"IssuesChannel", "action":"subscribe", "status":200, @@ -363,8 +363,7 @@ This file lives in `/var/log/gitlab/gitlab-rails/git_json.log` for Omnibus GitLab packages or in `/home/git/gitlab/log/git_json.log` for installations from source. -NOTE: **Note:** -After 12.2, this file was renamed from `githost.log` to +After GitLab version 12.2, this file was renamed from `githost.log` to `git_json.log` and stored in JSON format. GitLab has to interact with Git repositories, but in some rare cases @@ -599,7 +598,6 @@ installations from source. ## Unicorn Logs -NOTE: **Note:** Starting with GitLab 13.0, Puma is the default web server used in GitLab all-in-one package based installations as well as GitLab Helm chart deployments. @@ -674,10 +672,8 @@ This log records: - Information whenever [Rack Attack](../security/rack_attack.md) registers an abusive request. - Requests over the [Rate Limit](../user/admin_area/settings/rate_limits_on_raw_endpoints.md) on raw endpoints. - [Protected paths](../user/admin_area/settings/protected_paths.md) abusive requests. - -NOTE: **Note:** -In GitLab versions [12.3](https://gitlab.com/gitlab-org/gitlab/-/issues/29239) and greater, user ID and username are also -recorded on this log, if available. +- In GitLab versions [12.3](https://gitlab.com/gitlab-org/gitlab/-/issues/29239) and greater, + user ID and username, if available. ## `graphql_json.log` @@ -967,19 +963,9 @@ When [troubleshooting](troubleshooting/index.md) issues that aren't localized to previously listed components, it's helpful to simultaneously gather multiple logs and statistics from a GitLab instance. -### GitLabSOS - -If performance degradations or cascading errors occur that can't readily be attributed to one -of the previously listed GitLab components, [GitLabSOS](https://gitlab.com/gitlab-com/support/toolbox/gitlabsos/) -can provide a perspective spanning all of Omnibus GitLab. For more details and instructions -to run it, see [the GitLabSOS documentation](https://gitlab.com/gitlab-com/support/toolbox/gitlabsos/#gitlabsos). - -NOTE: **Note:** -GitLab Support likes to use this custom-made tool. - ### Briefly tail the main logs -If the bug or error is readily reproducible bug or error, save the main GitLab logs +If the bug or error is readily reproducible, save the main GitLab logs [to a file](troubleshooting/linux_cheat_sheet.md#files--dirs) while reproducing the problem once or more times: @@ -989,6 +975,16 @@ sudo gitlab-ctl tail | tee /tmp/<case-ID-and-keywords>.log Conclude the log gathering with <kbd>Ctrl</kbd> + <kbd>C</kbd>. +### GitLabSOS + +If performance degradations or cascading errors occur that can't readily be attributed to one +of the previously listed GitLab components, [GitLabSOS](https://gitlab.com/gitlab-com/support/toolbox/gitlabsos/) +can provide a perspective spanning all of Omnibus GitLab. For more details and instructions +to run it, see [the GitLabSOS documentation](https://gitlab.com/gitlab-com/support/toolbox/gitlabsos/#gitlabsos). + +NOTE: **Note:** +GitLab Support likes to use this custom-made tool. + ### Fast-stats [Fast-stats](https://gitlab.com/gitlab-com/support/toolbox/fast-stats) is a tool diff --git a/doc/administration/merge_request_diffs.md b/doc/administration/merge_request_diffs.md index 3f4cd6e2751..3c093d44780 100644 --- a/doc/administration/merge_request_diffs.md +++ b/doc/administration/merge_request_diffs.md @@ -17,12 +17,11 @@ By default, merge request diffs are stored in the database, in a table named `merge_request_diff_files`. Larger installations may find this table grows too large, in which case, switching to external storage is recommended. -## Using external storage - Merge request diffs can be stored on disk, or in object storage. In general, it -is better to store the diffs in the database than on disk. +is better to store the diffs in the database than on disk. A compromise is available +that only [stores outdated diffs](#alternative-in-database-storage) outside of database. -To enable external storage of merge request diffs, follow the instructions below. +## Using external storage **In Omnibus installations:** @@ -68,16 +67,40 @@ To enable external storage of merge request diffs, follow the instructions below ## Using object storage -CAUTION: **WARNING:** - Currently migrating to object storage is **non-reversible** +CAUTION: **Warning:** +Currently migrating to object storage is **non-reversible** Instead of storing the external diffs on disk, we recommended the use of an object store like AWS S3 instead. This configuration relies on valid AWS credentials to be configured already. +**In Omnibus installations:** + +1. Edit `/etc/gitlab/gitlab.rb` and add the following line: + + ```ruby + gitlab_rails['external_diffs_enabled'] = true + ``` + +1. Set [object storage settings](#object-storage-settings). +1. Save the file and [reconfigure GitLab](restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + +**In installations from source:** + +1. Edit `/home/git/gitlab/config/gitlab.yml` and add or amend the following + lines: + + ```yaml + external_diffs: + enabled: true + ``` + +1. Set [object storage settings](#object-storage-settings). +1. Save the file and [restart GitLab](restart_gitlab.md#installations-from-source) for the changes to take effect. + [Read more about using object storage with GitLab](object_storage.md). -## Object Storage Settings +### Object Storage Settings NOTE: **Note:** In GitLab 13.2 and later, we recommend using the @@ -92,12 +115,12 @@ then `object_store:`. On Omnibus installations, they are prefixed by |---------|-------------|---------| | `enabled` | Enable/disable object storage | `false` | | `remote_directory` | The bucket name where external diffs will be stored| | -| `direct_upload` | Set to true to enable direct upload of external diffs without the need of local shared storage. Option may be removed once we decide to support only single storage for all files. | `false` | -| `background_upload` | Set to false to disable automatic upload. Option may be removed once upload is direct to S3 | `true` | -| `proxy_download` | Set to true to enable proxying all files served. Option allows to reduce egress traffic as this allows clients to download directly from remote storage instead of proxying all data | `false` | +| `direct_upload` | Set to `true` to enable direct upload of external diffs without the need of local shared storage. Option may be removed once we decide to support only single storage for all files. | `false` | +| `background_upload` | Set to `false` to disable automatic upload. Option may be removed once upload is direct to S3 | `true` | +| `proxy_download` | Set to `true` to enable proxying all files served. Option allows to reduce egress traffic as this allows clients to download directly from remote storage instead of proxying all data | `false` | | `connection` | Various connection options described below | | -### S3 compatible connection settings +#### S3 compatible connection settings See [the available connection settings for different providers](object_storage.md#connection-settings). diff --git a/doc/administration/monitoring/github_imports.md b/doc/administration/monitoring/github_imports.md index 0d79684c951..64f45001438 100644 --- a/doc/administration/monitoring/github_imports.md +++ b/doc/administration/monitoring/github_imports.md @@ -1,6 +1,6 @@ --- stage: Monitor -group: APM +group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- diff --git a/doc/administration/monitoring/gitlab_self_monitoring_project/index.md b/doc/administration/monitoring/gitlab_self_monitoring_project/index.md index efe31997b25..fcaf9aaaaee 100644 --- a/doc/administration/monitoring/gitlab_self_monitoring_project/index.md +++ b/doc/administration/monitoring/gitlab_self_monitoring_project/index.md @@ -1,6 +1,6 @@ --- stage: Monitor -group: APM +group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- @@ -9,17 +9,18 @@ info: To determine the technical writer assigned to the Stage/Group associated w > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/32351) in GitLab 12.7, behind a disabled feature flag (`self_monitoring_project`). > - The feature flag was removed and the Self Monitoring Project was [made generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/198511) in GitLab 12.8. -GitLab has been adding the ability for administrators to see insights into the health of -their GitLab instance. In order to surface this experience in a native way, similar to how -you would interact with an application deployed via GitLab, a base project called -"GitLab self monitoring" with -[internal visibility](../../../public_access/public_access.md#internal-projects) will be -added under a group called "GitLab Instance Administrators" specifically created for -visualizing and configuring the monitoring of your GitLab instance. +GitLab has been adding the ability for administrators to see insights into the +health of their GitLab instance. To surface this experience in a native way +(similar to how you would interact with an application deployed using GitLab), +a base project called "GitLab self monitoring" with +[internal visibility](../../../public_access/public_access.md#internal-projects) +will be added under a group called "GitLab Instance Administrators" +specifically created for visualizing and configuring the monitoring of your +GitLab instance. -All administrators at the time of creation of the project and group will be added -as maintainers of the group and project, and as an admin, you'll be able to add new -members to the group in order to give them maintainer access to the project. +All administrators at the time of creation of the project and group will be +added as maintainers of the group and project, and as an admin, you'll be able +to add new members to the group to give them maintainer access to the project. This project is used to self monitor your GitLab instance. The metrics dashboard of the project shows some basic resource usage charts, such as CPU and memory usage @@ -74,7 +75,8 @@ you should ## Taking action on Prometheus alerts **(ULTIMATE)** You can [add a webhook](../../../operations/metrics/alerts.md#external-prometheus-instances) -to the Prometheus configuration in order for GitLab to receive notifications of any alerts. +to the Prometheus configuration for GitLab to receive notifications of any +alerts. Once the webhook is setup, you can [take action on incoming alerts](../../../operations/metrics/alerts.md#trigger-actions-from-alerts). diff --git a/doc/administration/monitoring/index.md b/doc/administration/monitoring/index.md index a54c25450c6..fbd4c1aa3dd 100644 --- a/doc/administration/monitoring/index.md +++ b/doc/administration/monitoring/index.md @@ -1,6 +1,6 @@ --- stage: Monitor -group: APM +group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- diff --git a/doc/administration/monitoring/ip_whitelist.md b/doc/administration/monitoring/ip_whitelist.md index 862a9368be8..480a4ea3598 100644 --- a/doc/administration/monitoring/ip_whitelist.md +++ b/doc/administration/monitoring/ip_whitelist.md @@ -1,6 +1,6 @@ --- stage: Monitor -group: APM +group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- diff --git a/doc/administration/monitoring/performance/gitlab_configuration.md b/doc/administration/monitoring/performance/gitlab_configuration.md index d09dabab40d..6677eb73664 100644 --- a/doc/administration/monitoring/performance/gitlab_configuration.md +++ b/doc/administration/monitoring/performance/gitlab_configuration.md @@ -1,6 +1,6 @@ --- stage: Monitor -group: APM +group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- diff --git a/doc/administration/monitoring/performance/grafana_configuration.md b/doc/administration/monitoring/performance/grafana_configuration.md index 136a2749e80..77ed5d442e6 100644 --- a/doc/administration/monitoring/performance/grafana_configuration.md +++ b/doc/administration/monitoring/performance/grafana_configuration.md @@ -1,6 +1,6 @@ --- stage: Monitor -group: APM +group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- @@ -17,7 +17,6 @@ or Grafana supplies package repositories (Yum/Apt) for easy installation. See [Grafana installation documentation](https://grafana.com/docs/grafana/latest/installation/) for detailed steps. -NOTE: **Note:** Before starting Grafana for the first time, set the admin user and password in `/etc/grafana/grafana.ini`. If you don't, the default password is `admin`. @@ -50,7 +49,6 @@ JSON file individually: 1. After the dashboard is imported, click the **Save dashboard** icon in the top bar:  - NOTE: **Note:** If you don't save the dashboard after importing it, the dashboard is removed when you navigate away from the page. diff --git a/doc/administration/monitoring/performance/img/performance_bar.png b/doc/administration/monitoring/performance/img/performance_bar.png Binary files differindex be06e0b2f94..380e2060b24 100644 --- a/doc/administration/monitoring/performance/img/performance_bar.png +++ b/doc/administration/monitoring/performance/img/performance_bar.png diff --git a/doc/administration/monitoring/performance/img/performance_bar_sql_queries.png b/doc/administration/monitoring/performance/img/performance_bar_sql_queries.png Binary files differindex 6b571f4e85c..ef74e0a3b6e 100644 --- a/doc/administration/monitoring/performance/img/performance_bar_sql_queries.png +++ b/doc/administration/monitoring/performance/img/performance_bar_sql_queries.png diff --git a/doc/administration/monitoring/performance/index.md b/doc/administration/monitoring/performance/index.md index 6f22327e499..68b89b837ac 100644 --- a/doc/administration/monitoring/performance/index.md +++ b/doc/administration/monitoring/performance/index.md @@ -1,6 +1,6 @@ --- stage: Monitor -group: APM +group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- @@ -11,7 +11,7 @@ GitLab comes with its own application performance measuring system as of GitLab Community and Enterprise editions. Apart from this introduction, you are advised to read through the following -documents in order to understand and properly configure GitLab Performance Monitoring: +documents to understand and properly configure GitLab Performance Monitoring: - [GitLab Configuration](gitlab_configuration.md) - [Prometheus documentation](../prometheus/index.md) diff --git a/doc/administration/monitoring/performance/performance_bar.md b/doc/administration/monitoring/performance/performance_bar.md index e247ec3708c..45dc4730cab 100644 --- a/doc/administration/monitoring/performance/performance_bar.md +++ b/doc/administration/monitoring/performance/performance_bar.md @@ -1,6 +1,6 @@ --- stage: Monitor -group: APM +group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- @@ -15,7 +15,7 @@ From left to right, it displays: - **Current Host**: the current host serving the page. - **Database queries**: the time taken (in milliseconds) and the total number - of database queries, displayed in the format `00ms / 00pg`. Click to display + of database queries, displayed in the format `00ms / 00 (00 cached) pg`. Click to display a modal window with more details:  - **Gitaly calls**: the time taken (in milliseconds) and the total number of diff --git a/doc/administration/monitoring/performance/request_profiling.md b/doc/administration/monitoring/performance/request_profiling.md index 5746b95eb44..6e03404fd26 100644 --- a/doc/administration/monitoring/performance/request_profiling.md +++ b/doc/administration/monitoring/performance/request_profiling.md @@ -1,6 +1,6 @@ --- stage: Monitor -group: APM +group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- @@ -25,7 +25,6 @@ To profile a request: curl --header 'X-Profile-Token: <token>' --header 'X-Profile-Mode: <mode>' "https://gitlab.example.com/group/project" ``` - NOTE: **Note:** Profiled requests can take longer than usual. After the request completes, you can view the profiling output from the diff --git a/doc/administration/monitoring/prometheus/gitlab_exporter.md b/doc/administration/monitoring/prometheus/gitlab_exporter.md index 686ed14ba42..971dafb4ba2 100644 --- a/doc/administration/monitoring/prometheus/gitlab_exporter.md +++ b/doc/administration/monitoring/prometheus/gitlab_exporter.md @@ -1,6 +1,6 @@ --- stage: Monitor -group: APM +group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- diff --git a/doc/administration/monitoring/prometheus/gitlab_metrics.md b/doc/administration/monitoring/prometheus/gitlab_metrics.md index ae31a3db023..adb1f719f3c 100644 --- a/doc/administration/monitoring/prometheus/gitlab_metrics.md +++ b/doc/administration/monitoring/prometheus/gitlab_metrics.md @@ -1,6 +1,6 @@ --- stage: Monitor -group: APM +group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- @@ -91,8 +91,8 @@ The following metrics are available: | `gitlab_transaction_rails_queue_duration_total` | Counter | 9.4 | Measures latency between GitLab Workhorse forwarding a request to Rails | `controller`, `action` | | `gitlab_transaction_view_duration_total` | Counter | 9.4 | Duration for views | `controller`, `action`, `view` | | `gitlab_view_rendering_duration_seconds` | Histogram | 10.2 | Duration for views (histogram) | `controller`, `action`, `view` | -| `http_requests_total` | Counter | 9.4 | Rack request count | `method` | -| `http_request_duration_seconds` | Histogram | 9.4 | HTTP response time from rack middleware | `method`, `status` | +| `http_requests_total` | Counter | 9.4 | Rack request count | `method`, `status` | +| `http_request_duration_seconds` | Histogram | 9.4 | HTTP response time from rack middleware | `method` | | `gitlab_transaction_db_count_total` | Counter | 13.1 | Counter for total number of SQL calls | `controller`, `action` | | `gitlab_transaction_db_write_count_total` | Counter | 13.1 | Counter for total number of write SQL calls | `controller`, `action` | | `gitlab_transaction_db_cached_count_total` | Counter | 13.1 | Counter for total number of cached SQL calls | `controller`, `action` | @@ -113,6 +113,8 @@ The following metrics are available: | `action_cable_pool_largest_size` | Gauge | 13.4 | Largest number of worker threads observed so far in ActionCable thread pool | `server_mode` | | `action_cable_pool_pending_tasks` | Gauge | 13.4 | Number of tasks waiting to be executed in ActionCable thread pool | `server_mode` | | `action_cable_pool_tasks_total` | Gauge | 13.4 | Total number of tasks executed in ActionCable thread pool | `server_mode` | +| `gitlab_issuable_fast_count_by_state_total` | Counter | 13.5 | Total number of row count operations on issue/merge request list pages | | +| `gitlab_issuable_fast_count_by_state_failures_total` | Counter | 13.5 | Number of soft-failed row count operations on issue/merge request list pages | | ## Metrics controlled by a feature flag @@ -122,6 +124,8 @@ The following metrics can be controlled by feature flags: |:---------------------------------------------------------------|:-------------------------------------------------------------------| | `gitlab_method_call_duration_seconds` | `prometheus_metrics_method_instrumentation` | | `gitlab_view_rendering_duration_seconds` | `prometheus_metrics_view_instrumentation` | +| `gitlab_issuable_fast_count_by_state_total` | `soft_fail_count_by_state` | +| `gitlab_issuable_fast_count_by_state_failures_total` | `soft_fail_count_by_state` | ## Sidekiq metrics @@ -184,12 +188,12 @@ configuration option in `gitlab.yml`. These metrics are served from the | `geo_package_files_synced` | Gauge | 13.3 | Number of syncable package files synced on secondary | `url` | | `geo_package_files_failed` | Gauge | 13.3 | Number of syncable package files failed to sync on secondary | `url` | | `geo_package_files_registry` | Gauge | 13.3 | Number of package files in the registry | `url` | -| `geo_terraform_states` | Gauge | 13.3 | Number of terraform states on primary | `url` | -| `geo_terraform_states_checksummed` | Gauge | 13.3 | Number of terraform states checksummed on primary | `url` | -| `geo_terraform_states_checksum_failed` | Gauge | 13.3 | Number of terraform states failed to calculate the checksum on primary | `url` | -| `geo_terraform_states_synced` | Gauge | 13.3 | Number of syncable terraform states synced on secondary | `url` | -| `geo_terraform_states_failed` | Gauge | 13.3 | Number of syncable terraform states failed to sync on secondary | `url` | -| `geo_terraform_states_registry` | Gauge | 13.3 | Number of terraform states in the registry | `url` | +| `geo_terraform_state_versions` | Gauge | 13.5 | Number of terraform state versions on primary | `url` | +| `geo_terraform_state_versions_checksummed` | Gauge | 13.5 | Number of terraform state versions checksummed on primary | `url` | +| `geo_terraform_state_versions_checksum_failed` | Gauge | 13.5 | Number of terraform state versions failed to calculate the checksum on primary | `url` | +| `geo_terraform_state_versions_synced` | Gauge | 13.5 | Number of syncable terraform state versions synced on secondary | `url` | +| `geo_terraform_state_versions_failed` | Gauge | 13.5 | Number of syncable terraform state versions failed to sync on secondary | `url` | +| `geo_terraform_state_versions_registry` | Gauge | 13.5 | Number of terraform state versions in the registry | `url` | | `global_search_bulk_cron_queue_size` | Gauge | 12.10 | Number of database records waiting to be synchronized to Elasticsearch | | | `global_search_awaiting_indexing_queue_size` | Gauge | 13.2 | Number of database updates waiting to be synchronized to Elasticsearch while indexing is paused | | | `geo_merge_request_diffs` | Gauge | 13.4 | Number of merge request diffs on primary | `url` | @@ -204,6 +208,9 @@ configuration option in `gitlab.yml`. These metrics are served from the | `geo_snippet_repositories_synced` | Gauge | 13.4 | Number of syncable snippets synced on secondary | `url` | | `geo_snippet_repositories_failed` | Gauge | 13.4 | Number of syncable snippets failed on secondary | `url` | | `geo_snippet_repositories_registry` | Gauge | 13.4 | Number of syncable snippets in the registry | `url` | +| `limited_capacity_worker_running_jobs` | Gauge | 13.5 | Number of running jobs | `worker` | +| `limited_capacity_worker_max_running_jobs` | Gauge | 13.5 | Maximum number of running jobs | `worker` | +| `limited_capacity_worker_remaining_work_count` | Gauge | 13.5 | Number of jobs waiting to be enqueued | `worker` | ## Database load balancing metrics **(PREMIUM ONLY)** diff --git a/doc/administration/monitoring/prometheus/index.md b/doc/administration/monitoring/prometheus/index.md index 7d93e9797be..63231996dcc 100644 --- a/doc/administration/monitoring/prometheus/index.md +++ b/doc/administration/monitoring/prometheus/index.md @@ -1,6 +1,6 @@ --- stage: Monitor -group: APM +group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- @@ -32,7 +32,7 @@ dashboard tool like [Grafana](https://grafana.com). ## Configuring Prometheus NOTE: **Note:** -For installations from source, you'll have to install and configure it yourself. +For installations from source, you must install and configure it yourself. Prometheus and its exporters are on by default, starting with GitLab 9.0. Prometheus will run as the `gitlab-prometheus` user and listen on @@ -60,9 +60,9 @@ it's not recommended to change the port Prometheus listens on, as this might affect or conflict with other services running on the GitLab server. Proceed at your own risk. -In order to access Prometheus from outside the GitLab server you will need to -set a FQDN or IP in `prometheus['listen_address']`. -To change the address/port that Prometheus listens on: +To access Prometheus from outside the GitLab server, set an FQDN or IP in +`prometheus['listen_address']`. To change the address/port that Prometheus +listens on: 1. Edit `/etc/gitlab/gitlab.rb` 1. Add or find and uncomment the following line: @@ -179,7 +179,7 @@ The next step is to tell all the other nodes where the monitoring node is: take effect. NOTE: **Note:** -Once monitoring using Service Discovery is enabled with `consul['monitoring_service_discovery'] = true`, +After monitoring using Service Discovery is enabled with `consul['monitoring_service_discovery'] = true`, ensure that `prometheus['scrape_configs']` is not set in `/etc/gitlab/gitlab.rb`. Setting both `consul['monitoring_service_discovery'] = true` and `prometheus['scrape_configs']` in `/etc/gitlab/gitlab.rb` will result in errors. @@ -312,7 +312,6 @@ To use an external Prometheus server: You can visit `http://localhost:9090` for the dashboard that Prometheus offers by default. -NOTE: **Note:** If SSL has been enabled on your GitLab instance, you may not be able to access Prometheus on the same browser as GitLab if using the same FQDN due to [HSTS](https://en.wikipedia.org/wiki/HTTP_Strict_Transport_Security). We plan to [provide access via GitLab](https://gitlab.com/gitlab-org/multi-user-prometheus), but in the interim there are diff --git a/doc/administration/monitoring/prometheus/node_exporter.md b/doc/administration/monitoring/prometheus/node_exporter.md index 07f6b8b8e1e..dae1f02b196 100644 --- a/doc/administration/monitoring/prometheus/node_exporter.md +++ b/doc/administration/monitoring/prometheus/node_exporter.md @@ -1,6 +1,6 @@ --- stage: Monitor -group: APM +group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- @@ -10,7 +10,7 @@ The [node exporter](https://github.com/prometheus/node_exporter) enables you to various machine resources such as memory, disk and CPU utilization. NOTE: **Note:** -For installations from source you'll have to install and configure it yourself. +For installations from source you must install and configure it yourself. To enable the node exporter: diff --git a/doc/administration/monitoring/prometheus/pgbouncer_exporter.md b/doc/administration/monitoring/prometheus/pgbouncer_exporter.md index 62d0bf684b6..4554bc06401 100644 --- a/doc/administration/monitoring/prometheus/pgbouncer_exporter.md +++ b/doc/administration/monitoring/prometheus/pgbouncer_exporter.md @@ -1,6 +1,6 @@ --- stage: Monitor -group: APM +group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- @@ -12,7 +12,7 @@ The [PgBouncer exporter](https://github.com/prometheus-community/pgbouncer_expor you to measure various [PgBouncer](https://www.pgbouncer.org/) metrics. NOTE: **Note:** -For installations from source you'll have to install and configure it yourself. +For installations from source you must install and configure it yourself. To enable the PgBouncer exporter: diff --git a/doc/administration/monitoring/prometheus/postgres_exporter.md b/doc/administration/monitoring/prometheus/postgres_exporter.md index e3fff45fce3..9eb9ba3c59f 100644 --- a/doc/administration/monitoring/prometheus/postgres_exporter.md +++ b/doc/administration/monitoring/prometheus/postgres_exporter.md @@ -1,6 +1,6 @@ --- stage: Monitor -group: APM +group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- @@ -9,7 +9,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w The [PostgreSQL Server Exporter](https://github.com/wrouesnel/postgres_exporter) allows you to export various PostgreSQL metrics. NOTE: **Note:** -For installations from source you will have to install and configure it yourself. +For installations from source you must install and configure it yourself. To enable the PostgreSQL Server Exporter: @@ -20,7 +20,6 @@ To enable the PostgreSQL Server Exporter: postgres_exporter['enable'] = true ``` - NOTE: **Note:** If PostgreSQL Server Exporter is configured on a separate node, make sure that the local address is [listed in `trust_auth_cidr_addresses`](../../postgresql/replication_and_failover.md#network-information) or the exporter will not be able to connect to the database. diff --git a/doc/administration/monitoring/prometheus/redis_exporter.md b/doc/administration/monitoring/prometheus/redis_exporter.md index b7c66959349..16a758c9804 100644 --- a/doc/administration/monitoring/prometheus/redis_exporter.md +++ b/doc/administration/monitoring/prometheus/redis_exporter.md @@ -1,6 +1,6 @@ --- stage: Monitor -group: APM +group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- @@ -11,7 +11,7 @@ various [Redis](https://redis.io) metrics. For more information on what is expor [read the upstream documentation](https://github.com/oliver006/redis_exporter/blob/master/README.md#whats-exported). NOTE: **Note:** -For installations from source you'll have to install and configure it yourself. +For installations from source you must install and configure it yourself. To enable the Redis exporter: diff --git a/doc/administration/monitoring/prometheus/registry_exporter.md b/doc/administration/monitoring/prometheus/registry_exporter.md index 3d28b26b685..3bf4db8a665 100644 --- a/doc/administration/monitoring/prometheus/registry_exporter.md +++ b/doc/administration/monitoring/prometheus/registry_exporter.md @@ -1,6 +1,6 @@ --- stage: Monitor -group: APM +group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- diff --git a/doc/administration/nfs.md b/doc/administration/nfs.md index b54f05ad536..fabbfb2552e 100644 --- a/doc/administration/nfs.md +++ b/doc/administration/nfs.md @@ -1,4 +1,7 @@ --- +stage: none +group: unassigned +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference --- @@ -296,6 +299,25 @@ Having multiple NFS mounts will require manually making sure the data directorie are empty before attempting a restore. Read more about the [restore prerequisites](../raketasks/backup_restore.md). +## Testing NFS + +Once you've set up the NFS server and client, you can verify NFS is configured correctly +by testing the following commands: + +```shell +sudo mkdir /gitlab-nfs/test-dir +sudo chown git /gitlab-nfs/test-dir +sudo chgrp gitlab-www /gitlab-nfs/test-dir +sudo chgrp root /gitlab-nfs/test-dir +sudo chmod 2755 /gitlab-nfs/test-dir +sudo -u git mkdir /gitlab-nfs/test-dir/test2 +sudo -u git chmod 2755 /gitlab-nfs/test-dir/test2 +sudo ls -lah /gitlab-nfs/test-dir/test2 +sudo -u git rm -r /gitlab-nfs/test-dir +``` + +Any `Operation not permitted` errors means you should investigate your NFS server export options. + ## NFS in a Firewalled Environment If the traffic between your NFS server and NFS client(s) is subject to port filtering @@ -317,6 +339,28 @@ sudo ufw allow from <client_ip_address> to any port nfs ## Known issues +### Upgrade to Gitaly Cluster or disable caching if experiencing data loss + +CAUTION: **Caution:** +From GitLab 13.0, using NFS for Git repositories is deprecated. In GitLab 14.0, +support for NFS for Git repositories is scheduled to be removed. Upgrade to +[Gitaly Cluster](gitaly/praefect.md) as soon as possible. + +Customers and users have reported data loss on high-traffic repositories when using NFS for Git repositories. +For example, we have seen [inconsistent updates after a push](https://gitlab.com/gitlab-org/gitaly/-/issues/2589). The problem may be partially mitigated by adjusting caching using the following NFS client mount options: + +| Setting | Description | +| ------- | ----------- | +| `lookupcache=positive` | Tells the NFS client to honor `positive` cache results but invalidates any `negative` cache results. Negative cache results cause problems with Git. Specifically, a `git push` can fail to register uniformly across all NFS clients. The negative cache causes the clients to 'remember' that the files did not exist previously. +| `actimeo=0` | Sets the time to zero that the NFS client caches files and directories before requesting fresh information from a server. | +| `noac` | Tells the NFS client not to cache file attributes and forces application writes to become synchronous so that local changes to a file become visible on the server immediately. | + +CAUTION: **Caution:** +The `actimeo=0` and `noac` options both result in a significant reduction in performance, possibly leading to timeouts. +You may be able to avoid timeouts and data loss using `actimeo=0` and `lookupcache=positive` _without_ `noac`, however +we expect the performance reduction will still be significant. As noted above, we strongly recommend upgrading to +[Gitaly Cluster](gitaly/praefect.md) as soon as possible. + ### Avoid using AWS's Elastic File System (EFS) GitLab strongly recommends against using AWS Elastic File System (EFS). diff --git a/doc/administration/object_storage.md b/doc/administration/object_storage.md index 39365ffe404..8b788e6d91d 100644 --- a/doc/administration/object_storage.md +++ b/doc/administration/object_storage.md @@ -1,4 +1,7 @@ --- +stage: none +group: unassigned +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference --- @@ -17,7 +20,7 @@ GitLab has been tested on a number of object storage providers: - [Google Cloud Storage](https://cloud.google.com/storage) - [Digital Ocean Spaces](https://www.digitalocean.com/products/spaces/) - [Oracle Cloud Infrastructure](https://docs.cloud.oracle.com/en-us/iaas/Content/Object/Tasks/s3compatibleapi.htm) -- [Openstack Swift](https://docs.openstack.org/swift/latest/s3_compat.html) +- [OpenStack Swift](https://docs.openstack.org/swift/latest/s3_compat.html) - [Azure Blob storage](https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blobs-introduction) - On-premises hardware and appliances from various storage vendors. - MinIO. We have [a guide to deploying this](https://docs.gitlab.com/charts/advanced/external-object-storage/minio.html) within our Helm Chart documentation. @@ -369,7 +372,7 @@ Here are the valid connection parameters for Rackspace Cloud, provided by | `rackspace_username` | The username of the Rackspace account with access to the container | `joe.smith` | | `rackspace_api_key` | The API key of the Rackspace account with access to the container | `ABC123DEF456ABC123DEF456ABC123DE` | | `rackspace_region` | The Rackspace storage region to use, a three letter code from the [list of service access endpoints](https://developer.rackspace.com/docs/cloud-files/v1/general-api-info/service-access/) | `iad` | -| `rackspace_temp_url_key` | The private key you have set in the Rackspace API for temporary URLs. Read more [here](https://developer.rackspace.com/docs/cloud-files/v1/use-cases/public-access-to-your-cloud-files-account/#tempurl) | `ABC123DEF456ABC123DEF456ABC123DE` | +| `rackspace_temp_url_key` | The private key you have set in the Rackspace API for [temporary URLs](https://developer.rackspace.com/docs/cloud-files/v1/use-cases/public-access-to-your-cloud-files-account/#tempurl). | `ABC123DEF456ABC123DEF456ABC123DE` | NOTE: **Note:** Regardless of whether the container has public access enabled or disabled, Fog will @@ -383,7 +386,7 @@ and comparing the output of the returned headers. The following YAML shows how the `object_store` section defines object-specific configuration block and how the `enabled` and -`proxy_download` flags can be overriden. The `bucket` is the only +`proxy_download` flags can be overridden. The `bucket` is the only required parameter within each type: ```yaml @@ -503,13 +506,13 @@ supported by consolidated configuration form, refer to the following guides: |Object storage type|Supported by consolidated configuration?| |-------------------|----------------------------------------| | [Backups](../raketasks/backup_restore.md#uploading-backups-to-a-remote-cloud-storage)|No| -| [Job artifacts](job_artifacts.md#using-object-storage) and [incremental logging](job_logs.md#new-incremental-logging-architecture) | Yes | +| [Job artifacts](job_artifacts.md#using-object-storage) including archived job logs | Yes | | [LFS objects](lfs/index.md#storing-lfs-objects-in-remote-object-storage) | Yes | | [Uploads](uploads.md#using-object-storage) | Yes | | [Container Registry](packages/container_registry.md#use-object-storage) (optional feature) | No | | [Merge request diffs](merge_request_diffs.md#using-object-storage) | Yes | | [Mattermost](https://docs.mattermost.com/administration/config-settings.html#file-storage)| No | -| [Packages](packages/index.md#using-object-storage) (optional feature) **(PREMIUM ONLY)** | Yes | +| [Packages](packages/index.md#using-object-storage) (optional feature) | Yes | | [Dependency Proxy](packages/dependency_proxy.md#using-object-storage) (optional feature) **(PREMIUM ONLY)** | Yes | | [Pseudonymizer](pseudonymizer.md#configuration) (optional feature) **(ULTIMATE ONLY)** | No | | [Autoscale runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional for improved performance) | No | @@ -520,12 +523,13 @@ supported by consolidated configuration form, refer to the following guides: If you're working to [scale out](reference_architectures/index.md) your GitLab implementation, or add fault tolerance and redundancy, you may be looking at removing dependencies on block or network filesystems. -See the following guides and +See the following additional guides and [note that Pages requires disk storage](#gitlab-pages-requires-nfs): 1. Make sure the [`git` user home directory](https://docs.gitlab.com/omnibus/settings/configuration.html#moving-the-home-directory-for-a-user) is on local disk. 1. Configure [database lookup of SSH keys](operations/fast_ssh_key_lookup.md) to eliminate the need for a shared `authorized_keys` file. +1. [Prevent local disk usage for job logs](job_logs.md#prevent-local-disk-usage). ## Warnings, limitations, and known issues @@ -569,7 +573,8 @@ The dependency on disk storage also prevents Pages being deployed using the ### Incremental logging is required for CI to use object storage If you configure GitLab to use object storage for CI logs and artifacts, -[you must also enable incremental logging](job_artifacts.md#using-object-storage). +you can avoid [local disk usage for job logs](job_logs.md#data-flow) by enabling +[beta incremental logging](job_logs.md#new-incremental-logging-architecture). ### Proxy Download @@ -685,7 +690,7 @@ in the `storage_options` configuration section: | Setting | Description | |-------------------------------------|-------------| -| `server_side_encryption` | Encryption mode (AES256 or aws:kms) | +| `server_side_encryption` | Encryption mode (`AES256` or `aws:kms`) | | `server_side_encryption_kms_key_id` | Amazon Resource Name. Only needed when `aws:kms` is used in `server_side_encryption`. See the [Amazon documentation on using KMS encryption](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html) | As with the case for default encryption, these options only work when diff --git a/doc/administration/operations/cleaning_up_redis_sessions.md b/doc/administration/operations/cleaning_up_redis_sessions.md index d2aec2f7c47..a94f88439f2 100644 --- a/doc/administration/operations/cleaning_up_redis_sessions.md +++ b/doc/administration/operations/cleaning_up_redis_sessions.md @@ -1,3 +1,9 @@ +--- +stage: none +group: unassigned +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Cleaning up stale Redis sessions Since version 6.2, GitLab stores web user sessions as key-value pairs in Redis. diff --git a/doc/administration/operations/extra_sidekiq_processes.md b/doc/administration/operations/extra_sidekiq_processes.md index e589ecc4216..76dc9bf5510 100644 --- a/doc/administration/operations/extra_sidekiq_processes.md +++ b/doc/administration/operations/extra_sidekiq_processes.md @@ -1,3 +1,9 @@ +--- +stage: none +group: unassigned +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Run multiple Sidekiq processes **(CORE ONLY)** GitLab allows you to start multiple Sidekiq processes. diff --git a/doc/administration/operations/fast_ssh_key_lookup.md b/doc/administration/operations/fast_ssh_key_lookup.md index 6cd393be330..88ef69b29f2 100644 --- a/doc/administration/operations/fast_ssh_key_lookup.md +++ b/doc/administration/operations/fast_ssh_key_lookup.md @@ -1,3 +1,9 @@ +--- +stage: none +group: unassigned +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Fast lookup of authorized SSH keys in the database > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/1631) in [GitLab Starter](https://about.gitlab.com/pricing/) 9.3. @@ -114,7 +120,6 @@ This is a brief overview. Please refer to the above instructions for more contex 1. Enable writes to the `authorized_keys` file in Application Settings 1. Remove the `AuthorizedKeysCommand` lines from `/etc/ssh/sshd_config` or from `/assets/sshd_config` if you are using Omnibus Docker. 1. Reload `sshd`: `sudo service sshd reload` -1. Remove the `/opt/gitlab-shell/authorized_keys` file ## Compiling a custom version of OpenSSH for CentOS 6 diff --git a/doc/administration/operations/filesystem_benchmarking.md b/doc/administration/operations/filesystem_benchmarking.md index 64afd1b44f3..c55f253b772 100644 --- a/doc/administration/operations/filesystem_benchmarking.md +++ b/doc/administration/operations/filesystem_benchmarking.md @@ -1,3 +1,9 @@ +--- +stage: none +group: unassigned +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Filesystem Performance Benchmarking Filesystem performance has a big impact on overall GitLab performance, diff --git a/doc/administration/operations/index.md b/doc/administration/operations/index.md index 45b8e5ad448..aaeb8c723d0 100644 --- a/doc/administration/operations/index.md +++ b/doc/administration/operations/index.md @@ -1,3 +1,9 @@ +--- +stage: none +group: unassigned +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Performing Operations in GitLab Keep your GitLab instance up and running smoothly. diff --git a/doc/administration/operations/moving_repositories.md b/doc/administration/operations/moving_repositories.md index 4763c012538..94eea1e25b8 100644 --- a/doc/administration/operations/moving_repositories.md +++ b/doc/administration/operations/moving_repositories.md @@ -1,3 +1,9 @@ +--- +stage: none +group: unassigned +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Moving repositories managed by GitLab Sometimes you need to move all repositories managed by GitLab to diff --git a/doc/administration/operations/puma.md b/doc/administration/operations/puma.md index f5b09d7a978..2d53a790428 100644 --- a/doc/administration/operations/puma.md +++ b/doc/administration/operations/puma.md @@ -1,3 +1,9 @@ +--- +stage: none +group: unassigned +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Switching to Puma As of GitLab 12.9, [Puma](https://github.com/puma/puma) has replaced [Unicorn](https://yhbt.net/unicorn/) diff --git a/doc/administration/operations/rails_console.md b/doc/administration/operations/rails_console.md new file mode 100644 index 00000000000..9db9a885baf --- /dev/null +++ b/doc/administration/operations/rails_console.md @@ -0,0 +1,144 @@ +# The Rails Console + +The [Rails console](https://guides.rubyonrails.org/command_line.html#rails-console). +provides a way to interact with your GitLab instance from the command line. + +CAUTION: **Caution:** +The Rails console interacts directly with GitLab. In many cases, +there are no handrails to prevent you from permanently modifying, corrupting +or destroying production data. If you would like to explore the Rails console +with no consequences, you are strongly advised to do so in a test environment. + +The Rails console is for GitLab system administrators who are troubleshooting +a problem or need to retrieve some data that can only be done through direct +access of the GitLab application. + +## Starting a Rails console session + +**For Omnibus installations** + +```shell +sudo gitlab-rails console +``` + +**For installations from source** + +```shell +sudo -u git -H bundle exec rails console -e production +``` + +**For Kubernetes deployments** + +The console is in the task-runner pod. Refer to our [Kubernetes cheat sheet](../troubleshooting/kubernetes_cheat_sheet.md#gitlab-specific-kubernetes-information) for details. + +To exit the console, type: `quit`. + +## Output Rails console session history + +Enter the following command on the rails console to display +your command history. + +```ruby +puts Readline::HISTORY.to_a +``` + +You can then copy it to your clipboard and save for future reference. + +## Using the Rails Runner + +If you need to run some Ruby code in the context of your GitLab production +environment, you can do so using the [Rails Runner](https://guides.rubyonrails.org/command_line.html#rails-runner). +When executing a script file, the script must be accessible by the `git` user. + +When the command or script completes, the Rails Runner process finishes. +It is useful for running within other scripts or cron jobs for example. + +**For Omnibus installations** + +```shell +sudo gitlab-rails runner "RAILS_COMMAND" + +# Example with a two-line Ruby script +sudo gitlab-rails runner "user = User.first; puts user.username" + +# Example with a ruby script file (make sure to use the full path) +sudo gitlab-rails runner /path/to/script.rb +``` + +**For installations from source** + +```shell +sudo -u git -H bundle exec rails runner -e production "RAILS_COMMAND" + +# Example with a two-line Ruby script +sudo -u git -H bundle exec rails runner -e production "user = User.first; puts user.username" + +# Example with a ruby script file (make sure to use the full path) +sudo -u git -H bundle exec rails runner -e production /path/to/script.rb +``` + +Rails Runner does not produce the same output as the console. + +If you set a variable on the console, the console will generate useful debug output +such as the variable contents or properties of referenced entity: + +```ruby +irb(main):001:0> user = User.first +=> #<User id:1 @root> +``` + +Rails Runner does not do this: you have to be explicit about generating +output: + +```shell +$ sudo gitlab-rails runner "user = User.first" +$ sudo gitlab-rails runner "user = User.first; puts user.username ; puts user.id" +root +1 +``` + +Some basic knowledge of Ruby will be very useful. Try [this +30-minute tutorial](https://try.ruby-lang.org/) for a quick introduction. +Rails experience is helpful but not essential. + +### Troubleshooting Rails Runner + +The `gitlab-rails` command executes Rails Runner using a non-root account and group, by default: `git:git`. + +If the non-root account cannot find the Ruby script filename passed to `gitlab-rails runner` +you may get a syntax error, not an error that the file couldn't be accessed. + +A common reason for this is that the script has been put in the root account's home directory. + +`runner` tries to parse the path and file parameter as Ruby code. + +For example: + +```plaintext +[root ~]# echo 'puts "hello world"' > ./helloworld.rb +[root ~]# sudo gitlab-rails runner ./helloworld.rb +Please specify a valid ruby command or the path of a script to run. +Run 'rails runner -h' for help. + +/opt/gitlab/..../runner_command.rb:45: syntax error, unexpected '.' +./helloworld.rb +^ +[root ~]# sudo gitlab-rails runner /root/helloworld.rb +Please specify a valid ruby command or the path of a script to run. +Run 'rails runner -h' for help. + +/opt/gitlab/..../runner_command.rb:45: unknown regexp options - hllwrld +[root ~]# mv ~/helloworld.rb /tmp +[root ~]# sudo gitlab-rails runner /tmp/helloworld.rb +hello world +``` + +A meaningful error should be generated if the directory can be accessed, but the file cannot: + +```plaintext +[root ~]# chmod 400 /tmp/helloworld.rb +[root ~]# sudo gitlab-rails runner /tmp/helloworld.rb +Traceback (most recent call last): + [traceback removed] +/opt/gitlab/..../runner_command.rb:42:in `load': cannot load such file -- /tmp/helloworld.rb (LoadError) +``` diff --git a/doc/administration/operations/sidekiq_memory_killer.md b/doc/administration/operations/sidekiq_memory_killer.md index d1ff98a0079..d99468411e3 100644 --- a/doc/administration/operations/sidekiq_memory_killer.md +++ b/doc/administration/operations/sidekiq_memory_killer.md @@ -1,3 +1,9 @@ +--- +stage: none +group: unassigned +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Sidekiq MemoryKiller The GitLab Rails application code suffers from memory leaks. For web requests @@ -8,7 +14,7 @@ MemoryKiller applies the same approach to the Sidekiq processes used by GitLab to process background jobs. Unlike puma-worker-killer, which is enabled by default for all GitLab -installations since GitLab 13.0, the Sidekiq MemoryKiller is enabled by default +installations of GitLab 13.0 and later, the Sidekiq MemoryKiller is enabled by default _only_ for Omnibus packages. The reason for this is that the MemoryKiller relies on runit to restart Sidekiq after a memory-induced shutdown and GitLab installations from source do not all use runit or an equivalent. diff --git a/doc/administration/operations/ssh_certificates.md b/doc/administration/operations/ssh_certificates.md index c81eb15955d..7cbd8c74f90 100644 --- a/doc/administration/operations/ssh_certificates.md +++ b/doc/administration/operations/ssh_certificates.md @@ -1,3 +1,9 @@ +--- +stage: none +group: unassigned +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # User lookup via OpenSSH's AuthorizedPrincipalsCommand > [Available in](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/19911) GitLab diff --git a/doc/administration/operations/unicorn.md b/doc/administration/operations/unicorn.md index a1593c3a6c3..80dafde14aa 100644 --- a/doc/administration/operations/unicorn.md +++ b/doc/administration/operations/unicorn.md @@ -1,3 +1,9 @@ +--- +stage: none +group: unassigned +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Understanding Unicorn and unicorn-worker-killer NOTE: **Note:** diff --git a/doc/administration/packages/container_registry.md b/doc/administration/packages/container_registry.md index 74af5c8149b..56b7f01e1ad 100644 --- a/doc/administration/packages/container_registry.md +++ b/doc/administration/packages/container_registry.md @@ -10,15 +10,13 @@ info: To determine the technical writer assigned to the Stage/Group associated w > - Container Registry manifest `v1` support was added in GitLab 8.9 to support > Docker versions earlier than 1.10. -NOTE: **Note:** -This document is the administrator's guide. To learn how to use GitLab Container -Registry, see the [user documentation](../../user/packages/container_registry/index.md). +With the GitLab Container Registry, every project can have its +own space to store Docker images. -With the Container Registry integrated into GitLab, every project can have its -own space to store its Docker images. +Read more about the Docker Registry in [the Docker documentation](https://docs.docker.com/registry/introduction/). -You can read more about the Docker Registry at -<https://docs.docker.com/registry/introduction/>. +This document is the administrator's guide. To learn how to use the GitLab Container +Registry, see the [user documentation](../../user/packages/container_registry/index.md). ## Enable the Container Registry @@ -37,9 +35,8 @@ Otherwise, the Container Registry is not enabled. To enable it: - You can configure it for your [GitLab domain](#configure-container-registry-under-an-existing-gitlab-domain), or - You can configure it for [a different domain](#configure-container-registry-under-its-own-domain). -NOTE: **Note:** The Container Registry works under HTTPS by default. You can use HTTP -but it's not recommended and is out of the scope of this document. +but it's not recommended and is beyond the scope of this document. Read the [insecure Registry documentation](https://docs.docker.com/registry/insecure/) if you want to implement this. @@ -47,12 +44,12 @@ if you want to implement this. If you have installed GitLab from source: -1. You will have to [install Registry](https://docs.docker.com/registry/deploying/) by yourself. -1. After the installation is complete, you will have to configure the Registry's - settings in `gitlab.yml` in order to enable it. -1. Use the sample NGINX configuration file that is found under +1. You must [install Registry](https://docs.docker.com/registry/deploying/) by yourself. +1. After the installation is complete, to enable it, you must configure the Registry's + settings in `gitlab.yml`. +1. Use the sample NGINX configuration file from under [`lib/support/nginx/registry-ssl`](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/support/nginx/registry-ssl) and edit it to match the - `host`, `port` and TLS certs paths. + `host`, `port`, and TLS certificate paths. The contents of `gitlab.yml` are: @@ -67,21 +64,20 @@ registry: issuer: gitlab-issuer ``` -where: +Where: | Parameter | Description | | --------- | ----------- | | `enabled` | `true` or `false`. Enables the Registry in GitLab. By default this is `false`. | -| `host` | The host URL under which the Registry will run and the users will be able to use. | -| `port` | The port under which the external Registry domain will listen on. | -| `api_url` | The internal API URL under which the Registry is exposed to. It defaults to `http://localhost:5000`. | +| `host` | The host URL under which the Registry runs and users can use. | +| `port` | The port the external Registry domain listens on. | +| `api_url` | The internal API URL under which the Registry is exposed. It defaults to `http://localhost:5000`. | | `key` | The private key location that is a pair of Registry's `rootcertbundle`. Read the [token auth configuration documentation](https://docs.docker.com/registry/configuration/#token). | | `path` | This should be the same directory like specified in Registry's `rootdirectory`. Read the [storage configuration documentation](https://docs.docker.com/registry/configuration/#storage). This path needs to be readable by the GitLab user, the web-server user and the Registry user. Read more in [#configure-storage-for-the-container-registry](#configure-storage-for-the-container-registry). | | `issuer` | This should be the same value as configured in Registry's `issuer`. Read the [token auth configuration documentation](https://docs.docker.com/registry/configuration/#token). | -NOTE: **Note:** A Registry init file is not shipped with GitLab if you install it from source. -Hence, [restarting GitLab](../restart_gitlab.md#installations-from-source) will not restart the Registry should +Hence, [restarting GitLab](../restart_gitlab.md#installations-from-source) does not restart the Registry should you modify its settings. Read the upstream documentation on how to achieve that. At the **absolute** minimum, make sure your [Registry configuration](https://docs.docker.com/registry/configuration/#auth) @@ -98,37 +94,34 @@ auth: ``` CAUTION: **Caution:** -If `auth` is not set up, users will be able to pull Docker images without authentication. +If `auth` is not set up, users can pull Docker images without authentication. ## Container Registry domain configuration There are two ways you can configure the Registry's external domain. Either: -- [Use the existing GitLab domain](#configure-container-registry-under-an-existing-gitlab-domain) where in that case - the Registry will have to listen on a port and reuse GitLab's TLS certificate, +- [Use the existing GitLab domain](#configure-container-registry-under-an-existing-gitlab-domain). + The Registry listens on a port and reuses GitLab's TLS certificate. - [Use a completely separate domain](#configure-container-registry-under-its-own-domain) with a new TLS certificate for that domain. -Since the container Registry requires a TLS certificate, in the end it all boils -down to how easy or pricey it is to get a new one. +Because the Container Registry requires a TLS certificate, cost may be a factor. -Please take this into consideration before configuring the Container Registry +Take this into consideration before configuring the Container Registry for the first time. ### Configure Container Registry under an existing GitLab domain If the Registry is configured to use the existing GitLab domain, you can -expose the Registry on a port so that you can reuse the existing GitLab TLS +expose the Registry on a port. This way you can reuse the existing GitLab TLS certificate. -Assuming that the GitLab domain is `https://gitlab.example.com` and the port the -Registry is exposed to the outside world is `5050`, here is what you need to set +If the GitLab domain is `https://gitlab.example.com` and the port to the outside world is `5050`, here is what you need to set in `gitlab.rb` or `gitlab.yml` if you are using Omnibus GitLab or installed GitLab from source respectively. -NOTE: **Note:** -Be careful to choose a port different than the one that Registry listens to (`5000` by default), -otherwise you will run into conflicts. +Ensure you choose a port different than the one that Registry listens to (`5000` by default), +otherwise conflicts occur. **Omnibus GitLab installations** @@ -139,7 +132,7 @@ otherwise you will run into conflicts. registry_external_url 'https://gitlab.example.com:5050' ``` - Note how the `registry_external_url` is listening on HTTPS under the + The `registry_external_url` is listening on HTTPS under the existing GitLab URL, but on a different port. If your TLS certificate is not in `/etc/gitlab/ssl/gitlab.example.com.crt` @@ -160,7 +153,6 @@ otherwise you will run into conflicts. openssl s_client -showcerts -servername gitlab.example.com -connect gitlab.example.com:5050 > cacert.pem ``` -NOTE: **Note:** If your certificate provider provides the CA Bundle certificates, append them to the TLS certificate file. **Installations from source** @@ -187,12 +179,11 @@ docker login gitlab.example.com:5050 ### Configure Container Registry under its own domain -If the Registry is configured to use its own domain, you will need a TLS -certificate for that specific domain (e.g., `registry.example.com`) or maybe +When the Registry is configured to use its own domain, you need a TLS +certificate for that specific domain (for example, `registry.example.com`). You might need a wildcard certificate if hosted under a subdomain of your existing GitLab -domain (e.g., `registry.gitlab.example.com`). +domain, for example, `registry.gitlab.example.com`. -NOTE: **Note:** As well as manually generated SSL certificates (explained here), certificates automatically generated by Let's Encrypt are also [supported in Omnibus installs](https://docs.gitlab.com/omnibus/settings/ssl.html#host-services). @@ -210,19 +201,19 @@ Let's assume that you want the container Registry to be accessible at chmod 600 /etc/gitlab/ssl/registry.gitlab.example.com.* ``` -1. Once the TLS certificate is in place, edit `/etc/gitlab/gitlab.rb` with: +1. After the TLS certificate is in place, edit `/etc/gitlab/gitlab.rb` with: ```ruby registry_external_url 'https://registry.gitlab.example.com' ``` - Note how the `registry_external_url` is listening on HTTPS. + The `registry_external_url` is listening on HTTPS. 1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. -If you have a [wildcard certificate](https://en.wikipedia.org/wiki/Wildcard_certificate), you need to specify the path to the -certificate in addition to the URL, in this case `/etc/gitlab/gitlab.rb` will -look like: +If you have a [wildcard certificate](https://en.wikipedia.org/wiki/Wildcard_certificate), you must specify the path to the +certificate in addition to the URL, in this case `/etc/gitlab/gitlab.rb` +looks like: ```ruby registry_nginx['ssl_certificate'] = "/etc/gitlab/ssl/certificate.pem" @@ -252,9 +243,8 @@ docker login registry.gitlab.example.com ## Disable Container Registry site-wide -NOTE: **Note:** -Disabling the Registry in the Rails GitLab application as set by the following -steps, will not remove any existing Docker images. This is handled by the +When you disable the Registry by following these steps, you do not +remove any existing Docker images. This is handled by the Registry application itself. **Omnibus GitLab** @@ -281,7 +271,7 @@ Registry application itself. ## Disable Container Registry for new projects site-wide -If the Container Registry is enabled, then it will be available on all new +If the Container Registry is enabled, then it should be available on all new projects. To disable this function and let the owners of a project to enable the Container Registry by themselves, follow the steps below. @@ -317,7 +307,7 @@ the Container Registry by themselves, follow the steps below. You can configure the Container Registry to use various storage backends by configuring a storage driver. By default the GitLab Container Registry -is configured to use the [filesystem driver](#use-filesystem) +is configured to use the [file system driver](#use-file-system) configuration. The different supported drivers are: @@ -331,15 +321,14 @@ The different supported drivers are: | swift | OpenStack Swift Object Storage | | oss | Aliyun OSS | -NOTE: **Note:** -Although most S3 compatible services (like [MinIO](https://min.io/)) should work with the registry, we only guarantee support for AWS S3. Because we cannot assert the correctness of third-party S3 implementations, we can debug issues, but we cannot patch the registry unless an issue is reproducible against an AWS S3 bucket. +Although most S3 compatible services (like [MinIO](https://min.io/)) should work with the Container Registry, we only guarantee support for AWS S3. Because we cannot assert the correctness of third-party S3 implementations, we can debug issues, but we cannot patch the registry unless an issue is reproducible against an AWS S3 bucket. Read more about the individual driver's configuration options in the [Docker Registry docs](https://docs.docker.com/registry/configuration/#storage). -### Use filesystem +### Use file system -If you want to store your images on the filesystem, you can change the storage +If you want to store your images on the file system, you can change the storage path for the Container Registry, follow the steps below. This path is accessible to: @@ -347,8 +336,7 @@ This path is accessible to: - The user running the Container Registry daemon. - The user running GitLab. -CAUTION: **Warning:** -You should confirm that all GitLab, Registry and web server users +All GitLab, Registry, and web server users must have access to this directory. **Omnibus GitLab installations** @@ -387,13 +375,10 @@ driver for the Container Registry. [Read more about using object storage with GitLab](../object_storage.md). CAUTION: **Warning:** -GitLab will not backup Docker images that are not stored on the -filesystem. Remember to enable backups with your object storage provider if +GitLab does not back up Docker images that are not stored on the +file system. Enable backups with your object storage provider if desired. -NOTE: **Note:** -`regionendpoint` is only required when configuring an S3 compatible service such as MinIO. It takes a URL such as `http://127.0.0.1:9000`. - **Omnibus GitLab installations** To configure the `s3` storage driver in Omnibus: @@ -412,14 +397,14 @@ To configure the `s3` storage driver in Omnibus: } ``` -1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + - `regionendpoint` is only required when configuring an S3 compatible service such as MinIO. It takes a URL such as `http://127.0.0.1:9000`. + - `your-s3-bucket` should be the name of a bucket that exists, and can't include subdirectories. -NOTE: **Note:** -`your-s3-bucket` should only be the name of a bucket that exists, and can't include subdirectories. +1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. **Installations from source** -Configuring the storage driver is done in your registry configuration YML file created +Configuring the storage driver is done in the registry configuration YML file created when you [deployed your Docker registry](https://docs.docker.com/registry/deploying/). `s3` storage driver example: @@ -438,8 +423,7 @@ storage: enabled: true ``` -NOTE: **Note:** -`your-s3-bucket` should only be the name of a bucket that exists, and can't include subdirectories. +`your-s3-bucket` should be the name of a bucket that exists, and can't include subdirectories. #### Migrate to object storage without downtime @@ -451,7 +435,7 @@ you can pull from the Container Registry, but you cannot push. 1. Optional: To reduce the amount of data to be migrated, run the [garbage collection tool without downtime](#performing-garbage-collection-without-downtime). 1. This example uses the `aws` CLI. If you haven't configured the CLI before, you have to configure your credentials by running `sudo aws configure`. - Because a non-admin user likely can't access the Container Registry folder, + Because a non-administrator user likely can't access the Container Registry folder, ensure you use `sudo`. To check your credential configuration, run [`ls`](https://awscli.amazonaws.com/v2/documentation/api/latest/reference/s3/ls.html) to list all buckets. @@ -483,14 +467,14 @@ you can pull from the Container Registry, but you cannot push. sudo aws --endpoint-url https://your-object-storage-backend.com s3 sync registry s3://mybucket --delete --dryrun ``` - After verifying the command is going to perform as expected, remove the + After verifying the command performs as expected, remove the [`--dryrun`](https://docs.aws.amazon.com/cli/latest/reference/s3/sync.html) flag and run the command. DANGER: **Danger:** The [`--delete`](https://docs.aws.amazon.com/cli/latest/reference/s3/sync.html) - flag will delete files that exist in the destination but not in the source. - Make sure not to swap the source and destination, or you will delete all data in the Registry. + flag deletes files that exist in the destination but not in the source. + If you swap the source and destination, all data in the Registry is deleted. 1. Verify all Container Registry files have been uploaded to object storage by looking at the file count returned by these two commands: @@ -560,15 +544,11 @@ However, this behavior is undesirable for registries used by internal hosts that ### Storage limitations Currently, there is no storage limitation, which means a user can upload an -infinite amount of Docker images with arbitrary sizes. This setting will be +infinite amount of Docker images with arbitrary sizes. This setting should be configurable in future releases. ## Change the registry's internal port -NOTE: **Note:** -This is not to be confused with the port that GitLab itself uses to expose -the Registry to the world. - The Registry server listens on localhost at port `5000` by default, which is the address for which the Registry server should accept connections. In the examples below we set the Registry's port to `5001`. @@ -589,7 +569,7 @@ In the examples below we set the Registry's port to `5001`. [`http:addr`](https://docs.docker.com/registry/configuration/#http) value: ```yaml - http + http: addr: localhost:5001 ``` @@ -603,9 +583,8 @@ on how to achieve that. ## Use an external container registry with GitLab as an auth endpoint -NOTE: **Note:** -In using an external container registry, some features associated with the -container registry may be unavailable or have [inherent risks](./../../user/packages/container_registry/index.md#use-with-external-container-registries) +If you use an external container registry, some features associated with the +container registry may be unavailable or have [inherent risks](./../../user/packages/container_registry/index.md#use-with-external-container-registries). **Omnibus GitLab** @@ -619,13 +598,12 @@ You can use GitLab as an auth endpoint with an external container registry. gitlab_rails['registry_issuer'] = "omnibus-gitlab-issuer" ``` - NOTE: **Note:** `gitlab_rails['registry_enabled'] = true` is needed to enable GitLab's Container Registry features and authentication endpoint. GitLab's bundled - Container Registry service will not be started even with this enabled. + Container Registry service does not start, even with this enabled. 1. A certificate-key pair is required for GitLab and the external container - registry to communicate securely. You will need to create a certificate-key + registry to communicate securely. You need to create a certificate-key pair, configuring the external container registry with the public certificate and configuring GitLab with the private key. To do that, add the following to `/etc/gitlab/gitlab.rb`: @@ -641,11 +619,10 @@ You can use GitLab as an auth endpoint with an external container registry. gitlab_rails['registry_key_path'] = "/custom/path/to/registry-key.key" ``` - NOTE: **Note:** - The file specified at `registry_key_path` gets populated with the - content specified by `internal_key`, each time reconfigure is executed. If - no file is specified, Omnibus GitLab will default it to - `/var/opt/gitlab/gitlab-rails/etc/gitlab-registry.key` and will populate + Each time reconfigure is executed, the file specified at `registry_key_path` + gets populated with the content specified by `internal_key`. If + no file is specified, Omnibus GitLab defaults it to + `/var/opt/gitlab/gitlab-rails/etc/gitlab-registry.key` and populates it. 1. To change the container registry URL displayed in the GitLab Container @@ -686,8 +663,7 @@ response to events happening within the registry. Read more about the Container Registry notifications configuration options in the [Docker Registry notifications documentation](https://docs.docker.com/registry/notifications/). -NOTE: **Note:** -Multiple endpoints can be configured for the Container Registry. +You can configure multiple endpoints for the Container Registry. **Omnibus GitLab installations** @@ -761,28 +737,16 @@ project.container_repositories.find_each do |repo| end ``` -NOTE: **Note:** You can also [run cleanup on a schedule](../../user/packages/container_registry/index.md#cleanup-policy). ## Container Registry garbage collection -NOTE: **Note:** -The garbage collection tools are only available when you've installed GitLab -via an Omnibus package or the [cloud native chart](https://docs.gitlab.com/charts/charts/registry/#garbage-collection). - -DANGER: **Danger:** -By running the built-in garbage collection command, it will cause downtime to -the Container Registry. If you run this command on an instance in an environment -where one of your other instances is still writing to the Registry storage, -referenced manifests will be removed. To avoid that, make sure Registry is set to -[read-only mode](#performing-garbage-collection-without-downtime) before proceeding. - Container Registry can use considerable amounts of disk space. To clear up some unused layers, the registry includes a garbage collect command. GitLab offers a set of APIs to manipulate the Container Registry and aid the process of removing unused tags. Currently, this is exposed using the API, but in the future, -these controls will be migrated to the GitLab interface. +these controls should migrate to the GitLab interface. Project maintainers can [delete Container Registry tags in bulk](../../api/container_registry.md#delete-registry-repository-tags-in-bulk) @@ -791,6 +755,15 @@ it only unlinks tags from manifests and image blobs. To recycle the Container Registry data in the whole GitLab instance, you can use the built-in command provided by `gitlab-ctl`. +Prerequisites: + +- You must have installed GitLab by using an Omnibus package or the + [cloud native chart](https://docs.gitlab.com/charts/charts/registry/#garbage-collection). +- You must set the Registry to [read-only mode](#performing-garbage-collection-without-downtime). + Running garbage collection causes downtime for the Container Registry. When you run this command + on an instance in an environment where another instances is still writing to the Registry storage, + referenced manifests are removed. + ### Understanding the content-addressable layers Consider the following example, where you first build the image: @@ -818,15 +791,14 @@ no longer directly accessible via the `:latest` tag. > [Introduced](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/987) in Omnibus GitLab 8.12. -There are a couple of considerations you need to note before running the -built-in command: +Before you run the built-in command, note the following: -- The built-in command will stop the registry before it starts the garbage collection. +- The built-in command stops the registry before it starts the garbage collection. - The garbage collect command takes some time to complete, depending on the amount of data that exists. -- If you changed the location of registry configuration file, you will need to +- If you changed the location of registry configuration file, you must specify its path. -- After the garbage collection is done, the registry should start up automatically. +- After the garbage collection is done, the registry should start automatically. If you did not change the default location of the configuration file, run: @@ -882,7 +854,6 @@ During this time, you will be able to pull from the Container Registry, but you will not be able to push. -NOTE: **Note:** By default, the [registry storage path](#configure-storage-for-the-container-registry) is `/var/opt/gitlab/gitlab-rails/shared/registry`. @@ -1065,7 +1036,7 @@ Start with a value between `25000000` (25MB) and `50000000` (50MB). s3: accesskey: 'AKIAKIAKI' secretkey: 'secret123' - bucket: 'gitlab-registry-bucket-AKIAKIAKI' + bucket: 'gitlab-registry-bucket-AKIAKIAKI' chunksize: 25000000 ``` @@ -1166,12 +1137,7 @@ curl localhost:5001/debug/vars ### Advanced Troubleshooting -NOTE: **Note:** -The following section is only recommended for experts. - -Sometimes it's not obvious what is wrong, and you may need to dive deeper into -the communication between the Docker client and the Registry to find out -what's wrong. We will use a concrete example in the past to illustrate how to +We will use a concrete example in the past to illustrate how to diagnose a problem with the S3 setup. #### Unexpected 403 error during push diff --git a/doc/administration/packages/dependency_proxy.md b/doc/administration/packages/dependency_proxy.md index 2f9cfecc9d4..fba3d51f741 100644 --- a/doc/administration/packages/dependency_proxy.md +++ b/doc/administration/packages/dependency_proxy.md @@ -11,7 +11,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w GitLab can be utilized as a dependency proxy for a variety of common package managers. This is the administration documentation. If you want to learn how to use the -dependency proxies, see the [user guide](../../user/group/dependency_proxy/index.md). +dependency proxies, see the [user guide](../../user/packages/dependency_proxy/index.md). ## Enabling the Dependency Proxy feature @@ -135,28 +135,28 @@ This section describes the earlier configuration format. ## ## The location where build dependency_proxy are stored (default: shared/dependency_proxy). ## - #storage_path: shared/dependency_proxy + # storage_path: shared/dependency_proxy object_store: enabled: false - remote_directory: dependency_proxy # The bucket name. - #direct_upload: false # Use Object Storage directly for uploads instead of background uploads if enabled (Default: false). - #background_upload: true # Temporary option to limit automatic upload (Default: true). - #proxy_download: false # Passthrough all downloads via GitLab instead of using Redirects to Object Storage. + remote_directory: dependency_proxy # The bucket name. + # direct_upload: false # Use Object Storage directly for uploads instead of background uploads if enabled (Default: false). + # background_upload: true # Temporary option to limit automatic upload (Default: true). + # proxy_download: false # Passthrough all downloads via GitLab instead of using Redirects to Object Storage. connection: + ## + ## If the provider is AWS S3, use the following + ## + provider: AWS + region: us-east-1 + aws_access_key_id: AWS_ACCESS_KEY_ID + aws_secret_access_key: AWS_SECRET_ACCESS_KEY ## - ## If the provider is AWS S3, uncomment the following + ## If the provider is other than AWS (an S3-compatible one), comment out the previous 4 lines and use the following instead: ## - #provider: AWS - #region: us-east-1 - #aws_access_key_id: AWS_ACCESS_KEY_ID - #aws_secret_access_key: AWS_SECRET_ACCESS_KEY - ## - ## If the provider is other than AWS (an S3-compatible one), uncomment the following - ## - #host: 's3.amazonaws.com' # default: s3.amazonaws.com. - #aws_signature_version: 4 # For creation of signed URLs. Set to 2 if provider does not support v4. - #endpoint: 'https://s3.amazonaws.com' # Useful for S3-compliant services such as DigitalOcean Spaces. - #path_style: false # If true, use 'host/bucket_name/object' instead of 'bucket_name.host/object'. + # host: 's3.amazonaws.com' # default: s3.amazonaws.com. + # aws_signature_version: 4 # For creation of signed URLs. Set to 2 if provider does not support v4. + # endpoint: 'https://s3.amazonaws.com' # Useful for S3-compliant services such as DigitalOcean Spaces. + # path_style: false # If true, use 'host/bucket_name/object' instead of 'bucket_name.host/object'. ``` 1. [Restart GitLab](../restart_gitlab.md#installations-from-source "How to restart GitLab") for the changes to take effect. diff --git a/doc/administration/packages/index.md b/doc/administration/packages/index.md index 1061f3c33db..0b3a7ae9fa5 100644 --- a/doc/administration/packages/index.md +++ b/doc/administration/packages/index.md @@ -14,13 +14,14 @@ The Packages feature allows GitLab to act as a repository for the following: | Software repository | Description | Available in GitLab version | | ------------------- | ----------- | --------------------------- | -| [PyPi Repository](../../user/packages/pypi_repository/index.md) | The GitLab PyPi Repository enables every project in GitLab to have its own space to store [PyPi](https://pypi.org/) packages. | 12.10+ | +| [PyPI Repository](../../user/packages/pypi_repository/index.md) | The GitLab PyPI Repository enables every project in GitLab to have its own space to store [PyPI](https://pypi.org/) packages. | 12.10+ | | [Composer Repository](../../user/packages/composer_repository/index.md) | The GitLab Composer Repository enables every project in GitLab to have its own space to store [Composer](https://getcomposer.org/) packages. | 13.1+ | | [NuGet Repository](../../user/packages/nuget_repository/index.md) | The GitLab NuGet Repository enables every project in GitLab to have its own space to store [NuGet](https://www.nuget.org/) packages. | 12.8+ | | [Conan Repository](../../user/packages/conan_repository/index.md) | The GitLab Conan Repository enables every project in GitLab to have its own space to store [Conan](https://conan.io/) packages. | 12.4+ | | [Maven Repository](../../user/packages/maven_repository/index.md) | The GitLab Maven Repository enables every project in GitLab to have its own space to store [Maven](https://maven.apache.org/) packages. | 11.3+ | | [NPM Registry](../../user/packages/npm_registry/index.md) | The GitLab NPM Registry enables every project in GitLab to have its own space to store [NPM](https://www.npmjs.com/) packages. | 11.7+ | | [Go Proxy](../../user/packages/go_proxy/index.md) | The Go proxy for GitLab enables every project in GitLab to be fetched with the [Go proxy protocol](https://proxy.golang.org/). | 13.1+ | +| [Generic packages](../../user/packages/generic_packages/index.md) | Store arbitrary files, like release binaries. | 13.5+ | Don't you see your package management system supported yet? Please consider contributing @@ -142,33 +143,33 @@ We recommend using the [consolidated object storage settings](../object_storage. 1. Edit the `packages` section in `config/gitlab.yml` (uncomment where necessary): ```yaml - packages: - enabled: true + packages: + enabled: true + ## + ## The location where build packages are stored (default: shared/packages). + ## + # storage_path: shared/packages + object_store: + enabled: false + remote_directory: packages # The bucket name. + # direct_upload: false # Use Object Storage directly for uploads instead of background uploads if enabled (Default: false). + # background_upload: true # Temporary option to limit automatic upload (Default: true). + # proxy_download: false # Passthrough all downloads via GitLab instead of using Redirects to Object Storage. + connection: ## - ## The location where build packages are stored (default: shared/packages). + ## If the provider is AWS S3, use the following: ## - #storage_path: shared/packages - object_store: - enabled: false - remote_directory: packages # The bucket name. - #direct_upload: false # Use Object Storage directly for uploads instead of background uploads if enabled (Default: false). - #background_upload: true # Temporary option to limit automatic upload (Default: true). - #proxy_download: false # Passthrough all downloads via GitLab instead of using Redirects to Object Storage. - connection: - ## - ## If the provider is AWS S3, uncomment the following - ## - #provider: AWS - #region: us-east-1 - #aws_access_key_id: AWS_ACCESS_KEY_ID - #aws_secret_access_key: AWS_SECRET_ACCESS_KEY - ## - ## If the provider is other than AWS (an S3-compatible one), uncomment the following - ## - #host: 's3.amazonaws.com' # default: s3.amazonaws.com. - #aws_signature_version: 4 # For creation of signed URLs. Set to 2 if provider does not support v4. - #endpoint: 'https://s3.amazonaws.com' # Useful for S3-compliant services such as DigitalOcean Spaces. - #path_style: false # If true, use 'host/bucket_name/object' instead of 'bucket_name.host/object'. + provider: AWS + region: us-east-1 + aws_access_key_id: AWS_ACCESS_KEY_ID + aws_secret_access_key: AWS_SECRET_ACCESS_KEY + ## + ## If the provider is other than AWS (an S3-compatible one), comment out the previous 4 lines and use the following instead: + ## + # host: 's3.amazonaws.com' # default: s3.amazonaws.com. + # aws_signature_version: 4 # For creation of signed URLs. Set to 2 if provider does not support v4. + # endpoint: 'https://s3.amazonaws.com' # Useful for S3-compliant services such as DigitalOcean Spaces. + # path_style: false # If true, use 'host/bucket_name/object' instead of 'bucket_name.host/object'. ``` 1. Save the file and [restart GitLab](../restart_gitlab.md#installations-from-source) for the changes to take effect. diff --git a/doc/administration/pages/index.md b/doc/administration/pages/index.md index 3c0030be629..9f72293a730 100644 --- a/doc/administration/pages/index.md +++ b/doc/administration/pages/index.md @@ -53,8 +53,14 @@ supporting custom domains a secondary IP is not needed. Before proceeding with the Pages configuration, you will need to: -1. Have an exclusive root domain for serving GitLab Pages. Note that you cannot - use a subdomain of your GitLab's instance domain. +1. Have a domain for Pages that is not a subdomain of your GitLab's instance domain. + + | GitLab domain | Pages domain | Does it work? | + | :---: | :---: | :---: | + | `example.com` | `example.io` | **{check-circle}** Yes | + | `example.com` | `pages.example.com` | **{dotted-circle}** No | + | `gitlab.example.com` | `pages.example.com` | **{check-circle}** Yes | + 1. Configure a **wildcard DNS record**. 1. (Optional) Have a **wildcard certificate** for that domain if you decide to serve Pages under HTTPS. @@ -235,7 +241,7 @@ control over how the Pages daemon runs and serves content in your environment. | `pages_path` | The directory on disk where pages are stored, defaults to `GITLAB-RAILS/shared/pages`. | `pages_nginx[]` | | | `enable` | Include a virtual host `server{}` block for Pages inside NGINX. Needed for NGINX to proxy traffic back to the Pages daemon. Set to `false` if the Pages daemon should directly receive all requests, for example, when using [custom domains](index.md#custom-domains). -| `FF_ENABLE_REDIRECTS` | Feature flag to enable redirects. See the [redirects documentation](../../user/project/pages/redirects.md#enable-or-disable-redirects) for more info. | +| `FF_ENABLE_REDIRECTS` | Feature flag to disable redirects (enabled by default). Read the [redirects documentation](../../user/project/pages/redirects.md#disable-redirects) for more info. | --- @@ -370,8 +376,8 @@ Pages access control is disabled by default. To enable it: 1. Users can now configure it in their [projects' settings](../../user/project/pages/pages_access_control.md). NOTE: **Important:** -For multi-node setups, in order for this setting to be effective, it has to be applied -to all the App nodes as well as the Sidekiq nodes. +For this setting to be effective with multi-node setups, it has to be applied to +all the App nodes and Sidekiq nodes. #### Disabling public access to all Pages websites @@ -397,8 +403,7 @@ redeployed. This issue will be resolved by ### Running behind a proxy Like the rest of GitLab, Pages can be used in those environments where external -internet connectivity is gated by a proxy. In order to use a proxy for GitLab -pages: +internet connectivity is gated by a proxy. To use a proxy for GitLab Pages: 1. Configure in `/etc/gitlab/gitlab.rb`: @@ -425,10 +430,6 @@ Authority (CA) in the system certificate store. For Omnibus, this is fixed by [installing a custom CA in Omnibus GitLab](https://docs.gitlab.com/omnibus/settings/ssl.html#install-custom-public-certificates). -## Enable redirects - -In GitLab Pages, you can [enable the redirects feature](../../user/project/pages/redirects.md#enable-or-disable-redirects) to configure rules to forward one URL to another using HTTP redirects. - ## Activate verbose logging for daemon Verbose logging was [introduced](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/2533) in @@ -508,7 +509,8 @@ To override the global maximum pages size for a specific group: ## Running GitLab Pages on a separate server -You can run the GitLab Pages daemon on a separate server in order to decrease the load on your main application server. +You can run the GitLab Pages daemon on a separate server to decrease the load on +your main application server. To configure GitLab Pages on a separate server: @@ -595,7 +597,7 @@ database encryption. Proceed with caution. ```ruby pages_external_url "http://<pages_server_URL>" gitlab_pages['enable'] = false - gitlab_rails['pages_enabled']=false + pages_nginx['enable'] = false gitlab_rails['pages_path'] = "/mnt/pages" ``` @@ -779,3 +781,16 @@ For example, if there is a connection timeout: ```plaintext error="failed to connect to internal Pages API: Get \"https://gitlab.example.com:3000/api/v4/internal/pages/status\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)" ``` + +### 500 error with `securecookie: failed to generate random iv` and `Failed to save the session` + +This problem most likely results from an [out-dated operating system](https://docs.gitlab.com/omnibus/package-information/deprecated_os.html). +The [Pages daemon uses the `securecookie` library](https://gitlab.com/search?group_id=9970&project_id=734943&repository_ref=master&scope=blobs&search=securecookie&snippets=false) to get random strings via [crypto/rand in Go](https://golang.org/pkg/crypto/rand/#pkg-variables). +This requires the `getrandom` syscall or `/dev/urandom` to be available on the host OS. +Upgrading to an [officially supported operating system](https://about.gitlab.com/install/) is recommended. + +### The requested scope is invalid, malformed, or unknown + +This problem comes from the permissions of the GitLab Pages OAuth application. To fix it, go to +**Admin > Applications > GitLab Pages** and edit the application. Under **Scopes**, ensure that the +`api` scope is selected and save your changes. diff --git a/doc/administration/pages/source.md b/doc/administration/pages/source.md index 662817e7411..87217b141a4 100644 --- a/doc/administration/pages/source.md +++ b/doc/administration/pages/source.md @@ -347,10 +347,6 @@ world. Custom domains and TLS are supported. 1. Restart NGINX 1. [Restart GitLab](../restart_gitlab.md#installations-from-source) -## Enable redirects - -In GitLab Pages, you can [enable the redirects feature](../../user/project/pages/redirects.md#enable-or-disable-redirects) to configure rules to forward one URL to another using HTTP redirects. - ## NGINX caveats NOTE: **Note:** diff --git a/doc/administration/postgresql/external.md b/doc/administration/postgresql/external.md index 632b68fb014..4a164c66578 100644 --- a/doc/administration/postgresql/external.md +++ b/doc/administration/postgresql/external.md @@ -1,3 +1,9 @@ +--- +stage: Enablement +group: Database +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Configure GitLab using an external PostgreSQL service If you're hosting GitLab on a cloud provider, you can optionally use a diff --git a/doc/administration/postgresql/index.md b/doc/administration/postgresql/index.md index 2720d8e696b..c7ec46db654 100644 --- a/doc/administration/postgresql/index.md +++ b/doc/administration/postgresql/index.md @@ -1,11 +1,14 @@ --- +stage: Enablement +group: Database +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference --- # Configuring PostgreSQL for scaling In this section, you'll be guided through configuring a PostgreSQL database to -be used with GitLab in one of our [Scalable and Highly Available Setups](../reference_architectures/index.md). +be used with GitLab in one of our [reference architectures](../reference_architectures/index.md). There are essentially three setups to choose from. ## PostgreSQL replication and failover with Omnibus GitLab **(PREMIUM ONLY)** diff --git a/doc/administration/postgresql/pgbouncer.md b/doc/administration/postgresql/pgbouncer.md index 9db3e017359..7760b197267 100644 --- a/doc/administration/postgresql/pgbouncer.md +++ b/doc/administration/postgresql/pgbouncer.md @@ -1,4 +1,7 @@ --- +stage: Enablement +group: Database +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference --- @@ -145,6 +148,35 @@ ote_pid | tls (1 row) ``` +## Procedure for bypassing PgBouncer + +Some database changes have to be done directly, and not through PgBouncer. This includes database restores and GitLab upgrades (because of the database migrations). + +1. To find the primary node, run the following on a database node: + + ```shell + sudo gitlab-ctl repmgr cluster show + ``` + +1. Edit `/etc/gitlab/gitlab.rb` on the application node you're performing the task on, and update + `gitlab_rails['db_host']` and `gitlab_rails['db_port']` with the database + primary's host and port. + +1. Run reconfigure: + + ```shell + sudo gitlab-ctl reconfigure + ``` + +Once you've performed the tasks or procedure, switch back to using PgBouncer: + +1. Change back `/etc/gitlab/gitlab.rb` to point to PgBouncer. +1. Run reconfigure: + + ```shell + sudo gitlab-ctl reconfigure + ``` + ## Troubleshooting In case you are experiencing any issues connecting through PgBouncer, the first diff --git a/doc/administration/postgresql/replication_and_failover.md b/doc/administration/postgresql/replication_and_failover.md index bc2af167e6c..0a40b61fd3c 100644 --- a/doc/administration/postgresql/replication_and_failover.md +++ b/doc/administration/postgresql/replication_and_failover.md @@ -1,10 +1,16 @@ +--- +stage: Enablement +group: Database +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # PostgreSQL replication and failover with Omnibus GitLab **(PREMIUM ONLY)** -This document will focus only on configuration supported with [GitLab Premium](https://about.gitlab.com/pricing/), using the Omnibus GitLab package. -If you are a Community Edition or Starter user, consider using a cloud hosted solution. -This document will not cover installations from source. +This document focuses on configuration supported with [GitLab Premium](https://about.gitlab.com/pricing/), using the Omnibus GitLab package. +If you're a Community Edition or Starter user, consider using a cloud hosted solution. +This document doesn't cover installations from source. -If a setup with replication and failover is not what you were looking for, see +If a setup with replication and failover isn't what you were looking for, see the [database configuration document](https://docs.gitlab.com/omnibus/settings/database.html) for the Omnibus GitLab packages. @@ -87,9 +93,9 @@ information. #### Network information -PostgreSQL does not listen on any network interface by default. It needs to know -which IP address to listen on in order to be accessible to other services. -Similarly, PostgreSQL access is controlled based on the network source. +PostgreSQL doesn't listen on any network interface by default. It needs to know +which IP address to listen on to be accessible to other services. Similarly, +PostgreSQL access is controlled based on the network source. This is why you will need: @@ -419,7 +425,7 @@ Check the [Troubleshooting section](#troubleshooting) before proceeding. 1. Make sure you collect [`CONSUL_SERVER_NODES`](#consul-information), [`CONSUL_PASSWORD_HASH`](#consul-information), and [`PGBOUNCER_PASSWORD_HASH`](#pgbouncer-information) before executing the next step. -1. One each node, edit the `/etc/gitlab/gitlab.rb` config file and replace values noted in the `# START user configuration` section as below: +1. One each node, edit the `/etc/gitlab/gitlab.rb` configuration file and replace values noted in the `# START user configuration` section as below: ```ruby # Disable all components except PgBouncer and Consul agent @@ -1263,6 +1269,11 @@ with: sudo gitlab-ctl stop patroni ``` +Note that stopping or restarting Patroni service on the leader node will trigger the automatic failover. If you +want to signal Patroni to reload its configuration or restart PostgreSQL process without triggering the failover, you +must use the `reload` or `restart` sub-commands of `gitlab-ctl patroni` instead. These two sub-commands are wrappers of +the same `patronictl` commands. + ### Manual failover procedure for Patroni While Patroni supports automatic failover, you also have the ability to perform @@ -1348,3 +1359,93 @@ You can switch an exiting database cluster to use Patroni instead of repmgr with 1. Repeat the last two steps for all replica nodes. `gitlab.rb` should look the same on all nodes. 1. Optional: You can remove `gitlab_repmgr` database and role on the primary. + +### Upgrading PostgreSQL major version in a Patroni cluster + +As of GitLab 13.3, PostgreSQL 11.7 and 12.3 are both shipped with Omnibus GitLab. GitLab still +uses PostgreSQL 11 by default. Therefore `gitlab-ctl pg-upgrade` does not automatically upgrade +to PostgreSQL 12. If you want to upgrade to PostgreSQL 12, you must ask for it explicitly. + +CAUTION: **Warning:** +The procedure for upgrading PostgreSQL in a Patroni cluster is different than when upgrading using repmgr. +The following outlines the key differences and important considerations that need to be accounted for when +upgrading PostgreSQL. + +Here are a few key facts that you must consider before upgrading PostgreSQL: + +- The main point is that you will have to **shut down the Patroni cluster**. This means that your + GitLab deployment will be down for the duration of database upgrade or, at least, as long as your leader + node is upgraded. This can be **a significant downtime depending on the size of your database**. + +- Upgrading PostgreSQL creates a new data directory with a new control data. From Patroni's perspective + this is a new cluster that needs to be bootstrapped again. Therefore, as part of the upgrade procedure, + the cluster state, which is stored in Consul, will be wiped out. Once the upgrade is completed, Patroni + will be instructed to bootstrap a new cluster. **Note that this will change your _cluster ID_**. + +- The procedures for upgrading leader and replicas are not the same. That is why it is important to use the + right procedure on each node. + +- Upgrading a replica node **deletes the data directory and resynchronizes it** from the leader using the + configured replication method (currently `pg_basebackup` is the only available option). It might take some + time for replica to catch up with the leader, depending on the size of your database. + +- An overview of the upgrade procedure is outlined in [Patoni's documentation](https://patroni.readthedocs.io/en/latest/existing_data.html#major-upgrade-of-postgresql-version). + You can still use `gitlab-ctl pg-upgrade` which implements this procedure with a few adjustments. + +Considering these, you should carefully plan your PostgreSQL upgrade: + +1. Find out which node is the leader and which node is a replica: + + ```shell + gitlab-ctl patroni members + ``` + + NOTE: **Note:** + `gitlab-ctl pg-upgrade` tries to detect the role of the node. If for any reason the auto-detection + does not work or you believe it did not detect the role correctly, you can use the `--leader` or `--replica` + arguments to manually override it. + +1. Stop Patroni **only on replicas**. + + ```shell + sudo gitlab-ctl stop patroni + ``` + +1. Enable the maintenance mode on the **application node**: + + ```shell + sudo gitlab-ctl deploy-page up + ``` + +1. Upgrade PostgreSQL on **the leader node** and make sure that the upgrade is completed successfully: + + ```shell + sudo gitlab-ctl pg-upgrade -V 12 + ``` + +1. Check the status of the leader and cluster. You can only proceed if you have a healthy leader: + + ```shell + gitlab-ctl patroni check-leader + + # OR + + gitlab-ctl patroni members + ``` + +1. You can now disable the maintenance mode on the **application node**: + + ```shell + sudo gitlab-ctl deploy-page down + ``` + +1. Upgrade PostgreSQL **on replicas** (you can do this in parallel on all of them): + + ```shell + sudo gitlab-ctl pg-upgrade -V 12 + ``` + +NOTE: **Note:** +Reverting PostgreSQL upgrade with `gitlab-ctl revert-pg-upgrade` has the same considerations as +`gitlab-ctl pg-upgrade`. You should follow the same procedure by first stopping the replicas, +then reverting the leader, and finally reverting the replicas. diff --git a/doc/administration/postgresql/standalone.md b/doc/administration/postgresql/standalone.md index 2747749066e..2ac74e8a4a0 100644 --- a/doc/administration/postgresql/standalone.md +++ b/doc/administration/postgresql/standalone.md @@ -1,15 +1,21 @@ +--- +stage: Enablement +group: Database +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Standalone PostgreSQL using Omnibus GitLab **(CORE ONLY)** If you wish to have your database service hosted separately from your GitLab -application server(s), you can do this using the PostgreSQL binaries packaged +application servers, you can do this using the PostgreSQL binaries packaged together with Omnibus GitLab. This is recommended as part of our [reference architecture for up to 2,000 users](../reference_architectures/2k_users.md). ## Setting it up -1. SSH into the PostgreSQL server. -1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab - package you want using **steps 1 and 2** from the GitLab downloads page. +1. SSH in to the PostgreSQL server. +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package you want using *steps 1 and 2* from the GitLab downloads page. - Do not complete any other steps on the download page. 1. Generate a password hash for PostgreSQL. This assumes you will use the default username of `gitlab` (recommended). The command will request a password diff --git a/doc/administration/raketasks/check.md b/doc/administration/raketasks/check.md index 15014fffd01..bdccd6d5fe9 100644 --- a/doc/administration/raketasks/check.md +++ b/doc/administration/raketasks/check.md @@ -148,9 +148,38 @@ above. for the affected project(s). If the issue persists, try triggering `gc` via the -[Rails Console](../troubleshooting/navigating_gitlab_via_rails_console.md#starting-a-rails-console-session): +[Rails Console](../operations/rails_console.md#starting-a-rails-console-session): ```ruby p = Project.find_by_path("project-name") Projects::HousekeepingService.new(p, :gc).execute ``` + +### Delete references to missing remote uploads + +`gitlab-rake gitlab:uploads:check VERBOSE=1` detects remote objects that do not exist because they were +deleted externally but their references still exist in the GitLab database. + +Example output with error message: + +```shell +$ sudo gitlab-rake gitlab:uploads:check VERBOSE=1 +Checking integrity of Uploads +- 100..434: Failures: 2 +- Upload: 100: Remote object does not exist +- Upload: 101: Remote object does not exist +Done! +``` + +To delete these references to remote uploads that were deleted externally, open the [GitLab Rails Console](../operations/rails_console.md#starting-a-rails-console-session) and run: + +```ruby +uploads_deleted=0 +Upload.find_each do |upload| + next if upload.retrieve_uploader.file.exists? + uploads_deleted=uploads_deleted + 1 + p upload ### allow verification before destroy + # p upload.destroy! ### uncomment to actually destroy +end +p "#{uploads_deleted} remote objects were destroyed." +``` diff --git a/doc/administration/raketasks/doctor.md b/doc/administration/raketasks/doctor.md index 62d0af70706..c97aa5a4de1 100644 --- a/doc/administration/raketasks/doctor.md +++ b/doc/administration/raketasks/doctor.md @@ -47,9 +47,8 @@ I, [2020-06-11T17:18:15.575711 #27148] INFO -- : Done! ### Verbose mode -In order to get more detailed information about which -rows and columns cannot be decrypted, you can pass a VERBOSE -environment variable: +To get more detailed information about which rows and columns can't be +decrypted, you can pass a `VERBOSE` environment variable: **Omnibus Installation** diff --git a/doc/administration/raketasks/github_import.md b/doc/administration/raketasks/github_import.md index a46a2b34687..7f673a2c850 100644 --- a/doc/administration/raketasks/github_import.md +++ b/doc/administration/raketasks/github_import.md @@ -2,9 +2,8 @@ > [Introduced]( https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/10308) in GitLab 9.1. -In order to retrieve and import GitHub repositories, you will need a -[GitHub personal access token](https://github.com/settings/tokens). -A username should be passed as the second argument to the Rake task +To retrieve and import GitHub repositories, you need a [GitHub personal access token](https://github.com/settings/tokens). +A username should be passed as the second argument to the Rake task, which will become the owner of the project. You can resume an import with the same command. diff --git a/doc/administration/raketasks/uploads/sanitize.md b/doc/administration/raketasks/uploads/sanitize.md index 9586ab2c6f4..54aa62059cf 100644 --- a/doc/administration/raketasks/uploads/sanitize.md +++ b/doc/administration/raketasks/uploads/sanitize.md @@ -1,6 +1,6 @@ # Uploads sanitize Rake tasks **(CORE ONLY)** -Since GitLab 11.9, EXIF data is automatically stripped from JPG or TIFF image uploads. +In GitLab 11.9 and later, EXIF data is automatically stripped from JPG or TIFF image uploads. EXIF data may contain sensitive information (for example, GPS location), so you can remove EXIF data from existing images that were uploaded to an earlier version of GitLab. diff --git a/doc/administration/read_only_gitlab.md b/doc/administration/read_only_gitlab.md new file mode 100644 index 00000000000..681102a8c39 --- /dev/null +++ b/doc/administration/read_only_gitlab.md @@ -0,0 +1,125 @@ +--- +stage: Enablement +group: Distribution +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Place GitLab into a read-only state **(CORE ONLY)** + +CAUTION: **Warning:** +This document should be used as a temporary solution. +There's work in progress to make this +[possible with Geo](https://gitlab.com/groups/gitlab-org/-/epics/2149). + +In some cases, you might want to place GitLab under a read-only state. +The configuration for doing so depends on your desired outcome. + +## Make the repositories read-only + +The first thing you'll want to accomplish is to ensure that no changes can be +made to your repositories. There's two ways you can accomplish that: + +- Either stop Unicorn/Puma to make the internal API unreachable: + + ```shell + sudo gitlab-ctl stop puma # or unicorn + ``` + +- Or, open up a Rails console: + + ```shell + sudo gitlab-rails console + ``` + + And set the repositories for all projects read-only: + + ```ruby + Project.all.find_each { |project| project.update!(repository_read_only: true) } + ``` + + When you're ready to revert this, you can do so with the following command: + + ```ruby + Project.all.find_each { |project| project.update!(repository_read_only: false) } + ``` + +## Shut down the GitLab UI + +If you don't mind shutting down the GitLab UI, then the easiest approach is to +stop `sidekiq` and `puma`/`unicorn`, and you'll effectively ensure that no +changes can be made to GitLab: + +```shell +sudo gitlab-ctl stop sidekiq +sudo gitlab-ctl stop puma # or unicorn +``` + +When you're ready to revert this: + +```shell +sudo gitlab-ctl start sidekiq +sudo gitlab-ctl start puma # or unicorn +``` + +## Make the database read-only + +If you want to allow users to use the GitLab UI, then you'll need to ensure that +the database is read-only: + +1. Take a [GitLab backup](../raketasks/backup_restore.md#back-up-gitlab) + in case things don't go as expected. +1. Enter PostgreSQL on the console as an admin user: + + ```shell + sudo \ + -u gitlab-psql /opt/gitlab/embedded/bin/psql \ + -h /var/opt/gitlab/postgresql gitlabhq_production + ``` + +1. Create the `gitlab_read_only` user. Note that the password is set to `mypassword`, + change that to your liking: + + ```sql + -- NOTE: Use the password defined earlier + CREATE USER gitlab_read_only WITH password 'mypassword'; + GRANT CONNECT ON DATABASE gitlabhq_production to gitlab_read_only; + GRANT USAGE ON SCHEMA public TO gitlab_read_only; + GRANT SELECT ON ALL TABLES IN SCHEMA public TO gitlab_read_only; + GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO gitlab_read_only; + + -- Tables created by "gitlab" should be made read-only for "gitlab_read_only" + -- automatically. + ALTER DEFAULT PRIVILEGES FOR USER gitlab IN SCHEMA public GRANT SELECT ON TABLES TO gitlab_read_only; + ALTER DEFAULT PRIVILEGES FOR USER gitlab IN SCHEMA public GRANT SELECT ON SEQUENCES TO gitlab_read_only; + ``` + +1. Get the hashed password of the `gitlab_read_only` user and copy the result: + + ```shell + sudo gitlab-ctl pg-password-md5 gitlab_read_only + ``` + +1. Edit `/etc/gitlab/gitlab.rb` and add the password from the previous step: + + ```ruby + postgresql['sql_user_password'] = 'a2e20f823772650f039284619ab6f239' + postgresql['sql_user'] = "gitlab_read_only" + ``` + +1. Reconfigure GitLab and restart PostgreSQL: + + ```shell + sudo gitlab-ctl reconfigure + sudo gitlab-ctl restart postgresql + ``` + +When you're ready to revert the read-only state, you'll need to remove the added +lines in `/etc/gitlab/gitlab.rb`, and reconfigure GitLab and restart PostgreSQL: + +```shell +sudo gitlab-ctl reconfigure +sudo gitlab-ctl restart postgresql +``` + +Once you verify all works as expected, you can remove the `gitlab_read_only` +user from the database. diff --git a/doc/administration/redis/replication_and_failover.md b/doc/administration/redis/replication_and_failover.md index ca041adb1d8..72a52ba0a1f 100644 --- a/doc/administration/redis/replication_and_failover.md +++ b/doc/administration/redis/replication_and_failover.md @@ -124,9 +124,9 @@ each other over the network. ### Sentinel setup overview Sentinels watch both other Sentinels and Redis nodes. Whenever a Sentinel -detects that a Redis node is not responding, it will announce that to the -other Sentinels. They have to reach the **quorum**, that is the minimum amount -of Sentinels that agrees a node is down, in order to be able to start a failover. +detects that a Redis node isn't responding, it announces the node's status to +the other Sentinels. The Sentinels have to reach a _quorum_ (the minimum amount +of Sentinels agreeing a node is down) to be able to start a failover. Whenever the **quorum** is met, the **majority** of all known Sentinel nodes need to be available and reachable, so that they can elect the Sentinel **leader** diff --git a/doc/administration/redis/replication_and_failover_external.md b/doc/administration/redis/replication_and_failover_external.md index ce452d30fc2..7561f1c0fbf 100644 --- a/doc/administration/redis/replication_and_failover_external.md +++ b/doc/administration/redis/replication_and_failover_external.md @@ -71,7 +71,7 @@ requirements: - All Redis servers in this guide must be configured to use a TCP connection instead of a socket. To configure Redis to use TCP connections you need to - define both `bind` and `port` in the Redis config file. You can bind to all + define both `bind` and `port` in the Redis configuration file. You can bind to all interfaces (`0.0.0.0`) or specify the IP of the desired interface (e.g., one from an internal network). - Since Redis 3.2, you must define a password to receive external connections @@ -228,13 +228,13 @@ which ideally should not have Redis or Sentinels in the same machine: sentinels: - host: 10.0.0.1 - port: 26379 # point to sentinel, not to redis port + port: 26379 # point to sentinel, not to redis port - host: 10.0.0.2 - port: 26379 # point to sentinel, not to redis port + port: 26379 # point to sentinel, not to redis port - host: 10.0.0.3 - port: 26379 # point to sentinel, not to redis port + port: 26379 # point to sentinel, not to redis port ``` 1. [Restart GitLab](../restart_gitlab.md#installations-from-source) for the changes to take effect. @@ -353,13 +353,13 @@ or a failover promotes a different **Primary** node. sentinels: - host: 10.0.0.1 - port: 26379 # point to sentinel, not to redis port + port: 26379 # point to sentinel, not to redis port - host: 10.0.0.2 - port: 26379 # point to sentinel, not to redis port + port: 26379 # point to sentinel, not to redis port - host: 10.0.0.3 - port: 26379 # point to sentinel, not to redis port + port: 26379 # point to sentinel, not to redis port ``` 1. [Restart GitLab](../restart_gitlab.md#installations-from-source) for the changes to take effect. diff --git a/doc/administration/redis/troubleshooting.md b/doc/administration/redis/troubleshooting.md index 402b60e5b7b..c9976ac791d 100644 --- a/doc/administration/redis/troubleshooting.md +++ b/doc/administration/redis/troubleshooting.md @@ -146,13 +146,13 @@ production: sentinels: - host: 10.0.0.1 - port: 26379 # point to sentinel, not to redis port + port: 26379 # point to sentinel, not to redis port - host: 10.0.0.2 - port: 26379 # point to sentinel, not to redis port + port: 26379 # point to sentinel, not to redis port - host: 10.0.0.3 - port: 26379 # point to sentinel, not to redis port + port: 26379 # point to sentinel, not to redis port ``` When in doubt, read the [Redis Sentinel documentation](https://redis.io/topics/sentinel). diff --git a/doc/administration/reference_architectures/10k_users.md b/doc/administration/reference_architectures/10k_users.md index 5f8ab6683a9..7910905ce54 100644 --- a/doc/administration/reference_architectures/10k_users.md +++ b/doc/administration/reference_architectures/10k_users.md @@ -17,21 +17,21 @@ full list of reference architectures, see | Service | Nodes | Configuration | GCP | AWS | Azure | |--------------------------------------------|-------------|-------------------------|-----------------|-------------|----------| -| External load balancing node | 1 | 2 vCPU, 1.8GB memory | n1-highcpu-2 | c5.large | F2s v2 | -| Consul | 3 | 2 vCPU, 1.8GB memory | n1-highcpu-2 | c5.large | F2s v2 | -| PostgreSQL | 3 | 4 vCPU, 15GB memory | n1-standard-4 | m5.xlarge | D4s v3 | -| PgBouncer | 3 | 2 vCPU, 1.8GB memory | n1-highcpu-2 | c5.large | F2s v2 | -| Internal load balancing node | 1 | 2 vCPU, 1.8GB memory | n1-highcpu-2 | c5.large | F2s v2 | -| Redis - Cache | 3 | 4 vCPU, 15GB memory | n1-standard-4 | m5.xlarge | D4s v3 | -| Redis - Queues / Shared State | 3 | 4 vCPU, 15GB memory | n1-standard-4 | m5.xlarge | D4s v3 | -| Redis Sentinel - Cache | 3 | 1 vCPU, 1.7GB memory | g1-small | t2.small | B1MS | -| Redis Sentinel - Queues / Shared State | 3 | 1 vCPU, 1.7GB memory | g1-small | t2.small | B1MS | -| Gitaly | 2 (minimum) | 16 vCPU, 60GB memory | n1-standard-16 | m5.4xlarge | D16s v3 | -| Sidekiq | 4 | 4 vCPU, 15GB memory | n1-standard-4 | m5.xlarge | D4s v3 | -| GitLab Rails | 3 | 32 vCPU, 28.8GB memory | n1-highcpu-32 | c5.9xlarge | F32s v2 | -| Monitoring node | 1 | 4 vCPU, 3.6GB memory | n1-highcpu-4 | c5.xlarge | F4s v2 | -| Object Storage | n/a | n/a | n/a | n/a | n/a | -| NFS Server | 1 | 4 vCPU, 3.6GB memory | n1-highcpu-4 | c5.xlarge | F4s v2 | +| External load balancing node | 1 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| Consul | 3 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| PostgreSQL | 3 | 4 vCPU, 15 GB memory | n1-standard-4 | m5.xlarge | D4s v3 | +| PgBouncer | 3 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| Internal load balancing node | 1 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| Redis - Cache | 3 | 4 vCPU, 15 GB memory | n1-standard-4 | m5.xlarge | D4s v3 | +| Redis - Queues / Shared State | 3 | 4 vCPU, 15 GB memory | n1-standard-4 | m5.xlarge | D4s v3 | +| Redis Sentinel - Cache | 3 | 1 vCPU, 1.7 GB memory | g1-small | t2.small | B1MS | +| Redis Sentinel - Queues / Shared State | 3 | 1 vCPU, 1.7 GB memory | g1-small | t2.small | B1MS | +| Gitaly | 2 (minimum) | 16 vCPU, 60 GB memory | n1-standard-16 | m5.4xlarge | D16s v3 | +| Sidekiq | 4 | 4 vCPU, 15 GB memory | n1-standard-4 | m5.xlarge | D4s v3 | +| GitLab Rails | 3 | 32 vCPU, 28.8 GB memory | n1-highcpu-32 | c5.9xlarge | F32s v2 | +| Monitoring node | 1 | 4 vCPU, 3.6 GB memory | n1-highcpu-4 | c5.xlarge | F4s v2 | +| Object storage | n/a | n/a | n/a | n/a | n/a | +| NFS server | 1 | 4 vCPU, 3.6 GB memory | n1-highcpu-4 | c5.xlarge | F4s v2 | The Google Cloud Platform (GCP) architectures were built and tested using the [Intel Xeon E5 v3 (Haswell)](https://cloud.google.com/compute/docs/cpu-platforms) @@ -40,41 +40,43 @@ or higher, are required for your CPU or node counts. For more information, see our [Sysbench](https://github.com/akopytov/sysbench)-based [CPU benchmark](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Reference-Architectures/GCP-CPU-Benchmarks). -For data objects (such as LFS, Uploads, or Artifacts), an -[object storage service](#configure-the-object-storage) is recommended instead -of NFS where possible, due to better performance and availability. Since this -doesn't require a node to be set up, *Object Storage* is noted as not -applicable (n/a) in the previous table. +Due to better performance and availability, for data objects (such as LFS, +uploads, or artifacts), using an [object storage service](#configure-the-object-storage) +is recommended instead of using NFS. Using an object storage service also +doesn't require you to provision and maintain a node. ## Setup components To set up GitLab and its components to accommodate up to 10,000 users: 1. [Configure the external load balancing node](#configure-the-external-load-balancer) - that will handle the load balancing of the three GitLab application services nodes. + to handle the load balancing of the GitLab application services nodes. 1. [Configure Consul](#configure-consul). 1. [Configure PostgreSQL](#configure-postgresql), the database for GitLab. 1. [Configure PgBouncer](#configure-pgbouncer). -1. [Configure the internal load balancing node](#configure-the-internal-load-balancer) +1. [Configure the internal load balancing node](#configure-the-internal-load-balancer). 1. [Configure Redis](#configure-redis). 1. [Configure Gitaly](#configure-gitaly), which provides access to the Git repositories. 1. [Configure Sidekiq](#configure-sidekiq). 1. [Configure the main GitLab Rails application](#configure-gitlab-rails) - to run Puma/Unicorn, Workhorse, GitLab Shell, and to serve all frontend requests (UI, API, Git - over HTTP/SSH). -1. [Configure Prometheus](#configure-prometheus) to monitor your GitLab environment. -1. [Configure the Object Storage](#configure-the-object-storage) + to run Puma/Unicorn, Workhorse, GitLab Shell, and to serve all frontend + requests (which include UI, API, and Git over HTTP/SSH). +1. [Configure Prometheus](#configure-prometheus) to monitor your GitLab + environment. +1. [Configure the object storage](#configure-the-object-storage) used for shared data objects. -1. [Configure NFS (Optional)](#configure-nfs-optional) - to have shared disk storage service as an alternative to Gitaly and/or Object Storage (although - not recommended). NFS is required for GitLab Pages, you can skip this step if you're not using - that feature. +1. [Configure Advanced Search](#configure-advanced-search) (optional) for faster, + more advanced code search across your entire GitLab instance. +1. [Configure NFS](#configure-nfs-optional) (optional, and not recommended) + to have shared disk storage service as an alternative to Gitaly or object + storage. You can skip this step if you're not using GitLab Pages (which + requires NFS). -We start with all servers on the same 10.6.0.0/24 private network range, they -can connect to each other freely on those addresses. +The servers start on the same 10.6.0.0/24 private network range, and can +connect to each other freely on these addresses. -Here is a list and description of each machine and the assigned IP: +The following list includes descriptions of each server and its assigned IP: - `10.6.0.10`: External Load Balancer - `10.6.0.11`: Consul 1 @@ -112,19 +114,18 @@ Here is a list and description of each machine and the assigned IP: ## Configure the external load balancer -NOTE: **Note:** +In an active/active GitLab configuration, you'll need a load balancer to route +traffic to the application servers. The specifics on which load balancer to use +or its exact configuration is beyond the scope of GitLab documentation. We hope +that if you're managing multi-node systems like GitLab, you already have a load +balancer of choice. Some load balancer examples include HAProxy (open-source), +F5 Big-IP LTM, and Citrix Net Scaler. This documentation outline the ports and +protocols needed for use with GitLab. + This architecture has been tested and validated with [HAProxy](https://www.haproxy.org/) as the load balancer. Although other load balancers with similar feature sets could also be used, those load balancers have not been validated. -In an active/active GitLab configuration, you will need a load balancer to route -traffic to the application servers. The specifics on which load balancer to use -or the exact configuration is beyond the scope of GitLab documentation. We hope -that if you're managing multi-node systems like GitLab you have a load balancer of -choice already. Some examples including HAProxy (open-source), F5 Big-IP LTM, -and Citrix Net Scaler. This documentation will outline what ports and protocols -you need to use with GitLab. - The next question is how you will handle SSL in your environment. There are several different options: @@ -247,14 +248,11 @@ with the other servers. To configure Consul: -1. SSH into the server that will host Consul. -1. [Download/install](https://about.gitlab.com/install/) the - Omnibus GitLab Enterprise Edition package using **steps 1 and 2** from the - GitLab downloads page. - - Make sure you select the correct Omnibus package, with the same version - the GitLab application is running. - - Do not complete any other steps on the download page. - +1. SSH in to the server that will host Consul. +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to both follow _only_ installation steps 1 and 2 + on the page, and to select the correct Omnibus GitLab package, with the same version + and type (Community or Enterprise editions) as your current install. 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby @@ -282,10 +280,9 @@ To configure Consul: 1. Go through the steps again for all the other Consul nodes, and make sure you set up the correct IPs. -NOTE: **Note:** -A Consul leader will be elected when the provisioning of the third Consul server is completed. -Viewing the Consul logs `sudo gitlab-ctl tail consul` will display -`...[INFO] consul: New leader elected: ...` +A Consul leader is _elected_ when the provisioning of the third Consul server is +complete. Viewing the Consul logs `sudo gitlab-ctl tail consul` displays +`...[INFO] consul: New leader elected: ...`. You can list the current Consul members (server, client): @@ -352,7 +349,7 @@ in the second step, do not supply the `EXTERNAL_URL` value. #### PostgreSQL primary node -1. SSH into the PostgreSQL primary node. +1. SSH in to the PostgreSQL primary node. 1. Generate a password hash for the PostgreSQL username/password pair. This assumes you will use the default username of `gitlab` (recommended). The command will request a password and confirmation. Use the value that is output by this command in the next @@ -513,7 +510,7 @@ are supported and can be added if needed. #### PostgreSQL post-configuration -SSH into the **primary node**: +SSH in to the **primary node**: 1. Open a database prompt: @@ -548,7 +545,7 @@ SSH into the **primary node**: is not an IP address, it will need to be a resolvable name (via DNS or `/etc/hosts`) -SSH into the **secondary node**: +SSH in to the **secondary node**: 1. Set up the repmgr standby: @@ -662,7 +659,6 @@ The following IPs will be used as an example: 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. - NOTE: **Note:** If an error `execute[generate databases.ini]` occurs, this is due to an existing [known issue](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/4713). It will be resolved when you run a second `reconfigure` after the next step. @@ -796,35 +792,31 @@ to be used with GitLab. The following IPs will be used as an example: - `10.6.0.82`: Sentinel - Queues 2 - `10.6.0.83`: Sentinel - Queues 3 -NOTE: **Providing your own Redis instance:** -Managed Redis from cloud providers such as AWS ElastiCache will work. If these -services support high availability, be sure it is **not** the Redis Cluster type. -Redis version 5.0 or higher is required, as this is what ships with -Omnibus GitLab packages starting with GitLab 13.0. Older Redis versions -do not support an optional count argument to SPOP which is now required for -[Merge Trains](../../ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md). +### Providing your own Redis instance + +Managed Redis from cloud providers (such as AWS ElastiCache) will work. If these +services support high availability, be sure it _isn't_ of the Redis Cluster type. +Redis version 5.0 or higher is required, which is included with Omnibus GitLab +packages starting with GitLab 13.0. Older Redis versions don't support an +optional count argument to SPOP, which is required for [Merge Trains](../../ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md). Note the Redis node's IP address or hostname, port, and password (if required). -These will be necessary when configuring the -[GitLab application servers](#configure-gitlab-rails) later. +These will be necessary later when configuring the [GitLab application servers](#configure-gitlab-rails). ### Configure the Redis and Sentinel Cache cluster This is the section where we install and set up the new Redis Cache instances. -NOTE: **Note:** -Redis nodes (both primary and replica) will need the same password defined in -`redis['password']`. At any time during a failover the Sentinels can -reconfigure a node and change its status from primary to replica and vice versa. +Both the primary and replica Redis nodes need the same password defined in +`redis['password']`. At any time during a failover, the Sentinels can reconfigure +a node and change its status from primary to replica (and vice versa). #### Configure the primary Redis Cache node -1. SSH into the **Primary** Redis server. -1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab - package you want using **steps 1 and 2** from the GitLab downloads page. - - Make sure you select the correct Omnibus package, with the same version - and type (Community, Enterprise editions) of your current install. - - Do not complete any other steps on the download page. - +1. SSH in to the **Primary** Redis server. +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to both follow _only_ installation steps 1 and 2 + on the page, and to select the correct Omnibus GitLab package, with the same version + and type (Community or Enterprise editions) as your current install. 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby @@ -868,20 +860,17 @@ reconfigure a node and change its status from primary to replica and vice versa. 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. -NOTE: **Note:** -You can specify multiple roles like sentinel and Redis as: -`roles ['redis_sentinel_role', 'redis_master_role']`. -Read more about [roles](https://docs.gitlab.com/omnibus/roles/). +You can specify multiple roles, like sentinel and Redis, as: +`roles ['redis_sentinel_role', 'redis_master_role']`. Read more about +[roles](https://docs.gitlab.com/omnibus/roles/). #### Configure the replica Redis Cache nodes -1. SSH into the **replica** Redis server. -1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab - package you want using **steps 1 and 2** from the GitLab downloads page. - - Make sure you select the correct Omnibus package, with the same version - and type (Community, Enterprise editions) of your current install. - - Do not complete any other steps on the download page. - +1. SSH in to the **replica** Redis server. +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to both follow _only_ installation steps 1 and 2 + on the page, and to select the correct Omnibus GitLab package, with the same version + and type (Community or Enterprise editions) as your current install. 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby @@ -934,10 +923,9 @@ Read more about [roles](https://docs.gitlab.com/omnibus/roles/). 1. Go through the steps again for all the other replica nodes, and make sure to set up the IPs correctly. -NOTE: **Note:** -You can specify multiple roles like sentinel and Redis as: -`roles ['redis_sentinel_role', 'redis_master_role']`. -Read more about [roles](https://docs.gitlab.com/omnibus/roles/). +You can specify multiple roles, like sentinel and Redis, as: +`roles ['redis_sentinel_role', 'redis_master_role']`. Read more about +[roles](https://docs.gitlab.com/omnibus/roles/). These values don't have to be changed again in `/etc/gitlab/gitlab.rb` after a failover, as the nodes will be managed by the [Sentinels](#configure-the-sentinel-cache-nodes), and even after a @@ -955,13 +943,6 @@ are supported and can be added if needed. #### Configure the Sentinel Cache nodes -NOTE: **Note:** -If you are using an external Redis Sentinel instance, be sure -to exclude the `requirepass` parameter from the Sentinel -configuration. This parameter will cause clients to report `NOAUTH -Authentication required.`. [Redis Sentinel 3.2.x does not support -password authentication](https://github.com/antirez/redis/issues/3279). - Now that the Redis servers are all set up, let's configure the Sentinel servers. The following IPs will be used as an example: @@ -969,16 +950,19 @@ servers. The following IPs will be used as an example: - `10.6.0.72`: Sentinel - Cache 2 - `10.6.0.73`: Sentinel - Cache 3 -To configure the Sentinel Cache server: +NOTE: **Note:** +If you're using an external Redis Sentinel instance, be sure to exclude the +`requirepass` parameter from the Sentinel configuration. This parameter causes +clients to report `NOAUTH Authentication required.`. +[Redis Sentinel 3.2.x doesn't support password authentication](https://github.com/antirez/redis/issues/3279). -1. SSH into the server that will host Consul/Sentinel. -1. [Download/install](https://about.gitlab.com/install/) the - Omnibus GitLab Enterprise Edition package using **steps 1 and 2** from the - GitLab downloads page. - - Make sure you select the correct Omnibus package, with the same version - the GitLab application is running. - - Do not complete any other steps on the download page. +To configure the Sentinel Cache server: +1. SSH in to the server that will host Consul/Sentinel. +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to both follow _only_ installation steps 1 and 2 + on the page, and to select the correct Omnibus GitLab package, with the same version + and type (Community or Enterprise editions) as your current install. 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby @@ -1081,20 +1065,17 @@ To configure the Sentinel Cache server: This is the section where we install and set up the new Redis Queues instances. -NOTE: **Note:** -Redis nodes (both primary and replica) will need the same password defined in -`redis['password']`. At any time during a failover the Sentinels can -reconfigure a node and change its status from primary to replica and vice versa. +Both the primary and replica Redis nodes need the same password defined in +`redis['password']`. At any time during a failover, the Sentinels can reconfigure +a node and change its status from primary to replica (and vice versa). #### Configure the primary Redis Queues node -1. SSH into the **Primary** Redis server. -1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab - package you want using **steps 1 and 2** from the GitLab downloads page. - - Make sure you select the correct Omnibus package, with the same version - and type (Community, Enterprise editions) of your current install. - - Do not complete any other steps on the download page. - +1. SSH in to the **Primary** Redis server. +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to both follow _only_ installation steps 1 and 2 + on the page, and to select the correct Omnibus GitLab package, with the same version + and type (Community or Enterprise editions) as your current install. 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby @@ -1143,20 +1124,17 @@ reconfigure a node and change its status from primary to replica and vice versa. 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. -NOTE: **Note:** -You can specify multiple roles like sentinel and Redis as: -`roles ['redis_sentinel_role', 'redis_master_role']`. -Read more about [roles](https://docs.gitlab.com/omnibus/roles/). +You can specify multiple roles, like sentinel and Redis, as: +`roles ['redis_sentinel_role', 'redis_master_role']`. Read more about +[roles](https://docs.gitlab.com/omnibus/roles/). #### Configure the replica Redis Queues nodes -1. SSH into the **replica** Redis Queue server. -1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab - package you want using **steps 1 and 2** from the GitLab downloads page. - - Make sure you select the correct Omnibus package, with the same version - and type (Community, Enterprise editions) of your current install. - - Do not complete any other steps on the download page. - +1. SSH in to the **replica** Redis Queue server. +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to both follow _only_ installation steps 1 and 2 + on the page, and to select the correct Omnibus GitLab package, with the same version + and type (Community or Enterprise editions) as your current install. 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby @@ -1209,10 +1187,9 @@ Read more about [roles](https://docs.gitlab.com/omnibus/roles/). 1. Go through the steps again for all the other replica nodes, and make sure to set up the IPs correctly. -NOTE: **Note:** -You can specify multiple roles like sentinel and Redis as: -`roles ['redis_sentinel_role', 'redis_master_role']`. -Read more about [roles](https://docs.gitlab.com/omnibus/roles/). +You can specify multiple roles, like sentinel and Redis, as: +`roles ['redis_sentinel_role', 'redis_master_role']`. Read more about +[roles](https://docs.gitlab.com/omnibus/roles/). These values don't have to be changed again in `/etc/gitlab/gitlab.rb` after a failover, as the nodes will be managed by the [Sentinels](#configure-the-sentinel-queues-nodes), and even after a @@ -1230,13 +1207,6 @@ are supported and can be added if needed. #### Configure the Sentinel Queues nodes -NOTE: **Note:** -If you are using an external Redis Sentinel instance, be sure -to exclude the `requirepass` parameter from the Sentinel -configuration. This parameter will cause clients to report `NOAUTH -Authentication required.`. [Redis Sentinel 3.2.x does not support -password authentication](https://github.com/antirez/redis/issues/3279). - Now that the Redis servers are all set up, let's configure the Sentinel servers. The following IPs will be used as an example: @@ -1244,16 +1214,19 @@ servers. The following IPs will be used as an example: - `10.6.0.82`: Sentinel - Queues 2 - `10.6.0.83`: Sentinel - Queues 3 -To configure the Sentinel Queues server: +NOTE: **Note:** +If you're using an external Redis Sentinel instance, be sure to exclude the +`requirepass` parameter from the Sentinel configuration. This parameter causes +clients to report `NOAUTH Authentication required.`. +[Redis Sentinel 3.2.x doesn't support password authentication](https://github.com/antirez/redis/issues/3279). -1. SSH into the server that will host Sentinel. -1. [Download/install](https://about.gitlab.com/install/) the - Omnibus GitLab Enterprise Edition package using **steps 1 and 2** from the - GitLab downloads page. - - Make sure you select the correct Omnibus package, with the same version - the GitLab application is running. - - Do not complete any other steps on the download page. +To configure the Sentinel Queues server: +1. SSH in to the server that will host Sentinel. +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to both follow _only_ installation steps 1 and 2 + on the page, and to select the correct Omnibus GitLab package, with the same version + and type (Community or Enterprise editions) as your current install. 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby @@ -1362,52 +1335,48 @@ To configure the Sentinel Queues server: ## Configure Gitaly -Deploying Gitaly in its own server can benefit GitLab installations that are -larger than a single machine. - -The Gitaly node requirements are dependent on customer data, specifically the number of -projects and their repository sizes. Two nodes are recommended as an absolute minimum. -Each Gitaly node should store no more than 5TB of data and have the number of -[`gitaly-ruby` workers](../gitaly/index.md#gitaly-ruby) set to 20% of available CPUs. -Additional nodes should be considered in conjunction with a review of expected -data size and spread based on the recommendations above. - -It is also strongly recommended that all Gitaly nodes be set up with SSD disks with -a throughput of at least 8,000 IOPS for read operations and 2,000 IOPS for write, -as Gitaly has heavy I/O. These IOPS values are recommended only as a starter as with -time they may be adjusted higher or lower depending on the scale of your environment's workload. -If you're running the environment on a Cloud provider, you may need to refer to -their documentation on how to configure IOPS correctly. - -Some things to note: - -- The GitLab Rails application shards repositories into [repository storages](../repository_storage_paths.md). -- A Gitaly server can host one or more storages. -- A GitLab server can use one or more Gitaly servers. -- Gitaly addresses must be specified in such a way that they resolve - correctly for ALL Gitaly clients. +[Gitaly](../gitaly/index.md) server node requirements are dependent on data, +specifically the number of projects and those projects' sizes. It's recommended +that a Gitaly server node stores no more than 5 TB of data. Although this +reference architecture includes a recommendation for the number of Gitaly server +nodes to use, depending on your storage requirements, you may require additional +Gitaly server nodes. + +Due to Gitaly having notable input and output requirements, we strongly +recommend that all Gitaly nodes use solid-state drives (SSDs). These SSDs should +have a throughput of at least 8,000 input/output operations per second (IOPS) +for read operations and 2,000 IOPS for write operations. These IOPS values are +initial recommendations, and may be adjusted to greater or lesser values +depending on the scale of your environment's workload. If you're running the +environment on a Cloud provider, refer to their documentation about how to +configure IOPS correctly. + +Be sure to note the following items: + +- The GitLab Rails application shards repositories into + [repository storage paths](../repository_storage_paths.md). +- A Gitaly server can host one or more storage paths. +- A GitLab server can use one or more Gitaly server nodes. +- Gitaly addresses must be specified to be correctly resolvable for _all_ + Gitaly clients. - Gitaly servers must not be exposed to the public internet, as Gitaly's network traffic is unencrypted by default. The use of a firewall is highly recommended to restrict access to the Gitaly server. Another option is to [use TLS](#gitaly-tls-support). -TIP: **Tip:** -For more information about Gitaly's history and network architecture see the -[standalone Gitaly documentation](../gitaly/index.md). - -Note: **Note:** -The token referred to throughout the Gitaly documentation is -just an arbitrary password selected by the administrator. It is unrelated to -tokens created for the GitLab API or other similar web API tokens. +NOTE: **Note:** +The token referred to throughout the Gitaly documentation is an arbitrary +password selected by the administrator. This token is unrelated to tokens +created for the GitLab API or other similar web API tokens. -Below we describe how to configure two Gitaly servers, with IPs and -domain names: +This section describes how to configure two Gitaly servers, with the following +IPs and domain names: - `10.6.0.91`: Gitaly 1 (`gitaly1.internal`) - `10.6.0.92`: Gitaly 2 (`gitaly2.internal`) -The secret token is assumed to be `gitalysecret` and that -your GitLab installation has three repository storages: +Assumptions about your servers include having the secret token be `gitalysecret`, +and that your GitLab installation has three repository storages: - `default` on Gitaly 1 - `storage1` on Gitaly 1 @@ -1415,11 +1384,11 @@ your GitLab installation has three repository storages: On each node: -1. [Download/Install](https://about.gitlab.com/install/) the Omnibus GitLab - package you want using **steps 1 and 2** from the GitLab downloads page but - **without** providing the `EXTERNAL_URL` value. -1. Edit `/etc/gitlab/gitlab.rb` to configure storage paths, enable - the network listener and configure the token: +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to follow _only_ installation steps 1 and 2 + on the page, and _do not_ provide the `EXTERNAL_URL` value. +1. Edit `/etc/gitlab/gitlab.rb` to configure the storage paths, enable + the network listener, and configure the token: <!-- updates to following example must also be made at @@ -1467,39 +1436,39 @@ On each node: ``` 1. Append the following to `/etc/gitlab/gitlab.rb` for each respective server: - 1. On `gitaly1.internal`: - - ```ruby - git_data_dirs({ - 'default' => { - 'path' => '/var/opt/gitlab/git-data' - }, - 'storage1' => { - 'path' => '/mnt/gitlab/git-data' - }, - }) - ``` - - 1. On `gitaly2.internal`: - - ```ruby - git_data_dirs({ - 'storage2' => { - 'path' => '/mnt/gitlab/git-data' - }, - }) - ``` + - On `gitaly1.internal`: + + ```ruby + git_data_dirs({ + 'default' => { + 'path' => '/var/opt/gitlab/git-data' + }, + 'storage1' => { + 'path' => '/mnt/gitlab/git-data' + }, + }) + ``` + + - On `gitaly2.internal`: + + ```ruby + git_data_dirs({ + 'storage2' => { + 'path' => '/mnt/gitlab/git-data' + }, + }) + ``` <!-- updates to following example must also be made at https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab --> -1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace - the file of the same name on this server. If that file is not on this server, - add the file from your Consul server to this server. +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and + then replace the file of the same name on this server. If that file isn't on + this server, add the file from your Consul server to this server. -1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). +1. Save the file, and then [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). ### Gitaly TLS support @@ -1521,11 +1490,10 @@ Name. If you are addressing the Gitaly server by its IP address, you must add it as a Subject Alternative Name to the certificate. [gRPC does not support using an IP address as Common Name in a certificate](https://github.com/grpc/grpc/issues/2691). -NOTE: **Note:** -It is possible to configure Gitaly servers with both an -unencrypted listening address `listen_addr` and an encrypted listening -address `tls_listen_addr` at the same time. This allows you to do a -gradual transition from unencrypted to encrypted traffic, if necessary. +It's possible to configure Gitaly servers with both an unencrypted listening +address (`listen_addr`) and an encrypted listening address (`tls_listen_addr`) +at the same time. This allows you to do a gradual transition from unencrypted to +encrypted traffic, if necessary. To configure Gitaly with TLS: @@ -1580,10 +1548,10 @@ The following IPs will be used as an example: To configure the Sidekiq nodes, on each one: -1. SSH into the Sidekiq server. -1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab package -you want using steps 1 and 2 from the GitLab downloads page. -**Do not complete any other steps on the download page.** +1. SSH in to the Sidekiq server. +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to follow _only_ installation steps 1 and 2 + on the page. 1. Open `/etc/gitlab/gitlab.rb` with your editor: ```ruby @@ -1701,15 +1669,14 @@ You can also run [multiple Sidekiq processes](../operations/extra_sidekiq_proces ## Configure GitLab Rails -NOTE: **Note:** -In our architectures we run each GitLab Rails node using the Puma webserver -and have its number of workers set to 90% of available CPUs along with four threads. For -nodes that are running Rails with other components the worker value should be reduced -accordingly where we've found 50% achieves a good balance but this is dependent -on workload. - This section describes how to configure the GitLab application (Rails) component. +In our architecture, we run each GitLab Rails node using the Puma webserver, and +have its number of workers set to 90% of available CPUs, with four threads. For +nodes running Rails with other components, the worker value should be reduced +accordingly. We've determined that a worker value of 50% achieves a good balance, +but this is dependent on workload. + The following IPs will be used as an example: - `10.6.0.111`: GitLab application 1 @@ -1718,10 +1685,9 @@ The following IPs will be used as an example: On each node perform the following: -1. Download/install Omnibus GitLab using **steps 1 and 2** from - [GitLab downloads](https://about.gitlab.com/install/). Do not complete other - steps on the download page. - +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to follow _only_ installation steps 1 and 2 + on the page. 1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace the file of the same name on this server. If that file is not on this server, add the file from your Consul server to this server. @@ -1752,6 +1718,7 @@ On each node perform the following: roles ['application_role'] gitaly['enable'] = false nginx['enable'] = true + sidekiq['enable'] = false ## PostgreSQL connection details # Disable PostgreSQL on the application node @@ -1795,7 +1762,6 @@ On each node perform the following: # Set the network addresses that the exporters used for monitoring will listen on node_exporter['listen_address'] = '0.0.0.0:9100' gitlab_workhorse['prometheus_listen_addr'] = '0.0.0.0:9229' - sidekiq['listen_address'] = "0.0.0.0" puma['listen'] = '0.0.0.0' # Add the monitoring node's IP address to the monitoring whitelist and allow it to @@ -1836,7 +1802,7 @@ On each node perform the following: 1. Specify the necessary NFS mounts in `/etc/fstab`. The exact contents of `/etc/fstab` will depend on how you chose - to configure your NFS server. See the [NFS documentation](../high_availability/nfs.md) + to configure your NFS server. See the [NFS documentation](../nfs.md) for examples and the various options. 1. Create the shared directories. These may be different depending on your NFS @@ -1877,30 +1843,31 @@ On each node perform the following: 1. Optionally, from the Gitaly servers, confirm that Gitaly can perform callbacks to the internal API: ```shell - sudo /opt/gitlab/embedded/service/gitlab-shell/bin/check -config /opt/gitlab/embedded/service/gitlab-shell/config.yml + sudo /opt/gitlab/embedded/bin/gitaly-hooks check /var/opt/gitlab/gitaly/config.toml ``` -NOTE: **Note:** -When you specify `https` in the `external_url`, as in the example -above, GitLab assumes you have SSL certificates in `/etc/gitlab/ssl/`. If -certificates are not present, NGINX will fail to start. See the -[NGINX documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https) -for more information. +When you specify `https` in the `external_url`, as in the previous example, +GitLab expects that the SSL certificates are in `/etc/gitlab/ssl/`. If the +certificates aren't present, NGINX will fail to start. For more information, see +the [NGINX documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https). ### GitLab Rails post-configuration -Initialize the GitLab database, by running the following in one of the Rails nodes: +1. Designate one application node for running database migrations during + installation and updates. Initialize the GitLab database and ensure all + migrations ran: -```shell -sudo gitlab-rake gitlab:db:configure -``` + ```shell + sudo gitlab-rake gitlab:db:configure + ``` -NOTE: **Note:** -If you encounter a `rake aborted!` error stating that PgBouncer is failing to connect to -PostgreSQL it may be that your PgBouncer node's IP address is missing from -PostgreSQL's `trust_auth_cidr_addresses` in `gitlab.rb` on your database nodes. See -[PgBouncer error `ERROR: pgbouncer cannot connect to server`](troubleshooting.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server) -in the Troubleshooting section before proceeding. + If you encounter a `rake aborted!` error message stating that PgBouncer is + failing to connect to PostgreSQL, it may be that your PgBouncer node's IP + address is missing from PostgreSQL's `trust_auth_cidr_addresses` in `gitlab.rb` + on your database nodes. Before proceeding, see + [PgBouncer error `ERROR: pgbouncer cannot connect to server`](troubleshooting.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server). + +1. [Configure fast lookup of authorized SSH keys in the database](../operations/fast_ssh_key_lookup.md). <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> @@ -1920,11 +1887,10 @@ The following IP will be used as an example: To configure the Monitoring node: -1. SSH into the Monitoring node. -1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab - package you want using **steps 1 and 2** from the GitLab downloads page. - Do not complete any other steps on the download page. - +1. SSH in to the Monitoring node. +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to follow _only_ installation steps 1 and 2 + on the page. 1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace the file of the same name on this server. If that file is not on this server, add the file from your Consul server to this server. @@ -1988,28 +1954,44 @@ It's recommended over [NFS](#configure-nfs-optional) and in general it's better in larger setups as object storage is typically much more performant, reliable, and scalable. -Object storage options that GitLab has tested, or is aware of customers using include: +GitLab has been tested on a number of object storage providers: -- SaaS/Cloud solutions such as [Amazon S3](https://aws.amazon.com/s3/), [Google cloud storage](https://cloud.google.com/storage). +- [Amazon S3](https://aws.amazon.com/s3/) +- [Google Cloud Storage](https://cloud.google.com/storage) +- [Digital Ocean Spaces](https://www.digitalocean.com/products/spaces/) +- [Oracle Cloud Infrastructure](https://docs.cloud.oracle.com/en-us/iaas/Content/Object/Tasks/s3compatibleapi.htm) +- [Openstack Swift](https://docs.openstack.org/swift/latest/s3_compat.html) +- [Azure Blob storage](https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blobs-introduction) - On-premises hardware and appliances from various storage vendors. -- MinIO. There is [a guide to deploying this](https://docs.gitlab.com/charts/advanced/external-object-storage/minio.html) within our Helm Chart documentation. - -For configuring GitLab to use Object Storage refer to the following guides -based on what features you intend to use: - -1. Configure [object storage for backups](../../raketasks/backup_restore.md#uploading-backups-to-a-remote-cloud-storage). -1. Configure [object storage for job artifacts](../job_artifacts.md#using-object-storage) - including [incremental logging](../job_logs.md#new-incremental-logging-architecture). -1. Configure [object storage for LFS objects](../lfs/index.md#storing-lfs-objects-in-remote-object-storage). -1. Configure [object storage for uploads](../uploads.md#using-object-storage). -1. Configure [object storage for merge request diffs](../merge_request_diffs.md#using-object-storage). -1. Configure [object storage for Container Registry](../packages/container_registry.md#use-object-storage) (optional feature). -1. Configure [object storage for Mattermost](https://docs.mattermost.com/administration/config-settings.html#file-storage) (optional feature). -1. Configure [object storage for packages](../packages/index.md#using-object-storage) (optional feature). **(PREMIUM ONLY)** -1. Configure [object storage for Dependency Proxy](../packages/dependency_proxy.md#using-object-storage) (optional feature). **(PREMIUM ONLY)** -1. Configure [object storage for Pseudonymizer](../pseudonymizer.md#configuration) (optional feature). **(ULTIMATE ONLY)** -1. Configure [object storage for autoscale runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional - for improved performance). -1. Configure [object storage for Terraform state files](../terraform_state.md#using-object-storage). +- MinIO. We have [a guide to deploying this](https://docs.gitlab.com/charts/advanced/external-object-storage/minio.html) within our Helm Chart documentation. + +There are two ways of specifying object storage configuration in GitLab: + +- [Consolidated form](../object_storage.md#consolidated-object-storage-configuration): A single credential is + shared by all supported object types. +- [Storage-specific form](../object_storage.md#storage-specific-configuration): Every object defines its + own object storage [connection and configuration](../object_storage.md#connection-settings). + +Starting with GitLab 13.2, consolidated object storage configuration is available. It simplifies your GitLab configuration since the connection details are shared across object types. Refer to [Consolidated object storage configuration](../object_storage.md#consolidated-object-storage-configuration) guide for instructions on how to set it up. + +For configuring object storage in GitLab 13.1 and earlier, or for storage types not +supported by consolidated configuration form, refer to the following guides based +on what features you intend to use: + +|Object storage type|Supported by consolidated configuration?| +|-------------------|----------------------------------------| +| [Backups](../../raketasks/backup_restore.md#uploading-backups-to-a-remote-cloud-storage)|No| +| [Job artifacts](../job_artifacts.md#using-object-storage) including archived job logs | Yes | +| [LFS objects](../lfs/index.md#storing-lfs-objects-in-remote-object-storage) | Yes | +| [Uploads](../uploads.md#using-object-storage) | Yes | +| [Container Registry](../packages/container_registry.md#use-object-storage) (optional feature) | No | +| [Merge request diffs](../merge_request_diffs.md#using-object-storage) | Yes | +| [Mattermost](https://docs.mattermost.com/administration/config-settings.html#file-storage)| No | +| [Packages](../packages/index.md#using-object-storage) (optional feature) | Yes | +| [Dependency Proxy](../packages/dependency_proxy.md#using-object-storage) (optional feature) **(PREMIUM ONLY)** | Yes | +| [Pseudonymizer](../pseudonymizer.md#configuration) (optional feature) **(ULTIMATE ONLY)** | No | +| [Autoscale runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional for improved performance) | No | +| [Terraform state files](../terraform_state.md#using-object-storage) | Yes | Using separate buckets for each data type is the recommended approach for GitLab. @@ -2033,13 +2015,29 @@ work. </a> </div> +## Configure Advanced Search **(STARTER ONLY)** + +You can leverage Elasticsearch and [enable Advanced Search](../../integration/elasticsearch.md) +for faster, more advanced code search across your entire GitLab instance. + +Elasticsearch cluster design and requirements are dependent on your specific +data. For recommended best practices about how to set up your Elasticsearch +cluster alongside your instance, read how to +[choose the optimal cluster configuration](../../integration/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration). + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + ## Configure NFS (optional) [Object storage](#configure-the-object-storage), along with [Gitaly](#configure-gitaly) are recommended over NFS wherever possible for improved performance. If you intend to use GitLab Pages, this currently [requires NFS](troubleshooting.md#gitlab-pages-requires-nfs). -See how to [configure NFS](../high_availability/nfs.md). +See how to [configure NFS](../nfs.md). <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> diff --git a/doc/administration/reference_architectures/1k_users.md b/doc/administration/reference_architectures/1k_users.md index d376c1b7575..0cb7b8868c3 100644 --- a/doc/administration/reference_architectures/1k_users.md +++ b/doc/administration/reference_architectures/1k_users.md @@ -20,8 +20,8 @@ many organizations . | Users | Configuration | GCP | AWS | Azure | |--------------|-------------------------|----------------|-----------------|----------------| -| Up to 500 | 4 vCPU, 3.6GB memory | n1-highcpu-4 | c5.xlarge | F4s v2 | -| Up to 1,000 | 8 vCPU, 7.2GB memory | n1-highcpu-8 | c5.2xlarge | F8s v2 | +| Up to 500 | 4 vCPU, 3.6 GB memory | n1-highcpu-4 | c5.xlarge | F4s v2 | +| Up to 1,000 | 8 vCPU, 7.2 GB memory | n1-highcpu-8 | c5.2xlarge | F8s v2 | The Google Cloud Platform (GCP) architectures were built and tested using the [Intel Xeon E5 v3 (Haswell)](https://cloud.google.com/compute/docs/cpu-platforms) @@ -30,20 +30,28 @@ or higher, are required for your CPU or node counts. For more information, see our [Sysbench](https://github.com/akopytov/sysbench)-based [CPU benchmark](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Reference-Architectures/GCP-CPU-Benchmarks). -In addition to the stated configurations, we recommend having at least 2GB of +In addition to the stated configurations, we recommend having at least 2 GB of swap on your server, even if you currently have enough available memory. Having -swap will help reduce the chance of errors occurring if your available memory +swap helps to reduce the chance of errors occurring if your available memory changes. We also recommend configuring the kernel's swappiness setting to a lower value (such as `10`) to make the most of your memory, while still having the swap available when needed. ## Setup instructions -For this default reference architecture, to install GitLab use the standard +To install GitLab for this default reference architecture, use the standard [installation instructions](../../install/README.md). -NOTE: **Note:** -You can also optionally configure GitLab to use an -[external PostgreSQL service](../postgresql/external.md) or an -[external object storage service](../high_availability/object_storage.md) for -added performance and reliability at a reduced complexity cost. +You can also optionally configure GitLab to use an [external PostgreSQL service](../postgresql/external.md) +or an [external object storage service](../object_storage.md) for added +performance and reliability at a reduced complexity cost. + +## Configure Advanced Search **(STARTER ONLY)** + +You can leverage Elasticsearch and [enable Advanced Search](../../integration/elasticsearch.md) +for faster, more advanced code search across your entire GitLab instance. + +Elasticsearch cluster design and requirements are dependent on your specific +data. For recommended best practices about how to set up your Elasticsearch +cluster alongside your instance, read how to +[choose the optimal cluster configuration](../../integration/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration). diff --git a/doc/administration/reference_architectures/25k_users.md b/doc/administration/reference_architectures/25k_users.md index 2ef555bff29..bb6e2eb4376 100644 --- a/doc/administration/reference_architectures/25k_users.md +++ b/doc/administration/reference_architectures/25k_users.md @@ -17,21 +17,21 @@ full list of reference architectures, see | Service | Nodes | Configuration | GCP | AWS | Azure | |-----------------------------------------|-------------|-------------------------|-----------------|-------------|----------| -| External load balancing node | 1 | 4 vCPU, 3.6GB memory | n1-highcpu-4 | c5.xlarge | F4s v2 | -| Consul | 3 | 2 vCPU, 1.8GB memory | n1-highcpu-2 | c5.large | F2s v2 | -| PostgreSQL | 3 | 8 vCPU, 30GB memory | n1-standard-8 | m5.2xlarge | D8s v3 | -| PgBouncer | 3 | 2 vCPU, 1.8GB memory | n1-highcpu-2 | c5.large | F2s v2 | -| Internal load balancing node | 1 | 2 vCPU, 1.8GB memory | n1-highcpu-2 | c5.large | F2s v2 | -| Redis - Cache | 3 | 4 vCPU, 15GB memory | n1-standard-4 | m5.xlarge | D4s v3 | -| Redis - Queues / Shared State | 3 | 4 vCPU, 15GB memory | n1-standard-4 | m5.xlarge | D4s v3 | -| Redis Sentinel - Cache | 3 | 1 vCPU, 1.7GB memory | g1-small | t2.small | B1MS | -| Redis Sentinel - Queues / Shared State | 3 | 1 vCPU, 1.7GB memory | g1-small | t2.small | B1MS | -| Gitaly | 2 (minimum) | 32 vCPU, 120GB memory | n1-standard-32 | m5.8xlarge | D32s v3 | -| Sidekiq | 4 | 4 vCPU, 15GB memory | n1-standard-4 | m5.xlarge | D4s v3 | -| GitLab Rails | 5 | 32 vCPU, 28.8GB memory | n1-highcpu-32 | c5.9xlarge | F32s v2 | -| Monitoring node | 1 | 4 vCPU, 3.6GB memory | n1-highcpu-4 | c5.xlarge | F4s v2 | -| Object Storage | n/a | n/a | n/a | n/a | n/a | -| NFS Server | 1 | 4 vCPU, 3.6GB memory | n1-highcpu-4 | c5.xlarge | F4s v2 | +| External load balancing node | 1 | 4 vCPU, 3.6 GB memory | n1-highcpu-4 | c5.xlarge | F4s v2 | +| Consul | 3 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| PostgreSQL | 3 | 8 vCPU, 30 GB memory | n1-standard-8 | m5.2xlarge | D8s v3 | +| PgBouncer | 3 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| Internal load balancing node | 1 | 4 vCPU, 3.6GB memory | n1-highcpu-4 | c5.large | F2s v2 | +| Redis - Cache | 3 | 4 vCPU, 15 GB memory | n1-standard-4 | m5.xlarge | D4s v3 | +| Redis - Queues / Shared State | 3 | 4 vCPU, 15 GB memory | n1-standard-4 | m5.xlarge | D4s v3 | +| Redis Sentinel - Cache | 3 | 1 vCPU, 1.7 GB memory | g1-small | t2.small | B1MS | +| Redis Sentinel - Queues / Shared State | 3 | 1 vCPU, 1.7 GB memory | g1-small | t2.small | B1MS | +| Gitaly | 2 (minimum) | 32 vCPU, 120 GB memory | n1-standard-32 | m5.8xlarge | D32s v3 | +| Sidekiq | 4 | 4 vCPU, 15 GB memory | n1-standard-4 | m5.xlarge | D4s v3 | +| GitLab Rails | 5 | 32 vCPU, 28.8 GB memory | n1-highcpu-32 | c5.9xlarge | F32s v2 | +| Monitoring node | 1 | 4 vCPU, 3.6 GB memory | n1-highcpu-4 | c5.xlarge | F4s v2 | +| Object storage | n/a | n/a | n/a | n/a | n/a | +| NFS server | 1 | 4 vCPU, 3.6 GB memory | n1-highcpu-4 | c5.xlarge | F4s v2 | The Google Cloud Platform (GCP) architectures were built and tested using the [Intel Xeon E5 v3 (Haswell)](https://cloud.google.com/compute/docs/cpu-platforms) @@ -40,41 +40,43 @@ or higher, are required for your CPU or node counts. For more information, see our [Sysbench](https://github.com/akopytov/sysbench)-based [CPU benchmark](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Reference-Architectures/GCP-CPU-Benchmarks). -For data objects (such as LFS, Uploads, or Artifacts), an -[object storage service](#configure-the-object-storage) is recommended instead -of NFS where possible, due to better performance and availability. Since this -doesn't require a node to be set up, *Object Storage* is noted as not -applicable (n/a) in the previous table. +Due to better performance and availability, for data objects (such as LFS, +uploads, or artifacts), using an [object storage service](#configure-the-object-storage) +is recommended instead of using NFS. Using an object storage service also +doesn't require you to provision and maintain a node. ## Setup components To set up GitLab and its components to accommodate up to 25,000 users: 1. [Configure the external load balancing node](#configure-the-external-load-balancer) - that will handle the load balancing of the three GitLab application services nodes. + to handle the load balancing of the GitLab application services nodes. 1. [Configure Consul](#configure-consul). 1. [Configure PostgreSQL](#configure-postgresql), the database for GitLab. 1. [Configure PgBouncer](#configure-pgbouncer). -1. [Configure the internal load balancing node](#configure-the-internal-load-balancer) +1. [Configure the internal load balancing node](#configure-the-internal-load-balancer). 1. [Configure Redis](#configure-redis). 1. [Configure Gitaly](#configure-gitaly), which provides access to the Git repositories. 1. [Configure Sidekiq](#configure-sidekiq). 1. [Configure the main GitLab Rails application](#configure-gitlab-rails) - to run Puma/Unicorn, Workhorse, GitLab Shell, and to serve all frontend requests (UI, API, Git - over HTTP/SSH). -1. [Configure Prometheus](#configure-prometheus) to monitor your GitLab environment. -1. [Configure the Object Storage](#configure-the-object-storage) + to run Puma/Unicorn, Workhorse, GitLab Shell, and to serve all frontend + requests (which include UI, API, and Git over HTTP/SSH). +1. [Configure Prometheus](#configure-prometheus) to monitor your GitLab + environment. +1. [Configure the object storage](#configure-the-object-storage) used for shared data objects. -1. [Configure NFS (Optional)](#configure-nfs-optional) - to have shared disk storage service as an alternative to Gitaly and/or Object Storage (although - not recommended). NFS is required for GitLab Pages, you can skip this step if you're not using - that feature. +1. [Configure Advanced Search](#configure-advanced-search) (optional) for faster, + more advanced code search across your entire GitLab instance. +1. [Configure NFS](#configure-nfs-optional) (optional, and not recommended) + to have shared disk storage service as an alternative to Gitaly or object + storage. You can skip this step if you're not using GitLab Pages (which + requires NFS). -We start with all servers on the same 10.6.0.0/24 private network range, they -can connect to each other freely on those addresses. +The servers start on the same 10.6.0.0/24 private network range, and can +connect to each other freely on these addresses. -Here is a list and description of each machine and the assigned IP: +The following list includes descriptions of each server and its assigned IP: - `10.6.0.10`: External Load Balancer - `10.6.0.11`: Consul 1 @@ -112,19 +114,18 @@ Here is a list and description of each machine and the assigned IP: ## Configure the external load balancer -NOTE: **Note:** +In an active/active GitLab configuration, you'll need a load balancer to route +traffic to the application servers. The specifics on which load balancer to use +or its exact configuration is beyond the scope of GitLab documentation. We hope +that if you're managing multi-node systems like GitLab, you already have a load +balancer of choice. Some load balancer examples include HAProxy (open-source), +F5 Big-IP LTM, and Citrix Net Scaler. This documentation outline the ports and +protocols needed for use with GitLab. + This architecture has been tested and validated with [HAProxy](https://www.haproxy.org/) as the load balancer. Although other load balancers with similar feature sets could also be used, those load balancers have not been validated. -In an active/active GitLab configuration, you will need a load balancer to route -traffic to the application servers. The specifics on which load balancer to use -or the exact configuration is beyond the scope of GitLab documentation. We hope -that if you're managing multi-node systems like GitLab you have a load balancer of -choice already. Some examples including HAProxy (open-source), F5 Big-IP LTM, -and Citrix Net Scaler. This documentation will outline what ports and protocols -you need to use with GitLab. - The next question is how you will handle SSL in your environment. There are several different options: @@ -247,14 +248,11 @@ with the other servers. To configure Consul: -1. SSH into the server that will host Consul. -1. [Download/install](https://about.gitlab.com/install/) the - Omnibus GitLab Enterprise Edition package using **steps 1 and 2** from the - GitLab downloads page. - - Make sure you select the correct Omnibus package, with the same version - the GitLab application is running. - - Do not complete any other steps on the download page. - +1. SSH in to the server that will host Consul. +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to both follow _only_ installation steps 1 and 2 + on the page, and to select the correct Omnibus GitLab package, with the same version + and type (Community or Enterprise editions) as your current install. 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby @@ -282,10 +280,9 @@ To configure Consul: 1. Go through the steps again for all the other Consul nodes, and make sure you set up the correct IPs. -NOTE: **Note:** -A Consul leader will be elected when the provisioning of the third Consul server is completed. -Viewing the Consul logs `sudo gitlab-ctl tail consul` will display -`...[INFO] consul: New leader elected: ...` +A Consul leader is _elected_ when the provisioning of the third Consul server is +complete. Viewing the Consul logs `sudo gitlab-ctl tail consul` displays +`...[INFO] consul: New leader elected: ...`. You can list the current Consul members (server, client): @@ -352,7 +349,7 @@ in the second step, do not supply the `EXTERNAL_URL` value. #### PostgreSQL primary node -1. SSH into the PostgreSQL primary node. +1. SSH in to the PostgreSQL primary node. 1. Generate a password hash for the PostgreSQL username/password pair. This assumes you will use the default username of `gitlab` (recommended). The command will request a password and confirmation. Use the value that is output by this command in the next @@ -513,7 +510,7 @@ are supported and can be added if needed. #### PostgreSQL post-configuration -SSH into the **primary node**: +SSH in to the **primary node**: 1. Open a database prompt: @@ -548,7 +545,7 @@ SSH into the **primary node**: is not an IP address, it will need to be a resolvable name (via DNS or `/etc/hosts`) -SSH into the **secondary node**: +SSH in to the **secondary node**: 1. Set up the repmgr standby: @@ -662,7 +659,6 @@ The following IPs will be used as an example: 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. - NOTE: **Note:** If an error `execute[generate databases.ini]` occurs, this is due to an existing [known issue](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/4713). It will be resolved when you run a second `reconfigure` after the next step. @@ -796,35 +792,31 @@ to be used with GitLab. The following IPs will be used as an example: - `10.6.0.82`: Sentinel - Queues 2 - `10.6.0.83`: Sentinel - Queues 3 -NOTE: **Providing your own Redis instance:** -Managed Redis from cloud providers such as AWS ElastiCache will work. If these -services support high availability, be sure it is **not** the Redis Cluster type. -Redis version 5.0 or higher is required, as this is what ships with -Omnibus GitLab packages starting with GitLab 13.0. Older Redis versions -do not support an optional count argument to SPOP which is now required for -[Merge Trains](../../ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md). +### Providing your own Redis instance + +Managed Redis from cloud providers (such as AWS ElastiCache) will work. If these +services support high availability, be sure it _isn't_ of the Redis Cluster type. +Redis version 5.0 or higher is required, which is included with Omnibus GitLab +packages starting with GitLab 13.0. Older Redis versions don't support an +optional count argument to SPOP, which is required for [Merge Trains](../../ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md). Note the Redis node's IP address or hostname, port, and password (if required). -These will be necessary when configuring the -[GitLab application servers](#configure-gitlab-rails) later. +These will be necessary later when configuring the [GitLab application servers](#configure-gitlab-rails). ### Configure the Redis and Sentinel Cache cluster This is the section where we install and set up the new Redis Cache instances. -NOTE: **Note:** -Redis nodes (both primary and replica) will need the same password defined in -`redis['password']`. At any time during a failover the Sentinels can -reconfigure a node and change its status from primary to replica and vice versa. +Both the primary and replica Redis nodes need the same password defined in +`redis['password']`. At any time during a failover, the Sentinels can reconfigure +a node and change its status from primary to replica (and vice versa). #### Configure the primary Redis Cache node -1. SSH into the **Primary** Redis server. -1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab - package you want using **steps 1 and 2** from the GitLab downloads page. - - Make sure you select the correct Omnibus package, with the same version - and type (Community, Enterprise editions) of your current install. - - Do not complete any other steps on the download page. - +1. SSH in to the **Primary** Redis server. +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to both follow _only_ installation steps 1 and 2 + on the page, and to select the correct Omnibus GitLab package, with the same version + and type (Community or Enterprise editions) as your current install. 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby @@ -868,20 +860,17 @@ reconfigure a node and change its status from primary to replica and vice versa. 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. -NOTE: **Note:** -You can specify multiple roles like sentinel and Redis as: -`roles ['redis_sentinel_role', 'redis_master_role']`. -Read more about [roles](https://docs.gitlab.com/omnibus/roles/). +You can specify multiple roles, like sentinel and Redis, as: +`roles ['redis_sentinel_role', 'redis_master_role']`. Read more about +[roles](https://docs.gitlab.com/omnibus/roles/). #### Configure the replica Redis Cache nodes -1. SSH into the **replica** Redis server. -1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab - package you want using **steps 1 and 2** from the GitLab downloads page. - - Make sure you select the correct Omnibus package, with the same version - and type (Community, Enterprise editions) of your current install. - - Do not complete any other steps on the download page. - +1. SSH in to the **replica** Redis server. +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to both follow _only_ installation steps 1 and 2 + on the page, and to select the correct Omnibus GitLab package, with the same version + and type (Community or Enterprise editions) as your current install. 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby @@ -934,10 +923,9 @@ Read more about [roles](https://docs.gitlab.com/omnibus/roles/). 1. Go through the steps again for all the other replica nodes, and make sure to set up the IPs correctly. -NOTE: **Note:** -You can specify multiple roles like sentinel and Redis as: -`roles ['redis_sentinel_role', 'redis_master_role']`. -Read more about [roles](https://docs.gitlab.com/omnibus/roles/). +You can specify multiple roles, like sentinel and Redis, as: +`roles ['redis_sentinel_role', 'redis_master_role']`. Read more about +[roles](https://docs.gitlab.com/omnibus/roles/). These values don't have to be changed again in `/etc/gitlab/gitlab.rb` after a failover, as the nodes will be managed by the [Sentinels](#configure-the-sentinel-cache-nodes), and even after a @@ -955,13 +943,6 @@ are supported and can be added if needed. #### Configure the Sentinel Cache nodes -NOTE: **Note:** -If you are using an external Redis Sentinel instance, be sure -to exclude the `requirepass` parameter from the Sentinel -configuration. This parameter will cause clients to report `NOAUTH -Authentication required.`. [Redis Sentinel 3.2.x does not support -password authentication](https://github.com/antirez/redis/issues/3279). - Now that the Redis servers are all set up, let's configure the Sentinel servers. The following IPs will be used as an example: @@ -969,16 +950,19 @@ servers. The following IPs will be used as an example: - `10.6.0.72`: Sentinel - Cache 2 - `10.6.0.73`: Sentinel - Cache 3 -To configure the Sentinel Cache server: +NOTE: **Note:** +If you're using an external Redis Sentinel instance, be sure to exclude the +`requirepass` parameter from the Sentinel configuration. This parameter causes +clients to report `NOAUTH Authentication required.`. +[Redis Sentinel 3.2.x doesn't support password authentication](https://github.com/antirez/redis/issues/3279). -1. SSH into the server that will host Consul/Sentinel. -1. [Download/install](https://about.gitlab.com/install/) the - Omnibus GitLab Enterprise Edition package using **steps 1 and 2** from the - GitLab downloads page. - - Make sure you select the correct Omnibus package, with the same version - the GitLab application is running. - - Do not complete any other steps on the download page. +To configure the Sentinel Cache server: +1. SSH in to the server that will host Consul/Sentinel. +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to both follow _only_ installation steps 1 and 2 + on the page, and to select the correct Omnibus GitLab package, with the same version + and type (Community or Enterprise editions) as your current install. 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby @@ -1081,20 +1065,17 @@ To configure the Sentinel Cache server: This is the section where we install and set up the new Redis Queues instances. -NOTE: **Note:** -Redis nodes (both primary and replica) will need the same password defined in -`redis['password']`. At any time during a failover the Sentinels can -reconfigure a node and change its status from primary to replica and vice versa. +Both the primary and replica Redis nodes need the same password defined in +`redis['password']`. At any time during a failover, the Sentinels can reconfigure +a node and change its status from primary to replica (and vice versa). #### Configure the primary Redis Queues node -1. SSH into the **Primary** Redis server. -1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab - package you want using **steps 1 and 2** from the GitLab downloads page. - - Make sure you select the correct Omnibus package, with the same version - and type (Community, Enterprise editions) of your current install. - - Do not complete any other steps on the download page. - +1. SSH in to the **Primary** Redis server. +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to both follow _only_ installation steps 1 and 2 + on the page, and to select the correct Omnibus GitLab package, with the same version + and type (Community or Enterprise editions) as your current install. 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby @@ -1143,20 +1124,17 @@ reconfigure a node and change its status from primary to replica and vice versa. 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. -NOTE: **Note:** -You can specify multiple roles like sentinel and Redis as: -`roles ['redis_sentinel_role', 'redis_master_role']`. -Read more about [roles](https://docs.gitlab.com/omnibus/roles/). +You can specify multiple roles, like sentinel and Redis, as: +`roles ['redis_sentinel_role', 'redis_master_role']`. Read more about +[roles](https://docs.gitlab.com/omnibus/roles/). #### Configure the replica Redis Queues nodes -1. SSH into the **replica** Redis Queue server. -1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab - package you want using **steps 1 and 2** from the GitLab downloads page. - - Make sure you select the correct Omnibus package, with the same version - and type (Community, Enterprise editions) of your current install. - - Do not complete any other steps on the download page. - +1. SSH in to the **replica** Redis Queue server. +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to both follow _only_ installation steps 1 and 2 + on the page, and to select the correct Omnibus GitLab package, with the same version + and type (Community or Enterprise editions) as your current install. 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby @@ -1209,10 +1187,9 @@ Read more about [roles](https://docs.gitlab.com/omnibus/roles/). 1. Go through the steps again for all the other replica nodes, and make sure to set up the IPs correctly. -NOTE: **Note:** -You can specify multiple roles like sentinel and Redis as: -`roles ['redis_sentinel_role', 'redis_master_role']`. -Read more about [roles](https://docs.gitlab.com/omnibus/roles/). +You can specify multiple roles, like sentinel and Redis, as: +`roles ['redis_sentinel_role', 'redis_master_role']`. Read more about +[roles](https://docs.gitlab.com/omnibus/roles/). These values don't have to be changed again in `/etc/gitlab/gitlab.rb` after a failover, as the nodes will be managed by the [Sentinels](#configure-the-sentinel-queues-nodes), and even after a @@ -1230,13 +1207,6 @@ are supported and can be added if needed. #### Configure the Sentinel Queues nodes -NOTE: **Note:** -If you are using an external Redis Sentinel instance, be sure -to exclude the `requirepass` parameter from the Sentinel -configuration. This parameter will cause clients to report `NOAUTH -Authentication required.`. [Redis Sentinel 3.2.x does not support -password authentication](https://github.com/antirez/redis/issues/3279). - Now that the Redis servers are all set up, let's configure the Sentinel servers. The following IPs will be used as an example: @@ -1244,16 +1214,19 @@ servers. The following IPs will be used as an example: - `10.6.0.82`: Sentinel - Queues 2 - `10.6.0.83`: Sentinel - Queues 3 -To configure the Sentinel Queues server: +NOTE: **Note:** +If you're using an external Redis Sentinel instance, be sure to exclude the +`requirepass` parameter from the Sentinel configuration. This parameter causes +clients to report `NOAUTH Authentication required.`. +[Redis Sentinel 3.2.x doesn't support password authentication](https://github.com/antirez/redis/issues/3279). -1. SSH into the server that will host Sentinel. -1. [Download/install](https://about.gitlab.com/install/) the - Omnibus GitLab Enterprise Edition package using **steps 1 and 2** from the - GitLab downloads page. - - Make sure you select the correct Omnibus package, with the same version - the GitLab application is running. - - Do not complete any other steps on the download page. +To configure the Sentinel Queues server: +1. SSH in to the server that will host Sentinel. +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to both follow _only_ installation steps 1 and 2 + on the page, and to select the correct Omnibus package, with the same version + and type (Community or Enterprise editions) as your current install. 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby @@ -1362,52 +1335,48 @@ To configure the Sentinel Queues server: ## Configure Gitaly -Deploying Gitaly in its own server can benefit GitLab installations that are -larger than a single machine. - -The Gitaly node requirements are dependent on customer data, specifically the number of -projects and their repository sizes. Two nodes are recommended as an absolute minimum. -Each Gitaly node should store no more than 5TB of data and have the number of -[`gitaly-ruby` workers](../gitaly/index.md#gitaly-ruby) set to 20% of available CPUs. -Additional nodes should be considered in conjunction with a review of expected -data size and spread based on the recommendations above. - -It is also strongly recommended that all Gitaly nodes be set up with SSD disks with -a throughput of at least 8,000 IOPS for read operations and 2,000 IOPS for write, -as Gitaly has heavy I/O. These IOPS values are recommended only as a starter as with -time they may be adjusted higher or lower depending on the scale of your environment's workload. -If you're running the environment on a Cloud provider, you may need to refer to -their documentation on how to configure IOPS correctly. - -Some things to note: - -- The GitLab Rails application shards repositories into [repository storages](../repository_storage_paths.md). -- A Gitaly server can host one or more storages. -- A GitLab server can use one or more Gitaly servers. -- Gitaly addresses must be specified in such a way that they resolve - correctly for ALL Gitaly clients. +[Gitaly](../gitaly/index.md) server node requirements are dependent on data, +specifically the number of projects and those projects' sizes. It's recommended +that a Gitaly server node stores no more than 5 TB of data. Although this +reference architecture includes a recommendation for the number of Gitaly server +nodes to use, depending on your storage requirements, you may require additional +Gitaly server nodes. + +Due to Gitaly having notable input and output requirements, we strongly +recommend that all Gitaly nodes use solid-state drives (SSDs). These SSDs should +have a throughput of at least 8,000 input/output operations per second (IOPS) +for read operations and 2,000 IOPS for write operations. These IOPS values are +initial recommendations, and may be adjusted to greater or lesser values +depending on the scale of your environment's workload. If you're running the +environment on a Cloud provider, refer to their documentation about how to +configure IOPS correctly. + +Be sure to note the following items: + +- The GitLab Rails application shards repositories into + [repository storage paths](../repository_storage_paths.md). +- A Gitaly server can host one or more storage paths. +- A GitLab server can use one or more Gitaly server nodes. +- Gitaly addresses must be specified to be correctly resolvable for _all_ + Gitaly clients. - Gitaly servers must not be exposed to the public internet, as Gitaly's network traffic is unencrypted by default. The use of a firewall is highly recommended to restrict access to the Gitaly server. Another option is to [use TLS](#gitaly-tls-support). -TIP: **Tip:** -For more information about Gitaly's history and network architecture see the -[standalone Gitaly documentation](../gitaly/index.md). - -Note: **Note:** -The token referred to throughout the Gitaly documentation is -just an arbitrary password selected by the administrator. It is unrelated to -tokens created for the GitLab API or other similar web API tokens. +NOTE: **Note:** +The token referred to throughout the Gitaly documentation is an arbitrary +password selected by the administrator. This token is unrelated to tokens +created for the GitLab API or other similar web API tokens. -Below we describe how to configure two Gitaly servers, with IPs and -domain names: +This section describes how to configure two Gitaly servers, with the following +IPs and domain names: - `10.6.0.91`: Gitaly 1 (`gitaly1.internal`) - `10.6.0.92`: Gitaly 2 (`gitaly2.internal`) -The secret token is assumed to be `gitalysecret` and that -your GitLab installation has three repository storages: +Assumptions about your servers include having the secret token be `gitalysecret`, +and that your GitLab installation has three repository storages: - `default` on Gitaly 1 - `storage1` on Gitaly 1 @@ -1415,11 +1384,11 @@ your GitLab installation has three repository storages: On each node: -1. [Download/Install](https://about.gitlab.com/install/) the Omnibus GitLab - package you want using **steps 1 and 2** from the GitLab downloads page but - **without** providing the `EXTERNAL_URL` value. -1. Edit `/etc/gitlab/gitlab.rb` to configure storage paths, enable - the network listener and configure the token: +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to follow _only_ installation steps 1 and 2 + on the page, and _do not_ provide the `EXTERNAL_URL` value. +1. Edit `/etc/gitlab/gitlab.rb` to configure the storage paths, enable + the network listener, and configure the token: <!-- updates to following example must also be made at @@ -1467,39 +1436,39 @@ On each node: ``` 1. Append the following to `/etc/gitlab/gitlab.rb` for each respective server: - 1. On `gitaly1.internal`: - - ```ruby - git_data_dirs({ - 'default' => { - 'path' => '/var/opt/gitlab/git-data' - }, - 'storage1' => { - 'path' => '/mnt/gitlab/git-data' - }, - }) - ``` - - 1. On `gitaly2.internal`: - - ```ruby - git_data_dirs({ - 'storage2' => { - 'path' => '/mnt/gitlab/git-data' - }, - }) - ``` + - On `gitaly1.internal`: + + ```ruby + git_data_dirs({ + 'default' => { + 'path' => '/var/opt/gitlab/git-data' + }, + 'storage1' => { + 'path' => '/mnt/gitlab/git-data' + }, + }) + ``` + + - On `gitaly2.internal`: + + ```ruby + git_data_dirs({ + 'storage2' => { + 'path' => '/mnt/gitlab/git-data' + }, + }) + ``` <!-- updates to following example must also be made at https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab --> -1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace - the file of the same name on this server. If that file is not on this server, - add the file from your Consul server to this server. +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and + then replace the file of the same name on this server. If that file isn't on + this server, add the file from your Consul server to this server. -1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). +1. Save the file, and then [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). ### Gitaly TLS support @@ -1521,11 +1490,10 @@ Name. If you are addressing the Gitaly server by its IP address, you must add it as a Subject Alternative Name to the certificate. [gRPC does not support using an IP address as Common Name in a certificate](https://github.com/grpc/grpc/issues/2691). -NOTE: **Note:** -It is possible to configure Gitaly servers with both an -unencrypted listening address `listen_addr` and an encrypted listening -address `tls_listen_addr` at the same time. This allows you to do a -gradual transition from unencrypted to encrypted traffic, if necessary. +It's possible to configure Gitaly servers with both an unencrypted listening +address (`listen_addr`) and an encrypted listening address (`tls_listen_addr`) +at the same time. This allows you to do a gradual transition from unencrypted to +encrypted traffic, if necessary. To configure Gitaly with TLS: @@ -1580,10 +1548,10 @@ The following IPs will be used as an example: To configure the Sidekiq nodes, on each one: -1. SSH into the Sidekiq server. -1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab package -you want using steps 1 and 2 from the GitLab downloads page. -**Do not complete any other steps on the download page.** +1. SSH in to the Sidekiq server. +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to follow _only_ installation steps 1 and 2 + on the page. 1. Open `/etc/gitlab/gitlab.rb` with your editor: ```ruby @@ -1701,15 +1669,14 @@ You can also run [multiple Sidekiq processes](../operations/extra_sidekiq_proces ## Configure GitLab Rails -NOTE: **Note:** -In our architectures we run each GitLab Rails node using the Puma webserver -and have its number of workers set to 90% of available CPUs along with four threads. For -nodes that are running Rails with other components the worker value should be reduced -accordingly where we've found 50% achieves a good balance but this is dependent -on workload. - This section describes how to configure the GitLab application (Rails) component. +In our architecture, we run each GitLab Rails node using the Puma webserver, and +have its number of workers set to 90% of available CPUs, with four threads. For +nodes running Rails with other components, the worker value should be reduced +accordingly. We've determined that a worker value of 50% achieves a good balance, +but this is dependent on workload. + The following IPs will be used as an example: - `10.6.0.111`: GitLab application 1 @@ -1718,10 +1685,9 @@ The following IPs will be used as an example: On each node perform the following: -1. Download/install Omnibus GitLab using **steps 1 and 2** from - [GitLab downloads](https://about.gitlab.com/install/). Do not complete other - steps on the download page. - +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to follow _only_ installation steps 1 and 2 + on the page. 1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace the file of the same name on this server. If that file is not on this server, add the file from your Consul server to this server. @@ -1752,6 +1718,7 @@ On each node perform the following: roles ['application_role'] gitaly['enable'] = false nginx['enable'] = true + sidekiq['enable'] = false ## PostgreSQL connection details # Disable PostgreSQL on the application node @@ -1795,7 +1762,6 @@ On each node perform the following: # Set the network addresses that the exporters used for monitoring will listen on node_exporter['listen_address'] = '0.0.0.0:9100' gitlab_workhorse['prometheus_listen_addr'] = '0.0.0.0:9229' - sidekiq['listen_address'] = "0.0.0.0" puma['listen'] = '0.0.0.0' # Add the monitoring node's IP address to the monitoring whitelist and allow it to @@ -1836,7 +1802,7 @@ On each node perform the following: 1. Specify the necessary NFS mounts in `/etc/fstab`. The exact contents of `/etc/fstab` will depend on how you chose - to configure your NFS server. See the [NFS documentation](../high_availability/nfs.md) + to configure your NFS server. See the [NFS documentation](../nfs.md) for examples and the various options. 1. Create the shared directories. These may be different depending on your NFS @@ -1877,30 +1843,31 @@ On each node perform the following: 1. Optionally, from the Gitaly servers, confirm that Gitaly can perform callbacks to the internal API: ```shell - sudo /opt/gitlab/embedded/service/gitlab-shell/bin/check -config /opt/gitlab/embedded/service/gitlab-shell/config.yml + sudo /opt/gitlab/embedded/bin/gitaly-hooks check /var/opt/gitlab/gitaly/config.toml ``` -NOTE: **Note:** -When you specify `https` in the `external_url`, as in the example -above, GitLab assumes you have SSL certificates in `/etc/gitlab/ssl/`. If -certificates are not present, NGINX will fail to start. See the -[NGINX documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https) -for more information. +When you specify `https` in the `external_url`, as in the previous example, +GitLab expects that the SSL certificates are in `/etc/gitlab/ssl/`. If the +certificates aren't present, NGINX will fail to start. For more information, see +the [NGINX documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https). ### GitLab Rails post-configuration -Initialize the GitLab database, by running the following in one of the Rails nodes: +1. Designate one application node for running database migrations during + installation and updates. Initialize the GitLab database and ensure all + migrations ran: -```shell -sudo gitlab-rake gitlab:db:configure -``` + ```shell + sudo gitlab-rake gitlab:db:configure + ``` -NOTE: **Note:** -If you encounter a `rake aborted!` error stating that PgBouncer is failing to connect to -PostgreSQL it may be that your PgBouncer node's IP address is missing from -PostgreSQL's `trust_auth_cidr_addresses` in `gitlab.rb` on your database nodes. See -[PgBouncer error `ERROR: pgbouncer cannot connect to server`](troubleshooting.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server) -in the Troubleshooting section before proceeding. + If you encounter a `rake aborted!` error message stating that PgBouncer is + failing to connect to PostgreSQL, it may be that your PgBouncer node's IP + address is missing from PostgreSQL's `trust_auth_cidr_addresses` in `gitlab.rb` + on your database nodes. Before proceeding, see + [PgBouncer error `ERROR: pgbouncer cannot connect to server`](troubleshooting.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server). + +1. [Configure fast lookup of authorized SSH keys in the database](../operations/fast_ssh_key_lookup.md). <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> @@ -1920,11 +1887,10 @@ The following IP will be used as an example: To configure the Monitoring node: -1. SSH into the Monitoring node. -1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab - package you want using **steps 1 and 2** from the GitLab downloads page. - Do not complete any other steps on the download page. - +1. SSH in to the Monitoring node. +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to follow _only_ installation steps 1 and 2 + on the page. 1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace the file of the same name on this server. If that file is not on this server, add the file from your Consul server to this server. @@ -1988,28 +1954,44 @@ It's recommended over [NFS](#configure-nfs-optional) and in general it's better in larger setups as object storage is typically much more performant, reliable, and scalable. -Object storage options that GitLab has tested, or is aware of customers using include: +GitLab has been tested on a number of object storage providers: -- SaaS/Cloud solutions such as [Amazon S3](https://aws.amazon.com/s3/), [Google cloud storage](https://cloud.google.com/storage). +- [Amazon S3](https://aws.amazon.com/s3/) +- [Google Cloud Storage](https://cloud.google.com/storage) +- [Digital Ocean Spaces](https://www.digitalocean.com/products/spaces/) +- [Oracle Cloud Infrastructure](https://docs.cloud.oracle.com/en-us/iaas/Content/Object/Tasks/s3compatibleapi.htm) +- [Openstack Swift](https://docs.openstack.org/swift/latest/s3_compat.html) +- [Azure Blob storage](https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blobs-introduction) - On-premises hardware and appliances from various storage vendors. -- MinIO. There is [a guide to deploying this](https://docs.gitlab.com/charts/advanced/external-object-storage/minio.html) within our Helm Chart documentation. - -For configuring GitLab to use Object Storage refer to the following guides -based on what features you intend to use: - -1. Configure [object storage for backups](../../raketasks/backup_restore.md#uploading-backups-to-a-remote-cloud-storage). -1. Configure [object storage for job artifacts](../job_artifacts.md#using-object-storage) - including [incremental logging](../job_logs.md#new-incremental-logging-architecture). -1. Configure [object storage for LFS objects](../lfs/index.md#storing-lfs-objects-in-remote-object-storage). -1. Configure [object storage for uploads](../uploads.md#using-object-storage). -1. Configure [object storage for merge request diffs](../merge_request_diffs.md#using-object-storage). -1. Configure [object storage for Container Registry](../packages/container_registry.md#use-object-storage) (optional feature). -1. Configure [object storage for Mattermost](https://docs.mattermost.com/administration/config-settings.html#file-storage) (optional feature). -1. Configure [object storage for packages](../packages/index.md#using-object-storage) (optional feature). **(PREMIUM ONLY)** -1. Configure [object storage for Dependency Proxy](../packages/dependency_proxy.md#using-object-storage) (optional feature). **(PREMIUM ONLY)** -1. Configure [object storage for Pseudonymizer](../pseudonymizer.md#configuration) (optional feature). **(ULTIMATE ONLY)** -1. Configure [object storage for autoscale runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional - for improved performance). -1. Configure [object storage for Terraform state files](../terraform_state.md#using-object-storage). +- MinIO. We have [a guide to deploying this](https://docs.gitlab.com/charts/advanced/external-object-storage/minio.html) within our Helm Chart documentation. + +There are two ways of specifying object storage configuration in GitLab: + +- [Consolidated form](../object_storage.md#consolidated-object-storage-configuration): A single credential is + shared by all supported object types. +- [Storage-specific form](../object_storage.md#storage-specific-configuration): Every object defines its + own object storage [connection and configuration](../object_storage.md#connection-settings). + +Starting with GitLab 13.2, consolidated object storage configuration is available. It simplifies your GitLab configuration since the connection details are shared across object types. Refer to [Consolidated object storage configuration](../object_storage.md#consolidated-object-storage-configuration) guide for instructions on how to set it up. + +For configuring object storage in GitLab 13.1 and earlier, or for storage types not +supported by consolidated configuration form, refer to the following guides based +on what features you intend to use: + +|Object storage type|Supported by consolidated configuration?| +|-------------------|----------------------------------------| +| [Backups](../../raketasks/backup_restore.md#uploading-backups-to-a-remote-cloud-storage)|No| +| [Job artifacts](../job_artifacts.md#using-object-storage) including archived job logs | Yes | +| [LFS objects](../lfs/index.md#storing-lfs-objects-in-remote-object-storage) | Yes | +| [Uploads](../uploads.md#using-object-storage) | Yes | +| [Container Registry](../packages/container_registry.md#use-object-storage) (optional feature) | No | +| [Merge request diffs](../merge_request_diffs.md#using-object-storage) | Yes | +| [Mattermost](https://docs.mattermost.com/administration/config-settings.html#file-storage)| No | +| [Packages](../packages/index.md#using-object-storage) (optional feature) | Yes | +| [Dependency Proxy](../packages/dependency_proxy.md#using-object-storage) (optional feature) **(PREMIUM ONLY)** | Yes | +| [Pseudonymizer](../pseudonymizer.md#configuration) (optional feature) **(ULTIMATE ONLY)** | No | +| [Autoscale runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional for improved performance) | No | +| [Terraform state files](../terraform_state.md#using-object-storage) | Yes | Using separate buckets for each data type is the recommended approach for GitLab. @@ -2033,13 +2015,29 @@ work. </a> </div> +## Configure Advanced Search **(STARTER ONLY)** + +You can leverage Elasticsearch and [enable Advanced Search](../../integration/elasticsearch.md) +for faster, more advanced code search across your entire GitLab instance. + +Elasticsearch cluster design and requirements are dependent on your specific +data. For recommended best practices about how to set up your Elasticsearch +cluster alongside your instance, read how to +[choose the optimal cluster configuration](../../integration/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration). + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + ## Configure NFS (optional) [Object storage](#configure-the-object-storage), along with [Gitaly](#configure-gitaly) are recommended over NFS wherever possible for improved performance. If you intend to use GitLab Pages, this currently [requires NFS](troubleshooting.md#gitlab-pages-requires-nfs). -See how to [configure NFS](../high_availability/nfs.md). +See how to [configure NFS](../nfs.md). <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> diff --git a/doc/administration/reference_architectures/2k_users.md b/doc/administration/reference_architectures/2k_users.md index 34b90964fbf..4863329b695 100644 --- a/doc/administration/reference_architectures/2k_users.md +++ b/doc/administration/reference_architectures/2k_users.md @@ -17,14 +17,14 @@ For a full list of reference architectures, see | Service | Nodes | Configuration | GCP | AWS | Azure | |------------------------------------------|--------|-------------------------|----------------|--------------|---------| -| Load balancer | 1 | 2 vCPU, 1.8GB memory | n1-highcpu-2 | c5.large | F2s v2 | -| PostgreSQL | 1 | 2 vCPU, 7.5GB memory | n1-standard-2 | m5.large | D2s v3 | -| Redis | 1 | 1 vCPU, 3.75GB memory | n1-standard-1 | m5.large | D2s v3 | -| Gitaly | 1 | 4 vCPU, 15GB memory | n1-standard-4 | m5.xlarge | D4s v3 | -| GitLab Rails | 2 | 8 vCPU, 7.2GB memory | n1-highcpu-8 | c5.2xlarge | F8s v2 | -| Monitoring node | 1 | 2 vCPU, 1.8GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| Load balancer | 1 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| PostgreSQL | 1 | 2 vCPU, 7.5 GB memory | n1-standard-2 | m5.large | D2s v3 | +| Redis | 1 | 1 vCPU, 3.75 GB memory | n1-standard-1 | m5.large | D2s v3 | +| Gitaly | 1 | 4 vCPU, 15 GB memory | n1-standard-4 | m5.xlarge | D4s v3 | +| GitLab Rails | 2 | 8 vCPU, 7.2 GB memory | n1-highcpu-8 | c5.2xlarge | F8s v2 | +| Monitoring node | 1 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | c5.large | F2s v2 | | Object storage | n/a | n/a | n/a | n/a | n/a | -| NFS server (optional, not recommended) | 1 | 4 vCPU, 3.6GB memory | n1-highcpu-4 | c5.xlarge | F4s v2 | +| NFS server (optional, not recommended) | 1 | 4 vCPU, 3.6 GB memory | n1-highcpu-4 | c5.xlarge | F4s v2 | The Google Cloud Platform (GCP) architectures were built and tested using the [Intel Xeon E5 v3 (Haswell)](https://cloud.google.com/compute/docs/cpu-platforms) @@ -43,7 +43,7 @@ doesn't require you to provision and maintain a node. To set up GitLab and its components to accommodate up to 2,000 users: 1. [Configure the external load balancing node](#configure-the-external-load-balancer) - to handle the load balancing of the two GitLab application services nodes. + to handle the load balancing of the GitLab application services nodes. 1. [Configure PostgreSQL](#configure-postgresql), the database for GitLab. 1. [Configure Redis](#configure-redis). 1. [Configure Gitaly](#configure-gitaly), which provides access to the Git @@ -55,6 +55,8 @@ To set up GitLab and its components to accommodate up to 2,000 users: environment. 1. [Configure the object storage](#configure-the-object-storage) used for shared data objects. +1. [Configure Advanced Search](#configure-advanced-search) (optional) for faster, + more advanced code search across your entire GitLab instance. 1. [Configure NFS](#configure-nfs-optional) (optional, and not recommended) to have shared disk storage service as an alternative to Gitaly or object storage. You can skip this step if you're not using GitLab Pages (which @@ -62,18 +64,17 @@ To set up GitLab and its components to accommodate up to 2,000 users: ## Configure the external load balancer -NOTE: **Note:** -This architecture has been tested and validated with [HAProxy](https://www.haproxy.org/). -Although you can use a load balancer with a similar set of features, GitLab -hasn't validated other load balancers. - In an active/active GitLab configuration, you'll need a load balancer to route -traffic to the application servers. The specifics for which load balancer to -use or its exact configuration is out of scope for the GitLab documentation. -If you're managing multi-node systems (including GitLab) you'll probably -already have a load balancer of choice. Some examples including HAProxy -(open-source), F5 Big-IP LTM, and Citrix Net Scaler. This documentation -includes the ports and protocols for use with GitLab. +traffic to the application servers. The specifics on which load balancer to use +or its exact configuration is beyond the scope of GitLab documentation. We hope +that if you're managing multi-node systems like GitLab, you already have a load +balancer of choice. Some load balancer examples include HAProxy (open-source), +F5 Big-IP LTM, and Citrix Net Scaler. This documentation outline the ports and +protocols needed for use with GitLab. + +This architecture has been tested and validated with [HAProxy](https://www.haproxy.org/) +as the load balancer. Although other load balancers with similar feature sets +could also be used, those load balancers have not been validated. The next question is how you will handle SSL in your environment. There are several different options: @@ -206,10 +207,10 @@ further configuration steps. ### Standalone PostgreSQL using Omnibus GitLab -1. SSH into the PostgreSQL server. -1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab - package you want using **steps 1 and 2** from the GitLab downloads page. - - Do not complete any other steps on the download page. +1. SSH in to the PostgreSQL server. +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to follow _only_ installation steps 1 and 2 + on the page. 1. Generate a password hash for PostgreSQL. This assumes you will use the default username of `gitlab` (recommended). The command will request a password and confirmation. Use the value that is output by this command in the next @@ -300,11 +301,10 @@ The Omnibus GitLab package can be used to configure a standalone Redis server. The steps below are the minimum necessary to configure a Redis server with Omnibus: -1. SSH into the Redis server. -1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab - package you want using **steps 1 and 2** from the GitLab downloads page. - - Do not complete any other steps on the download page. - +1. SSH in to the Redis server. +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to follow _only_ installation steps 1 and 2 + on the page. 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby @@ -393,12 +393,11 @@ The following procedure describes how to configure a single Gitaly server named `gitaly1.internal` with the secret token `gitalysecret`. We assume your GitLab installation has two repository storages: `default` and `storage1`. -To configure the Gitaly server: +To configure the Gitaly server, on the server node you want to use for Gitaly: -1. On the server node you want to use for Gitaly, - [download and install](https://about.gitlab.com/install/) your selected - Omnibus GitLab package using *steps 1 and 2* from the GitLab downloads page, - but *without* providing the `EXTERNAL_URL` value. +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to follow _only_ installation steps 1 and 2 + on the page, and _do not_ provide the `EXTERNAL_URL` value. 1. Edit the Gitaly server node's `/etc/gitlab/gitlab.rb` file to configure storage paths, enable the network listener, and to configure the token: @@ -464,7 +463,7 @@ To configure the Gitaly server: 1. Confirm that Gitaly can perform callbacks to the internal API: ```shell - sudo /opt/gitlab/embedded/service/gitlab-shell/bin/check -config /opt/gitlab/embedded/service/gitlab-shell/config.yml + sudo /opt/gitlab/embedded/bin/gitaly-hooks check /var/opt/gitlab/gitaly/config.toml ``` ### Gitaly TLS support @@ -487,11 +486,10 @@ Name. If you are addressing the Gitaly server by its IP address, you must add it as a Subject Alternative Name to the certificate. [gRPC does not support using an IP address as Common Name in a certificate](https://github.com/grpc/grpc/issues/2691). -NOTE: **Note:** -It is possible to configure Gitaly servers with both an -unencrypted listening address `listen_addr` and an encrypted listening -address `tls_listen_addr` at the same time. This allows you to do a -gradual transition from unencrypted to encrypted traffic, if necessary. +It's possible to configure Gitaly servers with both an unencrypted listening +address (`listen_addr`) and an encrypted listening address (`tls_listen_addr`) +at the same time. This allows you to do a gradual transition from unencrypted to +encrypted traffic, if necessary. To configure Gitaly with TLS: @@ -535,14 +533,14 @@ To configure Gitaly with TLS: ## Configure GitLab Rails -NOTE: **Note:** -In our architectures we run each GitLab Rails node using the Puma webserver -and have its number of workers set to 90% of available CPUs along with four threads. For -nodes that are running Rails with other components the worker value should be reduced -accordingly where we've found 50% achieves a good balance but this is dependent -on workload. - This section describes how to configure the GitLab application (Rails) component. + +In our architecture, we run each GitLab Rails node using the Puma webserver, and +have its number of workers set to 90% of available CPUs, with four threads. For +nodes running Rails with other components, the worker value should be reduced +accordingly. We've determined that a worker value of 50% achieves a good balance, +but this is dependent on workload. + On each node perform the following: 1. If you're [using NFS](#configure-nfs-optional): @@ -570,10 +568,10 @@ On each node perform the following: mkdir -p /var/opt/gitlab/.ssh /var/opt/gitlab/gitlab-rails/uploads /var/opt/gitlab/gitlab-rails/shared /var/opt/gitlab/gitlab-ci/builds /var/opt/gitlab/git-data ``` -1. Download/install Omnibus GitLab using **steps 1 and 2** from - [GitLab downloads](https://about.gitlab.com/install/). Do not complete other - steps on the download page. -1. Create/edit `/etc/gitlab/gitlab.rb` and use the following configuration. +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to follow _only_ installation steps 1 and 2 + on the page. +1. Create or edit `/etc/gitlab/gitlab.rb` and use the following configuration. To maintain uniformity of links across nodes, the `external_url` on the application server should point to the external URL that users will use to access GitLab. This would be the URL of the [load balancer](#configure-the-external-load-balancer) @@ -664,12 +662,33 @@ On each node perform the following: sudo gitlab-ctl tail gitaly ``` -NOTE: **Note:** -When you specify `https` in the `external_url`, as in the example -above, GitLab assumes you have SSL certificates in `/etc/gitlab/ssl/`. If -certificates are not present, NGINX will fail to start. See the -[NGINX documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https) -for more information. +1. Save the `/etc/gitlab/gitlab-secrets.json` file from one of the two + application nodes and install it on the other application node and the + [Gitaly node](#configure-gitaly) and + [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). + +When you specify `https` in the `external_url`, as in the previous example, +GitLab expects that the SSL certificates are in `/etc/gitlab/ssl/`. If the +certificates aren't present, NGINX will fail to start. For more information, see +the [NGINX documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https). + +### GitLab Rails post-configuration + +1. Designate one application node for running database migrations during + installation and updates. Initialize the GitLab database and ensure all + migrations ran: + + ```shell + sudo gitlab-rake gitlab:db:configure + ``` + + If you encounter a `rake aborted!` error message stating that PgBouncer is + failing to connect to PostgreSQL, it may be that your PgBouncer node's IP + address is missing from PostgreSQL's `trust_auth_cidr_addresses` in `gitlab.rb` + on your database nodes. Before proceeding, see + [PgBouncer error `ERROR: pgbouncer cannot connect to server`](troubleshooting.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server). + +1. [Configure fast lookup of authorized SSH keys in the database](../operations/fast_ssh_key_lookup.md). <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> @@ -683,10 +702,10 @@ The Omnibus GitLab package can be used to configure a standalone Monitoring node running [Prometheus](../monitoring/prometheus/index.md) and [Grafana](../monitoring/performance/grafana_configuration.md): -1. SSH into the Monitoring node. -1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab - package you want using **steps 1 and 2** from the GitLab downloads page. - Do not complete any other steps on the download page. +1. SSH in to the Monitoring node. +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to follow _only_ installation steps 1 and 2 + on the page. 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby @@ -806,30 +825,44 @@ data, and is recommended over [NFS](#configure-nfs-optional). In general, object storage services are better for larger environments, as object storage is typically much more performant, reliable, and scalable. -Object storage options that GitLab has either tested or is aware of customers -using, includes: - -- SaaS/Cloud solutions (such as [Amazon S3](https://aws.amazon.com/s3/) or - [Google Cloud Storage](https://cloud.google.com/storage)). -- On-premises hardware and appliances, from various storage vendors. -- MinIO ([Deployment guide](https://docs.gitlab.com/charts/advanced/external-object-storage/minio.html)). - -To configure GitLab to use object storage, refer to the following guides based -on the features you intend to use: - -1. [Object storage for backups](../../raketasks/backup_restore.md#uploading-backups-to-a-remote-cloud-storage). -1. [Object storage for job artifacts](../job_artifacts.md#using-object-storage) - including [incremental logging](../job_logs.md#new-incremental-logging-architecture). -1. [Object storage for LFS objects](../lfs/index.md#storing-lfs-objects-in-remote-object-storage). -1. [Object storage for uploads](../uploads.md#using-object-storage). -1. [Object storage for merge request diffs](../merge_request_diffs.md#using-object-storage). -1. [Object storage for Container Registry](../packages/container_registry.md#use-object-storage) (optional feature). -1. [Object storage for Mattermost](https://docs.mattermost.com/administration/config-settings.html#file-storage) (optional feature). -1. [Object storage for packages](../packages/index.md#using-object-storage) (optional feature). **(PREMIUM ONLY)** -1. [Object storage for Dependency Proxy](../packages/dependency_proxy.md#using-object-storage) (optional feature). **(PREMIUM ONLY)** -1. [Object storage for Pseudonymizer](../pseudonymizer.md#configuration) (optional feature). **(ULTIMATE ONLY)** -1. [Object storage for autoscale runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional, for improved performance). -1. [Object storage for Terraform state files](../terraform_state.md#using-object-storage). +GitLab has been tested on a number of object storage providers: + +- [Amazon S3](https://aws.amazon.com/s3/) +- [Google Cloud Storage](https://cloud.google.com/storage) +- [Digital Ocean Spaces](https://www.digitalocean.com/products/spaces/) +- [Oracle Cloud Infrastructure](https://docs.cloud.oracle.com/en-us/iaas/Content/Object/Tasks/s3compatibleapi.htm) +- [Openstack Swift](https://docs.openstack.org/swift/latest/s3_compat.html) +- [Azure Blob storage](https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blobs-introduction) +- On-premises hardware and appliances from various storage vendors. +- MinIO. We have [a guide to deploying this](https://docs.gitlab.com/charts/advanced/external-object-storage/minio.html) within our Helm Chart documentation. + +There are two ways of specifying object storage configuration in GitLab: + +- [Consolidated form](../object_storage.md#consolidated-object-storage-configuration): A single credential is + shared by all supported object types. +- [Storage-specific form](../object_storage.md#storage-specific-configuration): Every object defines its + own object storage [connection and configuration](../object_storage.md#connection-settings). + +Starting with GitLab 13.2, consolidated object storage configuration is available. It simplifies your GitLab configuration since the connection details are shared across object types. Refer to [Consolidated object storage configuration](../object_storage.md#consolidated-object-storage-configuration) guide for instructions on how to set it up. + +For configuring object storage in GitLab 13.1 and earlier, or for storage types not +supported by consolidated configuration form, refer to the following guides based +on what features you intend to use: + +|Object storage type|Supported by consolidated configuration?| +|-------------------|----------------------------------------| +| [Backups](../../raketasks/backup_restore.md#uploading-backups-to-a-remote-cloud-storage)|No| +| [Job artifacts](../job_artifacts.md#using-object-storage) including archived job logs | Yes | +| [LFS objects](../lfs/index.md#storing-lfs-objects-in-remote-object-storage) | Yes | +| [Uploads](../uploads.md#using-object-storage) | Yes | +| [Container Registry](../packages/container_registry.md#use-object-storage) (optional feature) | No | +| [Merge request diffs](../merge_request_diffs.md#using-object-storage) | Yes | +| [Mattermost](https://docs.mattermost.com/administration/config-settings.html#file-storage)| No | +| [Packages](../packages/index.md#using-object-storage) (optional feature) | Yes | +| [Dependency Proxy](../packages/dependency_proxy.md#using-object-storage) (optional feature) **(PREMIUM ONLY)** | Yes | +| [Pseudonymizer](../pseudonymizer.md#configuration) (optional feature) **(ULTIMATE ONLY)** | No | +| [Autoscale runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional for improved performance) | No | +| [Terraform state files](../terraform_state.md#using-object-storage) | Yes | Using separate buckets for each data type is the recommended approach for GitLab. @@ -851,6 +884,22 @@ functioning backups is encountered. </a> </div> +## Configure Advanced Search **(STARTER ONLY)** + +You can leverage Elasticsearch and [enable Advanced Search](../../integration/elasticsearch.md) +for faster, more advanced code search across your entire GitLab instance. + +Elasticsearch cluster design and requirements are dependent on your specific +data. For recommended best practices about how to set up your Elasticsearch +cluster alongside your instance, read how to +[choose the optimal cluster configuration](../../integration/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration). + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + ## Configure NFS (optional) For improved performance, [object storage](#configure-the-object-storage), diff --git a/doc/administration/reference_architectures/3k_users.md b/doc/administration/reference_architectures/3k_users.md index be944586e43..70d0cae6dfd 100644 --- a/doc/administration/reference_architectures/3k_users.md +++ b/doc/administration/reference_architectures/3k_users.md @@ -24,18 +24,18 @@ costly-to-operate environment by using the | Service | Nodes | Configuration | GCP | AWS | Azure | |--------------------------------------------|-------------|-----------------------|----------------|-------------|---------| -| External load balancing node | 1 | 2 vCPU, 1.8GB memory | n1-highcpu-2 | c5.large | F2s v2 | -| Redis | 3 | 2 vCPU, 7.5GB memory | n1-standard-2 | m5.large | D2s v3 | -| Consul + Sentinel | 3 | 2 vCPU, 1.8GB memory | n1-highcpu-2 | c5.large | F2s v2 | -| PostgreSQL | 3 | 2 vCPU, 7.5GB memory | n1-standard-2 | m5.large | D2s v3 | -| PgBouncer | 3 | 2 vCPU, 1.8GB memory | n1-highcpu-2 | c5.large | F2s v2 | -| Internal load balancing node | 1 | 2 vCPU, 1.8GB memory | n1-highcpu-2 | c5.large | F2s v2 | -| Gitaly | 2 (minimum) | 4 vCPU, 15GB memory | n1-standard-4 | m5.xlarge | D4s v3 | -| Sidekiq | 4 | 2 vCPU, 7.5GB memory | n1-standard-2 | m5.large | D2s v3 | -| GitLab Rails | 3 | 8 vCPU, 7.2GB memory | n1-highcpu-8 | c5.2xlarge | F8s v2 | -| Monitoring node | 1 | 2 vCPU, 1.8GB memory | n1-highcpu-2 | c5.large | F2s v2 | -| Object Storage | n/a | n/a | n/a | n/a | n/a | -| NFS Server (optional, not recommended) | 1 | 4 vCPU, 3.6GB memory | n1-highcpu-4 | c5.xlarge | F4s v2 | +| External load balancing node | 1 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| Redis | 3 | 2 vCPU, 7.5 GB memory | n1-standard-2 | m5.large | D2s v3 | +| Consul + Sentinel | 3 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| PostgreSQL | 3 | 2 vCPU, 7.5 GB memory | n1-standard-2 | m5.large | D2s v3 | +| PgBouncer | 3 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| Internal load balancing node | 1 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| Gitaly | 2 (minimum) | 4 vCPU, 15 GB memory | n1-standard-4 | m5.xlarge | D4s v3 | +| Sidekiq | 4 | 2 vCPU, 7.5 GB memory | n1-standard-2 | m5.large | D2s v3 | +| GitLab Rails | 3 | 8 vCPU, 7.2 GB memory | n1-highcpu-8 | c5.2xlarge | F8s v2 | +| Monitoring node | 1 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| Object storage | n/a | n/a | n/a | n/a | n/a | +| NFS server (optional, not recommended) | 1 | 4 vCPU, 3.6 GB memory | n1-highcpu-4 | c5.xlarge | F4s v2 | The Google Cloud Platform (GCP) architectures were built and tested using the [Intel Xeon E5 v3 (Haswell)](https://cloud.google.com/compute/docs/cpu-platforms) @@ -44,41 +44,43 @@ or higher, are required for your CPU or node counts. For more information, see our [Sysbench](https://github.com/akopytov/sysbench)-based [CPU benchmark](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Reference-Architectures/GCP-CPU-Benchmarks). -For data objects (such as LFS, Uploads, or Artifacts), an -[object storage service](#configure-the-object-storage) is recommended instead -of NFS where possible, due to better performance and availability. Since this -doesn't require a node to be set up, *Object Storage* is noted as not -applicable (n/a) in the previous table. +Due to better performance and availability, for data objects (such as LFS, +uploads, or artifacts), using an [object storage service](#configure-the-object-storage) +is recommended instead of using NFS. Using an object storage service also +doesn't require you to provision and maintain a node. ## Setup components To set up GitLab and its components to accommodate up to 3,000 users: 1. [Configure the external load balancing node](#configure-the-external-load-balancer) - that will handle the load balancing of the two GitLab application services nodes. + to handle the load balancing of the GitLab application services nodes. 1. [Configure Redis](#configure-redis). 1. [Configure Consul and Sentinel](#configure-consul-and-sentinel). 1. [Configure PostgreSQL](#configure-postgresql), the database for GitLab. 1. [Configure PgBouncer](#configure-pgbouncer). -1. [Configure the internal load balancing node](#configure-the-internal-load-balancer) +1. [Configure the internal load balancing node](#configure-the-internal-load-balancer). 1. [Configure Gitaly](#configure-gitaly), which provides access to the Git repositories. 1. [Configure Sidekiq](#configure-sidekiq). 1. [Configure the main GitLab Rails application](#configure-gitlab-rails) - to run Puma/Unicorn, Workhorse, GitLab Shell, and to serve all frontend requests (UI, API, Git - over HTTP/SSH). -1. [Configure Prometheus](#configure-prometheus) to monitor your GitLab environment. -1. [Configure the Object Storage](#configure-the-object-storage) + to run Puma/Unicorn, Workhorse, GitLab Shell, and to serve all frontend + requests (which include UI, API, and Git over HTTP/SSH). +1. [Configure Prometheus](#configure-prometheus) to monitor your GitLab + environment. +1. [Configure the object storage](#configure-the-object-storage) used for shared data objects. -1. [Configure NFS (Optional)](#configure-nfs-optional) - to have shared disk storage service as an alternative to Gitaly and/or Object Storage (although - not recommended). NFS is required for GitLab Pages, you can skip this step if you're not using - that feature. +1. [Configure Advanced Search](#configure-advanced-search) (optional) for faster, + more advanced code search across your entire GitLab instance. +1. [Configure NFS](#configure-nfs-optional) (optional, and not recommended) + to have shared disk storage service as an alternative to Gitaly or object + storage. You can skip this step if you're not using GitLab Pages (which + requires NFS). -We start with all servers on the same 10.6.0.0/16 private network range, they -can connect to each other freely on those addresses. +The servers start on the same 10.6.0.0/24 private network range, and can +connect to each other freely on these addresses. -Here is a list and description of each machine and the assigned IP: +The following list includes descriptions of each server and its assigned IP: - `10.6.0.10`: External Load Balancer - `10.6.0.61`: Redis Primary @@ -107,19 +109,18 @@ Here is a list and description of each machine and the assigned IP: ## Configure the external load balancer -NOTE: **Note:** +In an active/active GitLab configuration, you'll need a load balancer to route +traffic to the application servers. The specifics on which load balancer to use +or its exact configuration is beyond the scope of GitLab documentation. We hope +that if you're managing multi-node systems like GitLab, you already have a load +balancer of choice. Some load balancer examples include HAProxy (open-source), +F5 Big-IP LTM, and Citrix Net Scaler. This documentation outline the ports and +protocols needed for use with GitLab. + This architecture has been tested and validated with [HAProxy](https://www.haproxy.org/) as the load balancer. Although other load balancers with similar feature sets could also be used, those load balancers have not been validated. -In an active/active GitLab configuration, you will need a load balancer to route -traffic to the application servers. The specifics on which load balancer to use -or the exact configuration is beyond the scope of GitLab documentation. We hope -that if you're managing multi-node systems like GitLab you have a load balancer of -choice already. Some examples including HAProxy (open-source), F5 Big-IP LTM, -and Citrix Net Scaler. This documentation will outline what ports and protocols -you need to use with GitLab. - The next question is how you will handle SSL in your environment. There are several different options: @@ -277,20 +278,17 @@ The requirements for a Redis setup are the following: ([Internet](https://gitlab.com/gitlab-org/gitlab-foss/uploads/c4cc8cd353604bd80315f9384035ff9e/The_Internet_IT_Crowd.png)), using a firewall. -NOTE: **Note:** -Redis nodes (both primary and replica) will need the same password defined in -`redis['password']`. At any time during a failover the Sentinels can -reconfigure a node and change its status from primary to replica and vice versa. +Both the primary and replica Redis nodes need the same password defined in +`redis['password']`. At any time during a failover, the Sentinels can reconfigure +a node and change its status from primary to replica (and vice versa). #### Configuring the primary Redis instance -1. SSH into the **Primary** Redis server. -1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab - package you want using **steps 1 and 2** from the GitLab downloads page. - - Make sure you select the correct Omnibus package, with the same version - and type (Community, Enterprise editions) of your current install. - - Do not complete any other steps on the download page. - +1. SSH in to the **Primary** Redis server. +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to both follow _only_ installation steps 1 and 2 + on the page, and to select the correct Omnibus GitLab package, with the same version + and type (Community or Enterprise editions) as your current install. 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby @@ -334,18 +332,17 @@ reconfigure a node and change its status from primary to replica and vice versa. 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. -NOTE: **Note:** -You can specify multiple roles like sentinel and Redis as: -`roles ['redis_sentinel_role', 'redis_master_role']`. -Read more about [roles](https://docs.gitlab.com/omnibus/roles/). +You can specify multiple roles, like sentinel and Redis, as: +`roles ['redis_sentinel_role', 'redis_master_role']`. Read more about +[roles](https://docs.gitlab.com/omnibus/roles/). -You can list the current Redis Primary, Replica status via: +You can list the current Redis Primary, Replica status by using: ```shell /opt/gitlab/embedded/bin/redis-cli -h <host> -a 'redis-password-goes-here' info replication ``` -Show running GitLab services via: +Show running GitLab services by using: ```shell gitlab-ctl status @@ -363,13 +360,11 @@ run: redis-exporter: (pid 30075) 76861s; run: log: (pid 29674) 76896s #### Configuring the replica Redis instances -1. SSH into the **replica** Redis server. -1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab - package you want using **steps 1 and 2** from the GitLab downloads page. - - Make sure you select the correct Omnibus package, with the same version - and type (Community, Enterprise editions) of your current install. - - Do not complete any other steps on the download page. - +1. SSH in to the **replica** Redis server. +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to both follow _only_ installation steps 1 and 2 + on the page, and to select the correct Omnibus GitLab package, with the same version + and type (Community or Enterprise editions) as your current install. 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby @@ -422,10 +417,9 @@ run: redis-exporter: (pid 30075) 76861s; run: log: (pid 29674) 76896s 1. Go through the steps again for all the other replica nodes, and make sure to set up the IPs correctly. -NOTE: **Note:** -You can specify multiple roles like sentinel and Redis as: -`roles ['redis_sentinel_role', 'redis_master_role']`. -Read more about [roles](https://docs.gitlab.com/omnibus/roles/). +You can specify multiple roles, like sentinel and Redis, as: +`roles ['redis_sentinel_role', 'redis_master_role']`. Read more about +[roles](https://docs.gitlab.com/omnibus/roles/). These values don't have to be changed again in `/etc/gitlab/gitlab.rb` after a failover, as the nodes will be managed by the [Sentinels](#configure-consul-and-sentinel), and even after a @@ -443,13 +437,6 @@ are supported and can be added if needed. ## Configure Consul and Sentinel -NOTE: **Note:** -If you are using an external Redis Sentinel instance, be sure -to exclude the `requirepass` parameter from the Sentinel -configuration. This parameter will cause clients to report `NOAUTH -Authentication required.`. [Redis Sentinel 3.2.x does not support -password authentication](https://github.com/antirez/redis/issues/3279). - Now that the Redis servers are all set up, let's configure the Sentinel servers. The following IPs will be used as an example: @@ -457,16 +444,19 @@ servers. The following IPs will be used as an example: - `10.6.0.12`: Consul/Sentinel 2 - `10.6.0.13`: Consul/Sentinel 3 -To configure the Sentinel: +NOTE: **Note:** +If you're using an external Redis Sentinel instance, be sure to exclude the +`requirepass` parameter from the Sentinel configuration. This parameter causes +clients to report `NOAUTH Authentication required.`. +[Redis Sentinel 3.2.x doesn't support password authentication](https://github.com/antirez/redis/issues/3279). -1. SSH into the server that will host Consul/Sentinel. -1. [Download/install](https://about.gitlab.com/install/) the - Omnibus GitLab Enterprise Edition package using **steps 1 and 2** from the - GitLab downloads page. - - Make sure you select the correct Omnibus package, with the same version - the GitLab application is running. - - Do not complete any other steps on the download page. +To configure the Sentinel: +1. SSH in to the server that will host Consul/Sentinel. +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to both follow _only_ installation steps 1 and 2 + on the page, and to select the correct Omnibus GitLab package, with the same version + and type (Community or Enterprise editions) as your current install. 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby @@ -556,10 +546,9 @@ To configure the Sentinel: 1. Go through the steps again for all the other Consul/Sentinel nodes, and make sure you set up the correct IPs. -NOTE: **Note:** -A Consul leader will be elected when the provisioning of the third Consul server is completed. -Viewing the Consul logs `sudo gitlab-ctl tail consul` will display -`...[INFO] consul: New leader elected: ...` +A Consul leader is _elected_ when the provisioning of the third Consul server is +complete. Viewing the Consul logs `sudo gitlab-ctl tail consul` displays +`...[INFO] consul: New leader elected: ...`. You can list the current Consul members (server, client): @@ -627,7 +616,7 @@ in the second step, do not supply the `EXTERNAL_URL` value. #### PostgreSQL primary node -1. SSH into the PostgreSQL primary node. +1. SSH in to the PostgreSQL primary node. 1. Generate a password hash for the PostgreSQL username/password pair. This assumes you will use the default username of `gitlab` (recommended). The command will request a password and confirmation. Use the value that is output by this command in the next @@ -812,7 +801,7 @@ are supported and can be added if needed. #### PostgreSQL post-configuration -SSH into the **primary node**: +SSH in to the **primary node**: 1. Open a database prompt: @@ -848,7 +837,7 @@ SSH into the **primary node**: is not an IP address, it will need to be a resolvable name (via DNS or `/etc/hosts`) -SSH into the **secondary node**: +SSH in to the **secondary node**: 1. Set up the repmgr standby: @@ -1069,51 +1058,48 @@ Refer to your preferred Load Balancer's documentation for further guidance. ## Configure Gitaly -Deploying Gitaly in its own server can benefit GitLab installations that are -larger than a single machine. - -The Gitaly node requirements are dependent on customer data, specifically the number of -projects and their repository sizes. Two nodes are recommended as an absolute minimum. -Each Gitaly node should store no more than 5TB of data and have the number of -[`gitaly-ruby` workers](../gitaly/index.md#gitaly-ruby) set to 20% of available CPUs. -Additional nodes should be considered in conjunction with a review of expected -data size and spread based on the recommendations above. - -It is also strongly recommended that all Gitaly nodes be set up with SSD disks with -a throughput of at least 8,000 IOPS for read operations and 2,000 IOPS for write, -as Gitaly has heavy I/O. These IOPS values are recommended only as a starter as with -time they may be adjusted higher or lower depending on the scale of your environment's workload. -If you're running the environment on a Cloud provider, you may need to refer to -their documentation on how to configure IOPS correctly. - -Some things to note: - -- The GitLab Rails application shards repositories into [repository storages](../repository_storage_paths.md). -- A Gitaly server can host one or more storages. -- A GitLab server can use one or more Gitaly servers. -- Gitaly addresses must be specified in such a way that they resolve - correctly for ALL Gitaly clients. +[Gitaly](../gitaly/index.md) server node requirements are dependent on data, +specifically the number of projects and those projects' sizes. It's recommended +that a Gitaly server node stores no more than 5 TB of data. Although this +reference architecture includes a recommendation for the number of Gitaly server +nodes to use, depending on your storage requirements, you may require additional +Gitaly server nodes. + +Due to Gitaly having notable input and output requirements, we strongly +recommend that all Gitaly nodes use solid-state drives (SSDs). These SSDs should +have a throughput of at least 8,000 input/output operations per second (IOPS) +for read operations and 2,000 IOPS for write operations. These IOPS values are +initial recommendations, and may be adjusted to greater or lesser values +depending on the scale of your environment's workload. If you're running the +environment on a Cloud provider, refer to their documentation about how to +configure IOPS correctly. + +Be sure to note the following items: + +- The GitLab Rails application shards repositories into + [repository storage paths](../repository_storage_paths.md). +- A Gitaly server can host one or more storage paths. +- A GitLab server can use one or more Gitaly server nodes. +- Gitaly addresses must be specified to be correctly resolvable for _all_ + Gitaly clients. - Gitaly servers must not be exposed to the public internet, as Gitaly's network traffic is unencrypted by default. The use of a firewall is highly recommended to restrict access to the Gitaly server. Another option is to [use TLS](#gitaly-tls-support). -TIP: **Tip:** -For more information about Gitaly's history and network architecture see the -[standalone Gitaly documentation](../gitaly/index.md). - -Note: **Note:** The token referred to throughout the Gitaly documentation is -just an arbitrary password selected by the administrator. It is unrelated to -tokens created for the GitLab API or other similar web API tokens. +NOTE: **Note:** +The token referred to throughout the Gitaly documentation is an arbitrary +password selected by the administrator. This token is unrelated to tokens +created for the GitLab API or other similar web API tokens. -Below we describe how to configure two Gitaly servers, with IPs and -domain names: +This section describes how to configure two Gitaly servers, with the following +IPs and domain names: - `10.6.0.51`: Gitaly 1 (`gitaly1.internal`) - `10.6.0.52`: Gitaly 2 (`gitaly2.internal`) -The secret token is assumed to be `gitalysecret` and that -your GitLab installation has three repository storages: +Assumptions about your servers include having the secret token be `gitalysecret`, +and that your GitLab installation has three repository storages: - `default` on Gitaly 1 - `storage1` on Gitaly 1 @@ -1121,11 +1107,11 @@ your GitLab installation has three repository storages: On each node: -1. [Download/Install](https://about.gitlab.com/install/) the Omnibus GitLab - package you want using **steps 1 and 2** from the GitLab downloads page but - **without** providing the `EXTERNAL_URL` value. -1. Edit `/etc/gitlab/gitlab.rb` to configure storage paths, enable - the network listener and configure the token: +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to follow _only_ installation steps 1 and 2 + on the page, and _do not_ provide the `EXTERNAL_URL` value. +1. Edit `/etc/gitlab/gitlab.rb` to configure the storage paths, enable + the network listener, and configure the token: <!-- updates to following example must also be made at @@ -1189,39 +1175,39 @@ On each node: ``` 1. Append the following to `/etc/gitlab/gitlab.rb` for each respective server: - 1. On `gitaly1.internal`: - - ```ruby - git_data_dirs({ - 'default' => { - 'path' => '/var/opt/gitlab/git-data' - }, - 'storage1' => { - 'path' => '/mnt/gitlab/git-data' - }, - }) - ``` - - 1. On `gitaly2.internal`: - - ```ruby - git_data_dirs({ - 'storage2' => { - 'path' => '/mnt/gitlab/git-data' - }, - }) - ``` + - On `gitaly1.internal`: + + ```ruby + git_data_dirs({ + 'default' => { + 'path' => '/var/opt/gitlab/git-data' + }, + 'storage1' => { + 'path' => '/mnt/gitlab/git-data' + }, + }) + ``` + + - On `gitaly2.internal`: + + ```ruby + git_data_dirs({ + 'storage2' => { + 'path' => '/mnt/gitlab/git-data' + }, + }) + ``` <!-- updates to following example must also be made at https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab --> -1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). +1. Save the file, and then [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). 1. Confirm that Gitaly can perform callbacks to the internal API: ```shell - sudo /opt/gitlab/embedded/service/gitlab-shell/bin/check -config /opt/gitlab/embedded/service/gitlab-shell/config.yml + sudo /opt/gitlab/embedded/bin/gitaly-hooks check /var/opt/gitlab/gitaly/config.toml ``` 1. Verify the GitLab services are running: @@ -1259,11 +1245,10 @@ Name. If you are addressing the Gitaly server by its IP address, you must add it as a Subject Alternative Name to the certificate. [gRPC does not support using an IP address as Common Name in a certificate](https://github.com/grpc/grpc/issues/2691). -NOTE: **Note:** -It is possible to configure Gitaly servers with both an -unencrypted listening address `listen_addr` and an encrypted listening -address `tls_listen_addr` at the same time. This allows you to do a -gradual transition from unencrypted to encrypted traffic, if necessary. +It's possible to configure Gitaly servers with both an unencrypted listening +address (`listen_addr`) and an encrypted listening address (`tls_listen_addr`) +at the same time. This allows you to do a gradual transition from unencrypted to +encrypted traffic, if necessary. To configure Gitaly with TLS: @@ -1317,10 +1302,10 @@ The following IPs will be used as an example: To configure the Sidekiq nodes, one each one: -1. SSH into the Sidekiq server. -1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab package -you want using steps 1 and 2 from the GitLab downloads page. -**Do not complete any other steps on the download page.** +1. SSH in to the Sidekiq server. +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to follow _only_ installation steps 1 and 2 + on the page. 1. Open `/etc/gitlab/gitlab.rb` with your editor: ```ruby @@ -1431,14 +1416,14 @@ You can also run [multiple Sidekiq processes](../operations/extra_sidekiq_proces ## Configure GitLab Rails -NOTE: **Note:** -In our architectures we run each GitLab Rails node using the Puma webserver -and have its number of workers set to 90% of available CPUs along with four threads. For -nodes that are running Rails with other components the worker value should be reduced -accordingly where we've found 50% achieves a good balance but this is dependent -on workload. - This section describes how to configure the GitLab application (Rails) component. + +In our architecture, we run each GitLab Rails node using the Puma webserver, and +have its number of workers set to 90% of available CPUs, with four threads. For +nodes running Rails with other components, the worker value should be reduced +accordingly. We've determined that a worker value of 50% achieves a good balance, +but this is dependent on workload. + On each node perform the following: 1. If you're [using NFS](#configure-nfs-optional): @@ -1466,10 +1451,10 @@ On each node perform the following: mkdir -p /var/opt/gitlab/.ssh /var/opt/gitlab/gitlab-rails/uploads /var/opt/gitlab/gitlab-rails/shared /var/opt/gitlab/gitlab-ci/builds /var/opt/gitlab/git-data ``` -1. Download/install Omnibus GitLab using **steps 1 and 2** from - [GitLab downloads](https://about.gitlab.com/install/). Do not complete other - steps on the download page. -1. Create/edit `/etc/gitlab/gitlab.rb` and use the following configuration. +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to follow _only_ installation steps 1 and 2 + on the page. +1. Create or edit `/etc/gitlab/gitlab.rb` and use the following configuration. To maintain uniformity of links across nodes, the `external_url` on the application server should point to the external URL that users will use to access GitLab. This would be the URL of the [external load balancer](#configure-the-external-load-balancer) @@ -1582,6 +1567,11 @@ On each node perform the following: sudo gitlab-ctl tail gitaly ``` +1. Save the `/etc/gitlab/gitlab-secrets.json` file from one of the two + application nodes and install it on the other application node, the + [Gitaly node](#configure-gitaly) and the [Sidekiq node](#configure-sidekiq) and + [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). + 1. Verify the GitLab services are running: ```shell @@ -1600,12 +1590,10 @@ On each node perform the following: run: puma: (pid 4936) 8645s; run: log: (pid 29656) 79161s ``` -NOTE: **Note:** -When you specify `https` in the `external_url`, as in the example -above, GitLab assumes you have SSL certificates in `/etc/gitlab/ssl/`. If -certificates are not present, NGINX will fail to start. See the -[NGINX documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https) -for more information. +When you specify `https` in the `external_url`, as in the previous example, +GitLab expects that the SSL certificates are in `/etc/gitlab/ssl/`. If the +certificates aren't present, NGINX will fail to start. For more information, see +the [NGINX documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https). ### GitLab Rails post-configuration @@ -1615,12 +1603,11 @@ for more information. gitlab-rake gitlab:db:configure ``` - NOTE: **Note:** - If you encounter a `rake aborted!` error stating that PgBouncer is failing to connect to - PostgreSQL it may be that your PgBouncer node's IP address is missing from - PostgreSQL's `trust_auth_cidr_addresses` in `gitlab.rb` on your database nodes. See - [PgBouncer error `ERROR: pgbouncer cannot connect to server`](troubleshooting.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server) - in the Troubleshooting section before proceeding. + If you encounter a `rake aborted!` error message stating that PgBouncer is + failing to connect to PostgreSQL, it may be that your PgBouncer node's IP + address is missing from PostgreSQL's `trust_auth_cidr_addresses` in `gitlab.rb` + on your database nodes. Before proceeding, see + [PgBouncer error `ERROR: pgbouncer cannot connect to server`](troubleshooting.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server). 1. [Configure fast lookup of authorized SSH keys in the database](../operations/fast_ssh_key_lookup.md). @@ -1636,10 +1623,10 @@ The Omnibus GitLab package can be used to configure a standalone Monitoring node running [Prometheus](../monitoring/prometheus/index.md) and [Grafana](../monitoring/performance/grafana_configuration.md): -1. SSH into the Monitoring node. -1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab - package you want using **steps 1 and 2** from the GitLab downloads page. - Do not complete any other steps on the download page. +1. SSH in to the Monitoring node. +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to follow _only_ installation steps 1 and 2 + on the page. 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby @@ -1714,28 +1701,44 @@ It's recommended over [NFS](#configure-nfs-optional) and in general it's better in larger setups as object storage is typically much more performant, reliable, and scalable. -Object storage options that GitLab has tested, or is aware of customers using include: +GitLab has been tested on a number of object storage providers: -- SaaS/Cloud solutions such as [Amazon S3](https://aws.amazon.com/s3/), [Google cloud storage](https://cloud.google.com/storage). +- [Amazon S3](https://aws.amazon.com/s3/) +- [Google Cloud Storage](https://cloud.google.com/storage) +- [Digital Ocean Spaces](https://www.digitalocean.com/products/spaces/) +- [Oracle Cloud Infrastructure](https://docs.cloud.oracle.com/en-us/iaas/Content/Object/Tasks/s3compatibleapi.htm) +- [Openstack Swift](https://docs.openstack.org/swift/latest/s3_compat.html) +- [Azure Blob storage](https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blobs-introduction) - On-premises hardware and appliances from various storage vendors. -- MinIO. There is [a guide to deploying this](https://docs.gitlab.com/charts/advanced/external-object-storage/minio.html) within our Helm Chart documentation. - -For configuring GitLab to use Object Storage refer to the following guides -based on what features you intend to use: - -1. Configure [object storage for backups](../../raketasks/backup_restore.md#uploading-backups-to-a-remote-cloud-storage). -1. Configure [object storage for job artifacts](../job_artifacts.md#using-object-storage) - including [incremental logging](../job_logs.md#new-incremental-logging-architecture). -1. Configure [object storage for LFS objects](../lfs/index.md#storing-lfs-objects-in-remote-object-storage). -1. Configure [object storage for uploads](../uploads.md#using-object-storage). -1. Configure [object storage for merge request diffs](../merge_request_diffs.md#using-object-storage). -1. Configure [object storage for Container Registry](../packages/container_registry.md#use-object-storage) (optional feature). -1. Configure [object storage for Mattermost](https://docs.mattermost.com/administration/config-settings.html#file-storage) (optional feature). -1. Configure [object storage for packages](../packages/index.md#using-object-storage) (optional feature). **(PREMIUM ONLY)** -1. Configure [object storage for Dependency Proxy](../packages/dependency_proxy.md#using-object-storage) (optional feature). **(PREMIUM ONLY)** -1. Configure [object storage for Pseudonymizer](../pseudonymizer.md#configuration) (optional feature). **(ULTIMATE ONLY)** -1. Configure [object storage for autoscale runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional - for improved performance). -1. Configure [object storage for Terraform state files](../terraform_state.md#using-object-storage). +- MinIO. We have [a guide to deploying this](https://docs.gitlab.com/charts/advanced/external-object-storage/minio.html) within our Helm Chart documentation. + +There are two ways of specifying object storage configuration in GitLab: + +- [Consolidated form](../object_storage.md#consolidated-object-storage-configuration): A single credential is + shared by all supported object types. +- [Storage-specific form](../object_storage.md#storage-specific-configuration): Every object defines its + own object storage [connection and configuration](../object_storage.md#connection-settings). + +Starting with GitLab 13.2, consolidated object storage configuration is available. It simplifies your GitLab configuration since the connection details are shared across object types. Refer to [Consolidated object storage configuration](../object_storage.md#consolidated-object-storage-configuration) guide for instructions on how to set it up. + +For configuring object storage in GitLab 13.1 and earlier, or for storage types not +supported by consolidated configuration form, refer to the following guides based +on what features you intend to use: + +|Object storage type|Supported by consolidated configuration?| +|-------------------|----------------------------------------| +| [Backups](../../raketasks/backup_restore.md#uploading-backups-to-a-remote-cloud-storage)|No| +| [Job artifacts](../job_artifacts.md#using-object-storage) including archived job logs | Yes | +| [LFS objects](../lfs/index.md#storing-lfs-objects-in-remote-object-storage) | Yes | +| [Uploads](../uploads.md#using-object-storage) | Yes | +| [Container Registry](../packages/container_registry.md#use-object-storage) (optional feature) | No | +| [Merge request diffs](../merge_request_diffs.md#using-object-storage) | Yes | +| [Mattermost](https://docs.mattermost.com/administration/config-settings.html#file-storage)| No | +| [Packages](../packages/index.md#using-object-storage) (optional feature) | Yes | +| [Dependency Proxy](../packages/dependency_proxy.md#using-object-storage) (optional feature) **(PREMIUM ONLY)** | Yes | +| [Pseudonymizer](../pseudonymizer.md#configuration) (optional feature) **(ULTIMATE ONLY)** | No | +| [Autoscale runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional for improved performance) | No | +| [Terraform state files](../terraform_state.md#using-object-storage) | Yes | Using separate buckets for each data type is the recommended approach for GitLab. @@ -1759,6 +1762,22 @@ work. </a> </div> +## Configure Advanced Search **(STARTER ONLY)** + +You can leverage Elasticsearch and [enable Advanced Search](../../integration/elasticsearch.md) +for faster, more advanced code search across your entire GitLab instance. + +Elasticsearch cluster design and requirements are dependent on your specific +data. For recommended best practices about how to set up your Elasticsearch +cluster alongside your instance, read how to +[choose the optimal cluster configuration](../../integration/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration). + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + ## Configure NFS (optional) [Object storage](#configure-the-object-storage), along with [Gitaly](#configure-gitaly) diff --git a/doc/administration/reference_architectures/50k_users.md b/doc/administration/reference_architectures/50k_users.md index e812eed0227..44fbe2db504 100644 --- a/doc/administration/reference_architectures/50k_users.md +++ b/doc/administration/reference_architectures/50k_users.md @@ -17,21 +17,21 @@ full list of reference architectures, see | Service | Nodes | Configuration | GCP | AWS | Azure | |-----------------------------------------|-------------|-------------------------|-----------------|--------------|----------| -| External load balancing node | 1 | 8 vCPU, 7.2GB memory | n1-highcpu-8 | c5.2xlarge | F8s v2 | -| Consul | 3 | 2 vCPU, 1.8GB memory | n1-highcpu-2 | c5.large | F2s v2 | -| PostgreSQL | 3 | 16 vCPU, 60GB memory | n1-standard-16 | m5.4xlarge | D16s v3 | -| PgBouncer | 3 | 2 vCPU, 1.8GB memory | n1-highcpu-2 | c5.large | F2s v2 | -| Internal load balancing node | 1 | 8 vCPU, 7.2GB memory | n1-highcpu-8 | c5.2xlarge | F8s v2 | -| Redis - Cache | 3 | 4 vCPU, 15GB memory | n1-standard-4 | m5.xlarge | D4s v3 | -| Redis - Queues / Shared State | 3 | 4 vCPU, 15GB memory | n1-standard-4 | m5.xlarge | D4s v3 | -| Redis Sentinel - Cache | 3 | 1 vCPU, 1.7GB memory | g1-small | t2.small | B1MS | -| Redis Sentinel - Queues / Shared State | 3 | 1 vCPU, 1.7GB memory | g1-small | t2.small | B1MS | -| Gitaly | 2 (minimum) | 64 vCPU, 240GB memory | n1-standard-64 | m5.16xlarge | D64s v3 | -| Sidekiq | 4 | 4 vCPU, 15GB memory | n1-standard-4 | m5.xlarge | D4s v3 | -| GitLab Rails | 12 | 32 vCPU, 28.8GB memory | n1-highcpu-32 | c5.9xlarge | F32s v2 | -| Monitoring node | 1 | 4 vCPU, 3.6GB memory | n1-highcpu-4 | c5.xlarge | F4s v2 | -| Object Storage | n/a | n/a | n/a | n/a | n/a | -| NFS Server | 1 | 4 vCPU, 3.6GB memory | n1-highcpu-4 | c5.xlarge | F4s v2 | +| External load balancing node | 1 | 8 vCPU, 7.2 GB memory | n1-highcpu-8 | c5.2xlarge | F8s v2 | +| Consul | 3 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| PostgreSQL | 3 | 16 vCPU, 60 GB memory | n1-standard-16 | m5.4xlarge | D16s v3 | +| PgBouncer | 3 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| Internal load balancing node | 1 | 8 vCPU, 7.2 GB memory | n1-highcpu-8 | c5.2xlarge | F8s v2 | +| Redis - Cache | 3 | 4 vCPU, 15 GB memory | n1-standard-4 | m5.xlarge | D4s v3 | +| Redis - Queues / Shared State | 3 | 4 vCPU, 15 GB memory | n1-standard-4 | m5.xlarge | D4s v3 | +| Redis Sentinel - Cache | 3 | 1 vCPU, 1.7 GB memory | g1-small | t2.small | B1MS | +| Redis Sentinel - Queues / Shared State | 3 | 1 vCPU, 1.7 GB memory | g1-small | t2.small | B1MS | +| Gitaly | 2 (minimum) | 64 vCPU, 240 GB memory | n1-standard-64 | m5.16xlarge | D64s v3 | +| Sidekiq | 4 | 4 vCPU, 15 GB memory | n1-standard-4 | m5.xlarge | D4s v3 | +| GitLab Rails | 12 | 32 vCPU, 28.8 GB memory | n1-highcpu-32 | c5.9xlarge | F32s v2 | +| Monitoring node | 1 | 4 vCPU, 3.6 GB memory | n1-highcpu-4 | c5.xlarge | F4s v2 | +| Object storage | n/a | n/a | n/a | n/a | n/a | +| NFS server | 1 | 4 vCPU, 3.6 GB memory | n1-highcpu-4 | c5.xlarge | F4s v2 | The Google Cloud Platform (GCP) architectures were built and tested using the [Intel Xeon E5 v3 (Haswell)](https://cloud.google.com/compute/docs/cpu-platforms) @@ -40,41 +40,43 @@ or higher, are required for your CPU or node counts. For more information, see our [Sysbench](https://github.com/akopytov/sysbench)-based [CPU benchmark](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Reference-Architectures/GCP-CPU-Benchmarks). -For data objects (such as LFS, Uploads, or Artifacts), an -[object storage service](#configure-the-object-storage) is recommended instead -of NFS where possible, due to better performance and availability. Since this -doesn't require a node to be set up, *Object Storage* is noted as not -applicable (n/a) in the previous table. +Due to better performance and availability, for data objects (such as LFS, +uploads, or artifacts), using an [object storage service](#configure-the-object-storage) +is recommended instead of using NFS. Using an object storage service also +doesn't require you to provision and maintain a node. ## Setup components To set up GitLab and its components to accommodate up to 50,000 users: 1. [Configure the external load balancing node](#configure-the-external-load-balancer) - that will handle the load balancing of the three GitLab application services nodes. + to handle the load balancing of the GitLab application services nodes. 1. [Configure Consul](#configure-consul). 1. [Configure PostgreSQL](#configure-postgresql), the database for GitLab. 1. [Configure PgBouncer](#configure-pgbouncer). -1. [Configure the internal load balancing node](#configure-the-internal-load-balancer) +1. [Configure the internal load balancing node](#configure-the-internal-load-balancer). 1. [Configure Redis](#configure-redis). 1. [Configure Gitaly](#configure-gitaly), which provides access to the Git repositories. 1. [Configure Sidekiq](#configure-sidekiq). 1. [Configure the main GitLab Rails application](#configure-gitlab-rails) - to run Puma/Unicorn, Workhorse, GitLab Shell, and to serve all frontend requests (UI, API, Git - over HTTP/SSH). -1. [Configure Prometheus](#configure-prometheus) to monitor your GitLab environment. -1. [Configure the Object Storage](#configure-the-object-storage) + to run Puma/Unicorn, Workhorse, GitLab Shell, and to serve all frontend + requests (which include UI, API, and Git over HTTP/SSH). +1. [Configure Prometheus](#configure-prometheus) to monitor your GitLab + environment. +1. [Configure the object storage](#configure-the-object-storage) used for shared data objects. -1. [Configure NFS (Optional)](#configure-nfs-optional) - to have shared disk storage service as an alternative to Gitaly and/or Object Storage (although - not recommended). NFS is required for GitLab Pages, you can skip this step if you're not using - that feature. +1. [Configure Advanced Search](#configure-advanced-search) (optional) for faster, + more advanced code search across your entire GitLab instance. +1. [Configure NFS](#configure-nfs-optional) (optional, and not recommended) + to have shared disk storage service as an alternative to Gitaly or object + storage. You can skip this step if you're not using GitLab Pages (which + requires NFS). -We start with all servers on the same 10.6.0.0/24 private network range, they -can connect to each other freely on those addresses. +The servers start on the same 10.6.0.0/24 private network range, and can +connect to each other freely on these addresses. -Here is a list and description of each machine and the assigned IP: +The following list includes descriptions of each server and its assigned IP: - `10.6.0.10`: External Load Balancer - `10.6.0.11`: Consul 1 @@ -112,19 +114,18 @@ Here is a list and description of each machine and the assigned IP: ## Configure the external load balancer -NOTE: **Note:** +In an active/active GitLab configuration, you'll need a load balancer to route +traffic to the application servers. The specifics on which load balancer to use +or its exact configuration is beyond the scope of GitLab documentation. We hope +that if you're managing multi-node systems like GitLab, you already have a load +balancer of choice. Some load balancer examples include HAProxy (open-source), +F5 Big-IP LTM, and Citrix Net Scaler. This documentation outline the ports and +protocols needed for use with GitLab. + This architecture has been tested and validated with [HAProxy](https://www.haproxy.org/) as the load balancer. Although other load balancers with similar feature sets could also be used, those load balancers have not been validated. -In an active/active GitLab configuration, you will need a load balancer to route -traffic to the application servers. The specifics on which load balancer to use -or the exact configuration is beyond the scope of GitLab documentation. We hope -that if you're managing multi-node systems like GitLab you have a load balancer of -choice already. Some examples including HAProxy (open-source), F5 Big-IP LTM, -and Citrix Net Scaler. This documentation will outline what ports and protocols -you need to use with GitLab. - The next question is how you will handle SSL in your environment. There are several different options: @@ -247,14 +248,11 @@ with the other servers. To configure Consul: -1. SSH into the server that will host Consul. -1. [Download/install](https://about.gitlab.com/install/) the - Omnibus GitLab Enterprise Edition package using **steps 1 and 2** from the - GitLab downloads page. - - Make sure you select the correct Omnibus package, with the same version - the GitLab application is running. - - Do not complete any other steps on the download page. - +1. SSH in to the server that will host Consul. +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to both follow _only_ installation steps 1 and 2 + on the page, and to select the correct Omnibus GitLab package, with the same version + and type (Community or Enterprise editions) as your current install. 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby @@ -282,10 +280,9 @@ To configure Consul: 1. Go through the steps again for all the other Consul nodes, and make sure you set up the correct IPs. -NOTE: **Note:** -A Consul leader will be elected when the provisioning of the third Consul server is completed. -Viewing the Consul logs `sudo gitlab-ctl tail consul` will display -`...[INFO] consul: New leader elected: ...` +A Consul leader is _elected_ when the provisioning of the third Consul server is +complete. Viewing the Consul logs `sudo gitlab-ctl tail consul` displays +`...[INFO] consul: New leader elected: ...`. You can list the current Consul members (server, client): @@ -352,7 +349,7 @@ in the second step, do not supply the `EXTERNAL_URL` value. #### PostgreSQL primary node -1. SSH into the PostgreSQL primary node. +1. SSH in to the PostgreSQL primary node. 1. Generate a password hash for the PostgreSQL username/password pair. This assumes you will use the default username of `gitlab` (recommended). The command will request a password and confirmation. Use the value that is output by this command in the next @@ -513,7 +510,7 @@ are supported and can be added if needed. #### PostgreSQL post-configuration -SSH into the **primary node**: +SSH in to the **primary node**: 1. Open a database prompt: @@ -548,7 +545,7 @@ SSH into the **primary node**: is not an IP address, it will need to be a resolvable name (via DNS or `/etc/hosts`) -SSH into the **secondary node**: +SSH in to the **secondary node**: 1. Set up the repmgr standby: @@ -662,7 +659,6 @@ The following IPs will be used as an example: 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. - NOTE: **Note:** If an error `execute[generate databases.ini]` occurs, this is due to an existing [known issue](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/4713). It will be resolved when you run a second `reconfigure` after the next step. @@ -796,35 +792,31 @@ to be used with GitLab. The following IPs will be used as an example: - `10.6.0.82`: Sentinel - Queues 2 - `10.6.0.83`: Sentinel - Queues 3 -NOTE: **Providing your own Redis instance:** -Managed Redis from cloud providers such as AWS ElastiCache will work. If these -services support high availability, be sure it is **not** the Redis Cluster type. -Redis version 5.0 or higher is required, as this is what ships with -Omnibus GitLab packages starting with GitLab 13.0. Older Redis versions -do not support an optional count argument to SPOP which is now required for -[Merge Trains](../../ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md). +### Providing your own Redis instance + +Managed Redis from cloud providers (such as AWS ElastiCache) will work. If these +services support high availability, be sure it _isn't_ of the Redis Cluster type. +Redis version 5.0 or higher is required, which is included with Omnibus GitLab +packages starting with GitLab 13.0. Older Redis versions don't support an +optional count argument to SPOP, which is required for [Merge Trains](../../ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md). Note the Redis node's IP address or hostname, port, and password (if required). -These will be necessary when configuring the -[GitLab application servers](#configure-gitlab-rails) later. +These will be necessary later when configuring the [GitLab application servers](#configure-gitlab-rails). ### Configure the Redis and Sentinel Cache cluster This is the section where we install and set up the new Redis Cache instances. -NOTE: **Note:** -Redis nodes (both primary and replica) will need the same password defined in -`redis['password']`. At any time during a failover the Sentinels can -reconfigure a node and change its status from primary to replica and vice versa. +Both the primary and replica Redis nodes need the same password defined in +`redis['password']`. At any time during a failover, the Sentinels can reconfigure +a node and change its status from primary to replica (and vice versa). #### Configure the primary Redis Cache node -1. SSH into the **Primary** Redis server. -1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab - package you want using **steps 1 and 2** from the GitLab downloads page. - - Make sure you select the correct Omnibus package, with the same version - and type (Community, Enterprise editions) of your current install. - - Do not complete any other steps on the download page. - +1. SSH in to the **Primary** Redis server. +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to both follow _only_ installation steps 1 and 2 + on the page, and to select the correct Omnibus GitLab package, with the same version + and type (Community or Enterprise editions) as your current install. 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby @@ -868,20 +860,17 @@ reconfigure a node and change its status from primary to replica and vice versa. 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. -NOTE: **Note:** -You can specify multiple roles like sentinel and Redis as: -`roles ['redis_sentinel_role', 'redis_master_role']`. -Read more about [roles](https://docs.gitlab.com/omnibus/roles/). +You can specify multiple roles, like sentinel and Redis, as: +`roles ['redis_sentinel_role', 'redis_master_role']`. Read more about +[roles](https://docs.gitlab.com/omnibus/roles/). #### Configure the replica Redis Cache nodes -1. SSH into the **replica** Redis server. -1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab - package you want using **steps 1 and 2** from the GitLab downloads page. - - Make sure you select the correct Omnibus package, with the same version - and type (Community, Enterprise editions) of your current install. - - Do not complete any other steps on the download page. - +1. SSH in to the **replica** Redis server. +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to both follow _only_ installation steps 1 and 2 + on the page, and to select the correct Omnibus GitLab package, with the same version + and type (Community or Enterprise editions) as your current install. 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby @@ -934,10 +923,9 @@ Read more about [roles](https://docs.gitlab.com/omnibus/roles/). 1. Go through the steps again for all the other replica nodes, and make sure to set up the IPs correctly. -NOTE: **Note:** -You can specify multiple roles like sentinel and Redis as: -`roles ['redis_sentinel_role', 'redis_master_role']`. -Read more about [roles](https://docs.gitlab.com/omnibus/roles/). +You can specify multiple roles, like sentinel and Redis, as: +`roles ['redis_sentinel_role', 'redis_master_role']`. Read more about +[roles](https://docs.gitlab.com/omnibus/roles/). These values don't have to be changed again in `/etc/gitlab/gitlab.rb` after a failover, as the nodes will be managed by the [Sentinels](#configure-the-sentinel-cache-nodes), and even after a @@ -955,13 +943,6 @@ are supported and can be added if needed. #### Configure the Sentinel Cache nodes -NOTE: **Note:** -If you are using an external Redis Sentinel instance, be sure -to exclude the `requirepass` parameter from the Sentinel -configuration. This parameter will cause clients to report `NOAUTH -Authentication required.`. [Redis Sentinel 3.2.x does not support -password authentication](https://github.com/antirez/redis/issues/3279). - Now that the Redis servers are all set up, let's configure the Sentinel servers. The following IPs will be used as an example: @@ -969,16 +950,19 @@ servers. The following IPs will be used as an example: - `10.6.0.72`: Sentinel - Cache 2 - `10.6.0.73`: Sentinel - Cache 3 -To configure the Sentinel Cache server: +NOTE: **Note:** +If you're using an external Redis Sentinel instance, be sure to exclude the +`requirepass` parameter from the Sentinel configuration. This parameter causes +clients to report `NOAUTH Authentication required.`. +[Redis Sentinel 3.2.x doesn't support password authentication](https://github.com/antirez/redis/issues/3279). -1. SSH into the server that will host Consul/Sentinel. -1. [Download/install](https://about.gitlab.com/install/) the - Omnibus GitLab Enterprise Edition package using **steps 1 and 2** from the - GitLab downloads page. - - Make sure you select the correct Omnibus package, with the same version - the GitLab application is running. - - Do not complete any other steps on the download page. +To configure the Sentinel Cache server: +1. SSH in to the server that will host Consul/Sentinel. +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to both follow _only_ installation steps 1 and 2 + on the page, and to select the correct Omnibus GitLab package, with the same version + and type (Community or Enterprise editions) as your current install. 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby @@ -1081,20 +1065,17 @@ To configure the Sentinel Cache server: This is the section where we install and set up the new Redis Queues instances. -NOTE: **Note:** -Redis nodes (both primary and replica) will need the same password defined in -`redis['password']`. At any time during a failover the Sentinels can -reconfigure a node and change its status from primary to replica and vice versa. +Both the primary and replica Redis nodes need the same password defined in +`redis['password']`. At any time during a failover, the Sentinels can reconfigure +a node and change its status from primary to replica (and vice versa). #### Configure the primary Redis Queues node -1. SSH into the **Primary** Redis server. -1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab - package you want using **steps 1 and 2** from the GitLab downloads page. - - Make sure you select the correct Omnibus package, with the same version - and type (Community, Enterprise editions) of your current install. - - Do not complete any other steps on the download page. - +1. SSH in to the **Primary** Redis server. +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to both follow _only_ installation steps 1 and 2 + on the page, and to select the correct Omnibus GitLab package, with the same version + and type (Community or Enterprise editions) as your current install. 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby @@ -1143,20 +1124,17 @@ reconfigure a node and change its status from primary to replica and vice versa. 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. -NOTE: **Note:** -You can specify multiple roles like sentinel and Redis as: -`roles ['redis_sentinel_role', 'redis_master_role']`. -Read more about [roles](https://docs.gitlab.com/omnibus/roles/). +You can specify multiple roles, like sentinel and Redis, as: +`roles ['redis_sentinel_role', 'redis_master_role']`. Read more about +[roles](https://docs.gitlab.com/omnibus/roles/). #### Configure the replica Redis Queues nodes -1. SSH into the **replica** Redis Queue server. -1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab - package you want using **steps 1 and 2** from the GitLab downloads page. - - Make sure you select the correct Omnibus package, with the same version - and type (Community, Enterprise editions) of your current install. - - Do not complete any other steps on the download page. - +1. SSH in to the **replica** Redis Queue server. +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to both follow _only_ installation steps 1 and 2 + on the page, and to select the correct Omnibus GitLab package, with the same version + and type (Community or Enterprise editions) as your current install. 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby @@ -1209,10 +1187,9 @@ Read more about [roles](https://docs.gitlab.com/omnibus/roles/). 1. Go through the steps again for all the other replica nodes, and make sure to set up the IPs correctly. -NOTE: **Note:** -You can specify multiple roles like sentinel and Redis as: -`roles ['redis_sentinel_role', 'redis_master_role']`. -Read more about [roles](https://docs.gitlab.com/omnibus/roles/). +You can specify multiple roles, like sentinel and Redis, as: +`roles ['redis_sentinel_role', 'redis_master_role']`. Read more about +[roles](https://docs.gitlab.com/omnibus/roles/). These values don't have to be changed again in `/etc/gitlab/gitlab.rb` after a failover, as the nodes will be managed by the [Sentinels](#configure-the-sentinel-queues-nodes), and even after a @@ -1230,13 +1207,6 @@ are supported and can be added if needed. #### Configure the Sentinel Queues nodes -NOTE: **Note:** -If you are using an external Redis Sentinel instance, be sure -to exclude the `requirepass` parameter from the Sentinel -configuration. This parameter will cause clients to report `NOAUTH -Authentication required.`. [Redis Sentinel 3.2.x does not support -password authentication](https://github.com/antirez/redis/issues/3279). - Now that the Redis servers are all set up, let's configure the Sentinel servers. The following IPs will be used as an example: @@ -1244,16 +1214,19 @@ servers. The following IPs will be used as an example: - `10.6.0.82`: Sentinel - Queues 2 - `10.6.0.83`: Sentinel - Queues 3 -To configure the Sentinel Queues server: +NOTE: **Note:** +If you're using an external Redis Sentinel instance, be sure to exclude the +`requirepass` parameter from the Sentinel configuration. This parameter causes +clients to report `NOAUTH Authentication required.`. +[Redis Sentinel 3.2.x doesn't support password authentication](https://github.com/antirez/redis/issues/3279). -1. SSH into the server that will host Sentinel. -1. [Download/install](https://about.gitlab.com/install/) the - Omnibus GitLab Enterprise Edition package using **steps 1 and 2** from the - GitLab downloads page. - - Make sure you select the correct Omnibus package, with the same version - the GitLab application is running. - - Do not complete any other steps on the download page. +To configure the Sentinel Queues server: +1. SSH in to the server that will host Sentinel. +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to both follow _only_ installation steps 1 and 2 + on the page, and to select the correct Omnibus GitLab package, with the same version + and type (Community or Enterprise editions) as your current install. 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby @@ -1362,52 +1335,48 @@ To configure the Sentinel Queues server: ## Configure Gitaly -Deploying Gitaly in its own server can benefit GitLab installations that are -larger than a single machine. - -The Gitaly node requirements are dependent on customer data, specifically the number of -projects and their repository sizes. Two nodes are recommended as an absolute minimum. -Each Gitaly node should store no more than 5TB of data and have the number of -[`gitaly-ruby` workers](../gitaly/index.md#gitaly-ruby) set to 20% of available CPUs. -Additional nodes should be considered in conjunction with a review of expected -data size and spread based on the recommendations above. - -It is also strongly recommended that all Gitaly nodes be set up with SSD disks with -a throughput of at least 8,000 IOPS for read operations and 2,000 IOPS for write, -as Gitaly has heavy I/O. These IOPS values are recommended only as a starter as with -time they may be adjusted higher or lower depending on the scale of your environment's workload. -If you're running the environment on a Cloud provider, you may need to refer to -their documentation on how to configure IOPS correctly. - -Some things to note: - -- The GitLab Rails application shards repositories into [repository storages](../repository_storage_paths.md). -- A Gitaly server can host one or more storages. -- A GitLab server can use one or more Gitaly servers. -- Gitaly addresses must be specified in such a way that they resolve - correctly for ALL Gitaly clients. +[Gitaly](../gitaly/index.md) server node requirements are dependent on data, +specifically the number of projects and those projects' sizes. It's recommended +that a Gitaly server node stores no more than 5 TB of data. Although this +reference architecture includes a recommendation for the number of Gitaly server +nodes to use, depending on your storage requirements, you may require additional +Gitaly server nodes. + +Due to Gitaly having notable input and output requirements, we strongly +recommend that all Gitaly nodes use solid-state drives (SSDs). These SSDs should +have a throughput of at least 8,000 input/output operations per second (IOPS) +for read operations and 2,000 IOPS for write operations. These IOPS values are +initial recommendations, and may be adjusted to greater or lesser values +depending on the scale of your environment's workload. If you're running the +environment on a Cloud provider, refer to their documentation about how to +configure IOPS correctly. + +Be sure to note the following items: + +- The GitLab Rails application shards repositories into + [repository storage paths](../repository_storage_paths.md). +- A Gitaly server can host one or more storage paths. +- A GitLab server can use one or more Gitaly server nodes. +- Gitaly addresses must be specified to be correctly resolvable for _all_ + Gitaly clients. - Gitaly servers must not be exposed to the public internet, as Gitaly's network traffic is unencrypted by default. The use of a firewall is highly recommended to restrict access to the Gitaly server. Another option is to [use TLS](#gitaly-tls-support). -TIP: **Tip:** -For more information about Gitaly's history and network architecture see the -[standalone Gitaly documentation](../gitaly/index.md). - -Note: **Note:** -The token referred to throughout the Gitaly documentation is -just an arbitrary password selected by the administrator. It is unrelated to -tokens created for the GitLab API or other similar web API tokens. +NOTE: **Note:** +The token referred to throughout the Gitaly documentation is an arbitrary +password selected by the administrator. This token is unrelated to tokens +created for the GitLab API or other similar web API tokens. -Below we describe how to configure two Gitaly servers, with IPs and -domain names: +This section describes how to configure two Gitaly servers, with the following +IPs and domain names: - `10.6.0.91`: Gitaly 1 (`gitaly1.internal`) - `10.6.0.92`: Gitaly 2 (`gitaly2.internal`) -The secret token is assumed to be `gitalysecret` and that -your GitLab installation has three repository storages: +Assumptions about your servers include having the secret token be `gitalysecret`, +and that your GitLab installation has three repository storages: - `default` on Gitaly 1 - `storage1` on Gitaly 1 @@ -1415,11 +1384,11 @@ your GitLab installation has three repository storages: On each node: -1. [Download/Install](https://about.gitlab.com/install/) the Omnibus GitLab - package you want using **steps 1 and 2** from the GitLab downloads page but - **without** providing the `EXTERNAL_URL` value. -1. Edit `/etc/gitlab/gitlab.rb` to configure storage paths, enable - the network listener and configure the token: +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to follow _only_ installation steps 1 and 2 + on the page, and _do not_ provide the `EXTERNAL_URL` value. +1. Edit `/etc/gitlab/gitlab.rb` to configure the storage paths, enable + the network listener, and configure the token: <!-- updates to following example must also be made at @@ -1467,39 +1436,39 @@ On each node: ``` 1. Append the following to `/etc/gitlab/gitlab.rb` for each respective server: - 1. On `gitaly1.internal`: - - ```ruby - git_data_dirs({ - 'default' => { - 'path' => '/var/opt/gitlab/git-data' - }, - 'storage1' => { - 'path' => '/mnt/gitlab/git-data' - }, - }) - ``` - - 1. On `gitaly2.internal`: - - ```ruby - git_data_dirs({ - 'storage2' => { - 'path' => '/mnt/gitlab/git-data' - }, - }) - ``` + - On `gitaly1.internal`: + + ```ruby + git_data_dirs({ + 'default' => { + 'path' => '/var/opt/gitlab/git-data' + }, + 'storage1' => { + 'path' => '/mnt/gitlab/git-data' + }, + }) + ``` + + - On `gitaly2.internal`: + + ```ruby + git_data_dirs({ + 'storage2' => { + 'path' => '/mnt/gitlab/git-data' + }, + }) + ``` <!-- updates to following example must also be made at https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab --> -1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace - the file of the same name on this server. If that file is not on this server, - add the file from your Consul server to this server. +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and + then replace the file of the same name on this server. If that file isn't on + this server, add the file from your Consul server to this server. -1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). +1. Save the file, and then [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). ### Gitaly TLS support @@ -1521,11 +1490,10 @@ Name. If you are addressing the Gitaly server by its IP address, you must add it as a Subject Alternative Name to the certificate. [gRPC does not support using an IP address as Common Name in a certificate](https://github.com/grpc/grpc/issues/2691). -NOTE: **Note:** -It is possible to configure Gitaly servers with both an -unencrypted listening address `listen_addr` and an encrypted listening -address `tls_listen_addr` at the same time. This allows you to do a -gradual transition from unencrypted to encrypted traffic, if necessary. +It's possible to configure Gitaly servers with both an unencrypted listening +address (`listen_addr`) and an encrypted listening address (`tls_listen_addr`) +at the same time. This allows you to do a gradual transition from unencrypted to +encrypted traffic, if necessary. To configure Gitaly with TLS: @@ -1580,10 +1548,10 @@ The following IPs will be used as an example: To configure the Sidekiq nodes, on each one: -1. SSH into the Sidekiq server. -1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab package -you want using steps 1 and 2 from the GitLab downloads page. -**Do not complete any other steps on the download page.** +1. SSH in to the Sidekiq server. +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to follow _only_ installation steps 1 and 2 + on the page. 1. Open `/etc/gitlab/gitlab.rb` with your editor: ```ruby @@ -1701,15 +1669,14 @@ You can also run [multiple Sidekiq processes](../operations/extra_sidekiq_proces ## Configure GitLab Rails -NOTE: **Note:** -In our architectures we run each GitLab Rails node using the Puma webserver -and have its number of workers set to 90% of available CPUs along with four threads. For -nodes that are running Rails with other components the worker value should be reduced -accordingly where we've found 50% achieves a good balance but this is dependent -on workload. - This section describes how to configure the GitLab application (Rails) component. +In our architecture, we run each GitLab Rails node using the Puma webserver, and +have its number of workers set to 90% of available CPUs, with four threads. For +nodes running Rails with other components, the worker value should be reduced +accordingly. We've determined that a worker value of 50% achieves a good balance, +but this is dependent on workload. + The following IPs will be used as an example: - `10.6.0.111`: GitLab application 1 @@ -1718,10 +1685,9 @@ The following IPs will be used as an example: On each node perform the following: -1. Download/install Omnibus GitLab using **steps 1 and 2** from - [GitLab downloads](https://about.gitlab.com/install/). Do not complete other - steps on the download page. - +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to follow _only_ installation steps 1 and 2 + on the page. 1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace the file of the same name on this server. If that file is not on this server, add the file from your Consul server to this server. @@ -1752,6 +1718,7 @@ On each node perform the following: roles ['application_role'] gitaly['enable'] = false nginx['enable'] = true + sidekiq['enable'] = false ## PostgreSQL connection details # Disable PostgreSQL on the application node @@ -1795,7 +1762,6 @@ On each node perform the following: # Set the network addresses that the exporters used for monitoring will listen on node_exporter['listen_address'] = '0.0.0.0:9100' gitlab_workhorse['prometheus_listen_addr'] = '0.0.0.0:9229' - sidekiq['listen_address'] = "0.0.0.0" puma['listen'] = '0.0.0.0' # Add the monitoring node's IP address to the monitoring whitelist and allow it to @@ -1836,7 +1802,7 @@ On each node perform the following: 1. Specify the necessary NFS mounts in `/etc/fstab`. The exact contents of `/etc/fstab` will depend on how you chose - to configure your NFS server. See the [NFS documentation](../high_availability/nfs.md) + to configure your NFS server. See the [NFS documentation](../nfs.md) for examples and the various options. 1. Create the shared directories. These may be different depending on your NFS @@ -1877,30 +1843,31 @@ On each node perform the following: 1. Optionally, from the Gitaly servers, confirm that Gitaly can perform callbacks to the internal API: ```shell - sudo /opt/gitlab/embedded/service/gitlab-shell/bin/check -config /opt/gitlab/embedded/service/gitlab-shell/config.yml + sudo /opt/gitlab/embedded/bin/gitaly-hooks check /var/opt/gitlab/gitaly/config.toml ``` -NOTE: **Note:** -When you specify `https` in the `external_url`, as in the example -above, GitLab assumes you have SSL certificates in `/etc/gitlab/ssl/`. If -certificates are not present, NGINX will fail to start. See the -[NGINX documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https) -for more information. +When you specify `https` in the `external_url`, as in the previous example, +GitLab expects that the SSL certificates are in `/etc/gitlab/ssl/`. If the +certificates aren't present, NGINX will fail to start. For more information, see +the [NGINX documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https). ### GitLab Rails post-configuration -Initialize the GitLab database, by running the following in one of the Rails nodes: +1. Designate one application node for running database migrations during + installation and updates. Initialize the GitLab database and ensure all + migrations ran: -```shell -sudo gitlab-rake gitlab:db:configure -``` + ```shell + sudo gitlab-rake gitlab:db:configure + ``` -NOTE: **Note:** -If you encounter a `rake aborted!` error stating that PgBouncer is failing to connect to -PostgreSQL it may be that your PgBouncer node's IP address is missing from -PostgreSQL's `trust_auth_cidr_addresses` in `gitlab.rb` on your database nodes. See -[PgBouncer error `ERROR: pgbouncer cannot connect to server`](troubleshooting.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server) -in the Troubleshooting section before proceeding. + If you encounter a `rake aborted!` error message stating that PgBouncer is + failing to connect to PostgreSQL, it may be that your PgBouncer node's IP + address is missing from PostgreSQL's `trust_auth_cidr_addresses` in `gitlab.rb` + on your database nodes. Before proceeding, see + [PgBouncer error `ERROR: pgbouncer cannot connect to server`](troubleshooting.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server). + +1. [Configure fast lookup of authorized SSH keys in the database](../operations/fast_ssh_key_lookup.md). <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> @@ -1920,11 +1887,10 @@ The following IP will be used as an example: To configure the Monitoring node: -1. SSH into the Monitoring node. -1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab - package you want using **steps 1 and 2** from the GitLab downloads page. - Do not complete any other steps on the download page. - +1. SSH in to the Monitoring node. +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to follow _only_ installation steps 1 and 2 + on the page. 1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace the file of the same name on this server. If that file is not on this server, add the file from your Consul server to this server. @@ -1988,28 +1954,44 @@ It's recommended over [NFS](#configure-nfs-optional) and in general it's better in larger setups as object storage is typically much more performant, reliable, and scalable. -Object storage options that GitLab has tested, or is aware of customers using include: +GitLab has been tested on a number of object storage providers: -- SaaS/Cloud solutions such as [Amazon S3](https://aws.amazon.com/s3/), [Google cloud storage](https://cloud.google.com/storage). +- [Amazon S3](https://aws.amazon.com/s3/) +- [Google Cloud Storage](https://cloud.google.com/storage) +- [Digital Ocean Spaces](https://www.digitalocean.com/products/spaces/) +- [Oracle Cloud Infrastructure](https://docs.cloud.oracle.com/en-us/iaas/Content/Object/Tasks/s3compatibleapi.htm) +- [Openstack Swift](https://docs.openstack.org/swift/latest/s3_compat.html) +- [Azure Blob storage](https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blobs-introduction) - On-premises hardware and appliances from various storage vendors. -- MinIO. There is [a guide to deploying this](https://docs.gitlab.com/charts/advanced/external-object-storage/minio.html) within our Helm Chart documentation. - -For configuring GitLab to use Object Storage refer to the following guides -based on what features you intend to use: - -1. Configure [object storage for backups](../../raketasks/backup_restore.md#uploading-backups-to-a-remote-cloud-storage). -1. Configure [object storage for job artifacts](../job_artifacts.md#using-object-storage) - including [incremental logging](../job_logs.md#new-incremental-logging-architecture). -1. Configure [object storage for LFS objects](../lfs/index.md#storing-lfs-objects-in-remote-object-storage). -1. Configure [object storage for uploads](../uploads.md#using-object-storage). -1. Configure [object storage for merge request diffs](../merge_request_diffs.md#using-object-storage). -1. Configure [object storage for Container Registry](../packages/container_registry.md#use-object-storage) (optional feature). -1. Configure [object storage for Mattermost](https://docs.mattermost.com/administration/config-settings.html#file-storage) (optional feature). -1. Configure [object storage for packages](../packages/index.md#using-object-storage) (optional feature). **(PREMIUM ONLY)** -1. Configure [object storage for Dependency Proxy](../packages/dependency_proxy.md#using-object-storage) (optional feature). **(PREMIUM ONLY)** -1. Configure [object storage for Pseudonymizer](../pseudonymizer.md#configuration) (optional feature). **(ULTIMATE ONLY)** -1. Configure [object storage for autoscale runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional - for improved performance). -1. Configure [object storage for Terraform state files](../terraform_state.md#using-object-storage). +- MinIO. We have [a guide to deploying this](https://docs.gitlab.com/charts/advanced/external-object-storage/minio.html) within our Helm Chart documentation. + +There are two ways of specifying object storage configuration in GitLab: + +- [Consolidated form](../object_storage.md#consolidated-object-storage-configuration): A single credential is + shared by all supported object types. +- [Storage-specific form](../object_storage.md#storage-specific-configuration): Every object defines its + own object storage [connection and configuration](../object_storage.md#connection-settings). + +Starting with GitLab 13.2, consolidated object storage configuration is available. It simplifies your GitLab configuration since the connection details are shared across object types. Refer to [Consolidated object storage configuration](../object_storage.md#consolidated-object-storage-configuration) guide for instructions on how to set it up. + +For configuring object storage in GitLab 13.1 and earlier, or for storage types not +supported by consolidated configuration form, refer to the following guides based +on what features you intend to use: + +|Object storage type|Supported by consolidated configuration?| +|-------------------|----------------------------------------| +| [Backups](../../raketasks/backup_restore.md#uploading-backups-to-a-remote-cloud-storage)|No| +| [Job artifacts](../job_artifacts.md#using-object-storage) including archived job logs | Yes | +| [LFS objects](../lfs/index.md#storing-lfs-objects-in-remote-object-storage) | Yes | +| [Uploads](../uploads.md#using-object-storage) | Yes | +| [Container Registry](../packages/container_registry.md#use-object-storage) (optional feature) | No | +| [Merge request diffs](../merge_request_diffs.md#using-object-storage) | Yes | +| [Mattermost](https://docs.mattermost.com/administration/config-settings.html#file-storage)| No | +| [Packages](../packages/index.md#using-object-storage) (optional feature) | Yes | +| [Dependency Proxy](../packages/dependency_proxy.md#using-object-storage) (optional feature) **(PREMIUM ONLY)** | Yes | +| [Pseudonymizer](../pseudonymizer.md#configuration) (optional feature) **(ULTIMATE ONLY)** | No | +| [Autoscale runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional for improved performance) | No | +| [Terraform state files](../terraform_state.md#using-object-storage) | Yes | Using separate buckets for each data type is the recommended approach for GitLab. @@ -2033,13 +2015,29 @@ work. </a> </div> +## Configure Advanced Search **(STARTER ONLY)** + +You can leverage Elasticsearch and [enable Advanced Search](../../integration/elasticsearch.md) +for faster, more advanced code search across your entire GitLab instance. + +Elasticsearch cluster design and requirements are dependent on your specific +data. For recommended best practices about how to set up your Elasticsearch +cluster alongside your instance, read how to +[choose the optimal cluster configuration](../../integration/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration). + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + ## Configure NFS (optional) [Object storage](#configure-the-object-storage), along with [Gitaly](#configure-gitaly) are recommended over NFS wherever possible for improved performance. If you intend to use GitLab Pages, this currently [requires NFS](troubleshooting.md#gitlab-pages-requires-nfs). -See how to [configure NFS](../high_availability/nfs.md). +See how to [configure NFS](../nfs.md). <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> diff --git a/doc/administration/reference_architectures/5k_users.md b/doc/administration/reference_architectures/5k_users.md index 6dfa588b092..9f83950a927 100644 --- a/doc/administration/reference_architectures/5k_users.md +++ b/doc/administration/reference_architectures/5k_users.md @@ -24,18 +24,18 @@ costly-to-operate environment by using the | Service | Nodes | Configuration | GCP | AWS | Azure | |--------------------------------------------|-------------|-------------------------|----------------|-------------|----------| -| External load balancing node | 1 | 2 vCPU, 1.8GB memory | n1-highcpu-2 | c5.large | F2s v2 | -| Redis | 3 | 2 vCPU, 7.5GB memory | n1-standard-2 | m5.large | D2s v3 | -| Consul + Sentinel | 3 | 2 vCPU, 1.8GB memory | n1-highcpu-2 | c5.large | F2s v2 | -| PostgreSQL | 3 | 2 vCPU, 7.5GB memory | n1-standard-2 | m5.large | D2s v3 | -| PgBouncer | 3 | 2 vCPU, 1.8GB memory | n1-highcpu-2 | c5.large | F2s v2 | -| Internal load balancing node | 1 | 2 vCPU, 1.8GB memory | n1-highcpu-2 | c5.large | F2s v2 | -| Gitaly | 2 (minimum) | 8 vCPU, 30GB memory | n1-standard-8 | m5.2xlarge | D8s v3 | -| Sidekiq | 4 | 2 vCPU, 7.5GB memory | n1-standard-2 | m5.large | D2s v3 | -| GitLab Rails | 3 | 16 vCPU, 14.4GB memory | n1-highcpu-16 | c5.4xlarge | F16s v2 | -| Monitoring node | 1 | 2 vCPU, 1.8GB memory | n1-highcpu-2 | c5.large | F2s v2 | -| Object Storage | n/a | n/a | n/a | n/a | n/a | -| NFS Server (optional, not recommended) | 1 | 4 vCPU, 3.6GB memory | n1-highcpu-4 | c5.xlarge | F4s v2 | +| External load balancing node | 1 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| Redis | 3 | 2 vCPU, 7.5 GB memory | n1-standard-2 | m5.large | D2s v3 | +| Consul + Sentinel | 3 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| PostgreSQL | 3 | 2 vCPU, 7.5 GB memory | n1-standard-2 | m5.large | D2s v3 | +| PgBouncer | 3 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| Internal load balancing node | 1 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| Gitaly | 2 (minimum) | 8 vCPU, 30 GB memory | n1-standard-8 | m5.2xlarge | D8s v3 | +| Sidekiq | 4 | 2 vCPU, 7.5 GB memory | n1-standard-2 | m5.large | D2s v3 | +| GitLab Rails | 3 | 16 vCPU, 14.4 GB memory | n1-highcpu-16 | c5.4xlarge | F16s v2 | +| Monitoring node | 1 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| Object storage | n/a | n/a | n/a | n/a | n/a | +| NFS server (optional, not recommended) | 1 | 4 vCPU, 3.6 GB memory | n1-highcpu-4 | c5.xlarge | F4s v2 | The Google Cloud Platform (GCP) architectures were built and tested using the [Intel Xeon E5 v3 (Haswell)](https://cloud.google.com/compute/docs/cpu-platforms) @@ -44,41 +44,43 @@ or higher, are required for your CPU or node counts. For more information, see our [Sysbench](https://github.com/akopytov/sysbench)-based [CPU benchmark](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Reference-Architectures/GCP-CPU-Benchmarks). -For data objects (such as LFS, Uploads, or Artifacts), an -[object storage service](#configure-the-object-storage) is recommended instead -of NFS where possible, due to better performance and availability. Since this -doesn't require a node to be set up, *Object Storage* is noted as not -applicable (n/a) in the previous table. +Due to better performance and availability, for data objects (such as LFS, +uploads, or artifacts), using an [object storage service](#configure-the-object-storage) +is recommended instead of using NFS. Using an object storage service also +doesn't require you to provision and maintain a node. ## Setup components To set up GitLab and its components to accommodate up to 5,000 users: 1. [Configure the external load balancing node](#configure-the-external-load-balancer) - that will handle the load balancing of the two GitLab application services nodes. + to handle the load balancing of the GitLab application services nodes. 1. [Configure Redis](#configure-redis). 1. [Configure Consul and Sentinel](#configure-consul-and-sentinel). 1. [Configure PostgreSQL](#configure-postgresql), the database for GitLab. 1. [Configure PgBouncer](#configure-pgbouncer). -1. [Configure the internal load balancing node](#configure-the-internal-load-balancer) +1. [Configure the internal load balancing node](#configure-the-internal-load-balancer). 1. [Configure Gitaly](#configure-gitaly), which provides access to the Git repositories. 1. [Configure Sidekiq](#configure-sidekiq). 1. [Configure the main GitLab Rails application](#configure-gitlab-rails) - to run Puma/Unicorn, Workhorse, GitLab Shell, and to serve all frontend requests (UI, API, Git - over HTTP/SSH). -1. [Configure Prometheus](#configure-prometheus) to monitor your GitLab environment. -1. [Configure the Object Storage](#configure-the-object-storage) + to run Puma/Unicorn, Workhorse, GitLab Shell, and to serve all frontend + requests (which include UI, API, and Git over HTTP/SSH). +1. [Configure Prometheus](#configure-prometheus) to monitor your GitLab + environment. +1. [Configure the object storage](#configure-the-object-storage) used for shared data objects. -1. [Configure NFS (Optional)](#configure-nfs-optional) - to have shared disk storage service as an alternative to Gitaly and/or Object Storage (although - not recommended). NFS is required for GitLab Pages, you can skip this step if you're not using - that feature. +1. [Configure Advanced Search](#configure-advanced-search) (optional) for faster, + more advanced code search across your entire GitLab instance. +1. [Configure NFS](#configure-nfs-optional) (optional, and not recommended) + to have shared disk storage service as an alternative to Gitaly or object + storage. You can skip this step if you're not using GitLab Pages (which + requires NFS). -We start with all servers on the same 10.6.0.0/16 private network range, they -can connect to each other freely on those addresses. +The servers start on the same 10.6.0.0/24 private network range, and can +connect to each other freely on these addresses. -Here is a list and description of each machine and the assigned IP: +The following list includes descriptions of each server and its assigned IP: - `10.6.0.10`: External Load Balancer - `10.6.0.61`: Redis Primary @@ -107,19 +109,18 @@ Here is a list and description of each machine and the assigned IP: ## Configure the external load balancer -NOTE: **Note:** +In an active/active GitLab configuration, you'll need a load balancer to route +traffic to the application servers. The specifics on which load balancer to use +or its exact configuration is beyond the scope of GitLab documentation. We hope +that if you're managing multi-node systems like GitLab, you already have a load +balancer of choice. Some load balancer examples include HAProxy (open-source), +F5 Big-IP LTM, and Citrix Net Scaler. This documentation outline the ports and +protocols needed for use with GitLab. + This architecture has been tested and validated with [HAProxy](https://www.haproxy.org/) as the load balancer. Although other load balancers with similar feature sets could also be used, those load balancers have not been validated. -In an active/active GitLab configuration, you will need a load balancer to route -traffic to the application servers. The specifics on which load balancer to use -or the exact configuration is beyond the scope of GitLab documentation. We hope -that if you're managing multi-node systems like GitLab you have a load balancer of -choice already. Some examples including HAProxy (open-source), F5 Big-IP LTM, -and Citrix Net Scaler. This documentation will outline what ports and protocols -you need to use with GitLab. - The next question is how you will handle SSL in your environment. There are several different options: @@ -277,20 +278,17 @@ The requirements for a Redis setup are the following: ([Internet](https://gitlab.com/gitlab-org/gitlab-foss/uploads/c4cc8cd353604bd80315f9384035ff9e/The_Internet_IT_Crowd.png)), using a firewall. -NOTE: **Note:** -Redis nodes (both primary and replica) will need the same password defined in -`redis['password']`. At any time during a failover the Sentinels can -reconfigure a node and change its status from primary to replica and vice versa. +Both the primary and replica Redis nodes need the same password defined in +`redis['password']`. At any time during a failover, the Sentinels can reconfigure +a node and change its status from primary to replica (and vice versa). #### Configuring the primary Redis instance -1. SSH into the **Primary** Redis server. -1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab - package you want using **steps 1 and 2** from the GitLab downloads page. - - Make sure you select the correct Omnibus package, with the same version - and type (Community, Enterprise editions) of your current install. - - Do not complete any other steps on the download page. - +1. SSH in to the **Primary** Redis server. +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to both follow _only_ installation steps 1 and 2 + on the page, and to select the correct Omnibus GitLab package, with the same version + and type (Community or Enterprise editions) as your current install. 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby @@ -334,10 +332,9 @@ reconfigure a node and change its status from primary to replica and vice versa. 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. -NOTE: **Note:** -You can specify multiple roles like sentinel and Redis as: -`roles ['redis_sentinel_role', 'redis_master_role']`. -Read more about [roles](https://docs.gitlab.com/omnibus/roles/). +You can specify multiple roles, like sentinel and Redis, as: +`roles ['redis_sentinel_role', 'redis_master_role']`. Read more about +[roles](https://docs.gitlab.com/omnibus/roles/). You can list the current Redis Primary, Replica status via: @@ -363,13 +360,11 @@ run: redis-exporter: (pid 30075) 76861s; run: log: (pid 29674) 76896s #### Configuring the replica Redis instances -1. SSH into the **replica** Redis server. -1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab - package you want using **steps 1 and 2** from the GitLab downloads page. - - Make sure you select the correct Omnibus package, with the same version - and type (Community, Enterprise editions) of your current install. - - Do not complete any other steps on the download page. - +1. SSH in to the **replica** Redis server. +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to both follow _only_ installation steps 1 and 2 + on the page, and to select the correct Omnibus GitLab package, with the same version + and type (Community or Enterprise editions) as your current install. 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby @@ -422,10 +417,9 @@ run: redis-exporter: (pid 30075) 76861s; run: log: (pid 29674) 76896s 1. Go through the steps again for all the other replica nodes, and make sure to set up the IPs correctly. -NOTE: **Note:** -You can specify multiple roles like sentinel and Redis as: -`roles ['redis_sentinel_role', 'redis_master_role']`. -Read more about [roles](https://docs.gitlab.com/omnibus/roles/). +You can specify multiple roles, like sentinel and Redis, as: +`roles ['redis_sentinel_role', 'redis_master_role']`. Read more about +[roles](https://docs.gitlab.com/omnibus/roles/). These values don't have to be changed again in `/etc/gitlab/gitlab.rb` after a failover, as the nodes will be managed by the [Sentinels](#configure-consul-and-sentinel), and even after a @@ -443,13 +437,6 @@ are supported and can be added if needed. ## Configure Consul and Sentinel -NOTE: **Note:** -If you are using an external Redis Sentinel instance, be sure -to exclude the `requirepass` parameter from the Sentinel -configuration. This parameter will cause clients to report `NOAUTH -Authentication required.`. [Redis Sentinel 3.2.x does not support -password authentication](https://github.com/antirez/redis/issues/3279). - Now that the Redis servers are all set up, let's configure the Sentinel servers. The following IPs will be used as an example: @@ -457,16 +444,19 @@ servers. The following IPs will be used as an example: - `10.6.0.12`: Consul/Sentinel 2 - `10.6.0.13`: Consul/Sentinel 3 -To configure the Sentinel: +NOTE: **Note:** +If you're using an external Redis Sentinel instance, be sure to exclude the +`requirepass` parameter from the Sentinel configuration. This parameter causes +clients to report `NOAUTH Authentication required.`. +[Redis Sentinel 3.2.x doesn't support password authentication](https://github.com/antirez/redis/issues/3279). -1. SSH into the server that will host Consul/Sentinel. -1. [Download/install](https://about.gitlab.com/install/) the - Omnibus GitLab Enterprise Edition package using **steps 1 and 2** from the - GitLab downloads page. - - Make sure you select the correct Omnibus package, with the same version - the GitLab application is running. - - Do not complete any other steps on the download page. +To configure the Sentinel: +1. SSH in to the server that will host Consul/Sentinel. +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to both follow _only_ installation steps 1 and 2 + on the page, and to select the correct Omnibus GitLab package, with the same version + and type (Community or Enterprise editions) as your current install. 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby @@ -556,10 +546,9 @@ To configure the Sentinel: 1. Go through the steps again for all the other Consul/Sentinel nodes, and make sure you set up the correct IPs. -NOTE: **Note:** -A Consul leader will be elected when the provisioning of the third Consul server is completed. -Viewing the Consul logs `sudo gitlab-ctl tail consul` will display -`...[INFO] consul: New leader elected: ...` +A Consul leader is _elected_ when the provisioning of the third Consul server is +complete. Viewing the Consul logs `sudo gitlab-ctl tail consul` displays +`...[INFO] consul: New leader elected: ...`. You can list the current Consul members (server, client): @@ -627,7 +616,7 @@ in the second step, do not supply the `EXTERNAL_URL` value. #### PostgreSQL primary node -1. SSH into the PostgreSQL primary node. +1. SSH in to the PostgreSQL primary node. 1. Generate a password hash for the PostgreSQL username/password pair. This assumes you will use the default username of `gitlab` (recommended). The command will request a password and confirmation. Use the value that is output by this command in the next @@ -812,7 +801,7 @@ are supported and can be added if needed. #### PostgreSQL post-configuration -SSH into the **primary node**: +SSH in to the **primary node**: 1. Open a database prompt: @@ -847,7 +836,7 @@ SSH into the **primary node**: is not an IP address, it will need to be a resolvable name (via DNS or `/etc/hosts`) -SSH into the **secondary node**: +SSH in to the **secondary node**: 1. Set up the repmgr standby: @@ -1068,51 +1057,48 @@ Refer to your preferred Load Balancer's documentation for further guidance. ## Configure Gitaly -Deploying Gitaly in its own server can benefit GitLab installations that are -larger than a single machine. - -The Gitaly node requirements are dependent on customer data, specifically the number of -projects and their repository sizes. Two nodes are recommended as an absolute minimum. -Each Gitaly node should store no more than 5TB of data and have the number of -[`gitaly-ruby` workers](../gitaly/index.md#gitaly-ruby) set to 20% of available CPUs. -Additional nodes should be considered in conjunction with a review of expected -data size and spread based on the recommendations above. - -It is also strongly recommended that all Gitaly nodes be set up with SSD disks with -a throughput of at least 8,000 IOPS for read operations and 2,000 IOPS for write, -as Gitaly has heavy I/O. These IOPS values are recommended only as a starter as with -time they may be adjusted higher or lower depending on the scale of your environment's workload. -If you're running the environment on a Cloud provider, you may need to refer to -their documentation on how to configure IOPS correctly. - -Some things to note: - -- The GitLab Rails application shards repositories into [repository storages](../repository_storage_paths.md). -- A Gitaly server can host one or more storages. -- A GitLab server can use one or more Gitaly servers. -- Gitaly addresses must be specified in such a way that they resolve - correctly for ALL Gitaly clients. +[Gitaly](../gitaly/index.md) server node requirements are dependent on data, +specifically the number of projects and those projects' sizes. It's recommended +that a Gitaly server node stores no more than 5 TB of data. Although this +reference architecture includes a recommendation for the number of Gitaly server +nodes to use, depending on your storage requirements, you may require additional +Gitaly server nodes. + +Due to Gitaly having notable input and output requirements, we strongly +recommend that all Gitaly nodes use solid-state drives (SSDs). These SSDs should +have a throughput of at least 8,000 input/output operations per second (IOPS) +for read operations and 2,000 IOPS for write operations. These IOPS values are +initial recommendations, and may be adjusted to greater or lesser values +depending on the scale of your environment's workload. If you're running the +environment on a Cloud provider, refer to their documentation about how to +configure IOPS correctly. + +Be sure to note the following items: + +- The GitLab Rails application shards repositories into + [repository storage paths](../repository_storage_paths.md). +- A Gitaly server can host one or more storage paths. +- A GitLab server can use one or more Gitaly server nodes. +- Gitaly addresses must be specified to be correctly resolvable for _all_ + Gitaly clients. - Gitaly servers must not be exposed to the public internet, as Gitaly's network traffic is unencrypted by default. The use of a firewall is highly recommended to restrict access to the Gitaly server. Another option is to [use TLS](#gitaly-tls-support). -TIP: **Tip:** -For more information about Gitaly's history and network architecture see the -[standalone Gitaly documentation](../gitaly/index.md). - -Note: **Note:** The token referred to throughout the Gitaly documentation is -just an arbitrary password selected by the administrator. It is unrelated to -tokens created for the GitLab API or other similar web API tokens. +NOTE: **Note:** +The token referred to throughout the Gitaly documentation is an arbitrary +password selected by the administrator. This token is unrelated to tokens +created for the GitLab API or other similar web API tokens. -Below we describe how to configure two Gitaly servers, with IPs and -domain names: +This section describes how to configure two Gitaly servers, with the following +IPs and domain names: - `10.6.0.51`: Gitaly 1 (`gitaly1.internal`) - `10.6.0.52`: Gitaly 2 (`gitaly2.internal`) -The secret token is assumed to be `gitalysecret` and that -your GitLab installation has three repository storages: +Assumptions about your servers include having the secret token be `gitalysecret`, +and that your GitLab installation has three repository storages: - `default` on Gitaly 1 - `storage1` on Gitaly 1 @@ -1120,11 +1106,11 @@ your GitLab installation has three repository storages: On each node: -1. [Download/Install](https://about.gitlab.com/install/) the Omnibus GitLab - package you want using **steps 1 and 2** from the GitLab downloads page but - **without** providing the `EXTERNAL_URL` value. -1. Edit `/etc/gitlab/gitlab.rb` to configure storage paths, enable - the network listener and configure the token: +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to follow _only_ installation steps 1 and 2 + on the page, and _do not_ provide the `EXTERNAL_URL` value. +1. Edit `/etc/gitlab/gitlab.rb` to configure the storage paths, enable + the network listener, and configure the token: <!-- updates to following example must also be made at @@ -1188,39 +1174,39 @@ On each node: ``` 1. Append the following to `/etc/gitlab/gitlab.rb` for each respective server: - 1. On `gitaly1.internal`: - - ```ruby - git_data_dirs({ - 'default' => { - 'path' => '/var/opt/gitlab/git-data' - }, - 'storage1' => { - 'path' => '/mnt/gitlab/git-data' - }, - }) - ``` - - 1. On `gitaly2.internal`: - - ```ruby - git_data_dirs({ - 'storage2' => { - 'path' => '/mnt/gitlab/git-data' - }, - }) - ``` + - On `gitaly1.internal`: + + ```ruby + git_data_dirs({ + 'default' => { + 'path' => '/var/opt/gitlab/git-data' + }, + 'storage1' => { + 'path' => '/mnt/gitlab/git-data' + }, + }) + ``` + + - On `gitaly2.internal`: + + ```ruby + git_data_dirs({ + 'storage2' => { + 'path' => '/mnt/gitlab/git-data' + }, + }) + ``` <!-- updates to following example must also be made at https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab --> -1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). +1. Save the file, and then [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). 1. Confirm that Gitaly can perform callbacks to the internal API: ```shell - sudo /opt/gitlab/embedded/service/gitlab-shell/bin/check -config /opt/gitlab/embedded/service/gitlab-shell/config.yml + sudo /opt/gitlab/embedded/bin/gitaly-hooks check /var/opt/gitlab/gitaly/config.toml ``` 1. Verify the GitLab services are running: @@ -1258,11 +1244,10 @@ Name. If you are addressing the Gitaly server by its IP address, you must add it as a Subject Alternative Name to the certificate. [gRPC does not support using an IP address as Common Name in a certificate](https://github.com/grpc/grpc/issues/2691). -NOTE: **Note:** -It is possible to configure Gitaly servers with both an -unencrypted listening address `listen_addr` and an encrypted listening -address `tls_listen_addr` at the same time. This allows you to do a -gradual transition from unencrypted to encrypted traffic, if necessary. +It's possible to configure Gitaly servers with both an unencrypted listening +address (`listen_addr`) and an encrypted listening address (`tls_listen_addr`) +at the same time. This allows you to do a gradual transition from unencrypted to +encrypted traffic, if necessary. To configure Gitaly with TLS: @@ -1316,10 +1301,10 @@ The following IPs will be used as an example: To configure the Sidekiq nodes, one each one: -1. SSH into the Sidekiq server. -1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab package -you want using steps 1 and 2 from the GitLab downloads page. -**Do not complete any other steps on the download page.** +1. SSH in to the Sidekiq server. +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to follow _only_ installation steps 1 and 2 + on the page. 1. Open `/etc/gitlab/gitlab.rb` with your editor: ```ruby @@ -1430,14 +1415,14 @@ You can also run [multiple Sidekiq processes](../operations/extra_sidekiq_proces ## Configure GitLab Rails -NOTE: **Note:** -In our architectures we run each GitLab Rails node using the Puma webserver -and have its number of workers set to 90% of available CPUs along with four threads. For -nodes that are running Rails with other components the worker value should be reduced -accordingly where we've found 50% achieves a good balance but this is dependent -on workload. - This section describes how to configure the GitLab application (Rails) component. + +In our architecture, we run each GitLab Rails node using the Puma webserver, and +have its number of workers set to 90% of available CPUs, with four threads. For +nodes running Rails with other components, the worker value should be reduced +accordingly. We've determined that a worker value of 50% achieves a good balance, +but this is dependent on workload. + On each node perform the following: 1. If you're [using NFS](#configure-nfs-optional): @@ -1465,10 +1450,10 @@ On each node perform the following: mkdir -p /var/opt/gitlab/.ssh /var/opt/gitlab/gitlab-rails/uploads /var/opt/gitlab/gitlab-rails/shared /var/opt/gitlab/gitlab-ci/builds /var/opt/gitlab/git-data ``` -1. Download/install Omnibus GitLab using **steps 1 and 2** from - [GitLab downloads](https://about.gitlab.com/install/). Do not complete other - steps on the download page. -1. Create/edit `/etc/gitlab/gitlab.rb` and use the following configuration. +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to follow _only_ installation steps 1 and 2 + on the page. +1. Create or edit `/etc/gitlab/gitlab.rb` and use the following configuration. To maintain uniformity of links across nodes, the `external_url` on the application server should point to the external URL that users will use to access GitLab. This would be the URL of the [external load balancer](#configure-the-external-load-balancer) @@ -1581,6 +1566,11 @@ On each node perform the following: sudo gitlab-ctl tail gitaly ``` +1. Save the `/etc/gitlab/gitlab-secrets.json` file from one of the two + application nodes and install it on the other application node, the + [Gitaly node](#configure-gitaly) and the [Sidekiq node](#configure-sidekiq) and + [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). + 1. Verify the GitLab services are running: ```shell @@ -1599,12 +1589,10 @@ On each node perform the following: run: puma: (pid 4936) 8645s; run: log: (pid 29656) 79161s ``` -NOTE: **Note:** -When you specify `https` in the `external_url`, as in the example -above, GitLab assumes you have SSL certificates in `/etc/gitlab/ssl/`. If -certificates are not present, NGINX will fail to start. See the -[NGINX documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https) -for more information. +When you specify `https` in the `external_url`, as in the previous example, +GitLab expects that the SSL certificates are in `/etc/gitlab/ssl/`. If the +certificates aren't present, NGINX will fail to start. For more information, see +the [NGINX documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https). ### GitLab Rails post-configuration @@ -1614,12 +1602,11 @@ for more information. gitlab-rake gitlab:db:configure ``` - NOTE: **Note:** - If you encounter a `rake aborted!` error stating that PgBouncer is failing to connect to - PostgreSQL it may be that your PgBouncer node's IP address is missing from - PostgreSQL's `trust_auth_cidr_addresses` in `gitlab.rb` on your database nodes. See - [PgBouncer error `ERROR: pgbouncer cannot connect to server`](troubleshooting.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server) - in the Troubleshooting section before proceeding. + If you encounter a `rake aborted!` error message stating that PgBouncer is + failing to connect to PostgreSQL, it may be that your PgBouncer node's IP + address is missing from PostgreSQL's `trust_auth_cidr_addresses` in `gitlab.rb` + on your database nodes. Before proceeding, see + [PgBouncer error `ERROR: pgbouncer cannot connect to server`](troubleshooting.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server). 1. [Configure fast lookup of authorized SSH keys in the database](../operations/fast_ssh_key_lookup.md). @@ -1635,10 +1622,10 @@ The Omnibus GitLab package can be used to configure a standalone Monitoring node running [Prometheus](../monitoring/prometheus/index.md) and [Grafana](../monitoring/performance/grafana_configuration.md): -1. SSH into the Monitoring node. -1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab - package you want using **steps 1 and 2** from the GitLab downloads page. - Do not complete any other steps on the download page. +1. SSH in to the Monitoring node. +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to follow _only_ installation steps 1 and 2 + on the page. 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby @@ -1713,28 +1700,44 @@ It's recommended over [NFS](#configure-nfs-optional) and in general it's better in larger setups as object storage is typically much more performant, reliable, and scalable. -Object storage options that GitLab has tested, or is aware of customers using include: +GitLab has been tested on a number of object storage providers: -- SaaS/Cloud solutions such as [Amazon S3](https://aws.amazon.com/s3/), [Google cloud storage](https://cloud.google.com/storage). +- [Amazon S3](https://aws.amazon.com/s3/) +- [Google Cloud Storage](https://cloud.google.com/storage) +- [Digital Ocean Spaces](https://www.digitalocean.com/products/spaces/) +- [Oracle Cloud Infrastructure](https://docs.cloud.oracle.com/en-us/iaas/Content/Object/Tasks/s3compatibleapi.htm) +- [Openstack Swift](https://docs.openstack.org/swift/latest/s3_compat.html) +- [Azure Blob storage](https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blobs-introduction) - On-premises hardware and appliances from various storage vendors. -- MinIO. There is [a guide to deploying this](https://docs.gitlab.com/charts/advanced/external-object-storage/minio.html) within our Helm Chart documentation. - -For configuring GitLab to use Object Storage refer to the following guides -based on what features you intend to use: - -1. Configure [object storage for backups](../../raketasks/backup_restore.md#uploading-backups-to-a-remote-cloud-storage). -1. Configure [object storage for job artifacts](../job_artifacts.md#using-object-storage) - including [incremental logging](../job_logs.md#new-incremental-logging-architecture). -1. Configure [object storage for LFS objects](../lfs/index.md#storing-lfs-objects-in-remote-object-storage). -1. Configure [object storage for uploads](../uploads.md#using-object-storage). -1. Configure [object storage for merge request diffs](../merge_request_diffs.md#using-object-storage). -1. Configure [object storage for Container Registry](../packages/container_registry.md#use-object-storage) (optional feature). -1. Configure [object storage for Mattermost](https://docs.mattermost.com/administration/config-settings.html#file-storage) (optional feature). -1. Configure [object storage for packages](../packages/index.md#using-object-storage) (optional feature). **(PREMIUM ONLY)** -1. Configure [object storage for Dependency Proxy](../packages/dependency_proxy.md#using-object-storage) (optional feature). **(PREMIUM ONLY)** -1. Configure [object storage for Pseudonymizer](../pseudonymizer.md#configuration) (optional feature). **(ULTIMATE ONLY)** -1. Configure [object storage for autoscale runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional - for improved performance). -1. Configure [object storage for Terraform state files](../terraform_state.md#using-object-storage). +- MinIO. We have [a guide to deploying this](https://docs.gitlab.com/charts/advanced/external-object-storage/minio.html) within our Helm Chart documentation. + +There are two ways of specifying object storage configuration in GitLab: + +- [Consolidated form](../object_storage.md#consolidated-object-storage-configuration): A single credential is + shared by all supported object types. +- [Storage-specific form](../object_storage.md#storage-specific-configuration): Every object defines its + own object storage [connection and configuration](../object_storage.md#connection-settings). + +Starting with GitLab 13.2, consolidated object storage configuration is available. It simplifies your GitLab configuration since the connection details are shared across object types. Refer to [Consolidated object storage configuration](../object_storage.md#consolidated-object-storage-configuration) guide for instructions on how to set it up. + +For configuring object storage in GitLab 13.1 and earlier, or for storage types not +supported by consolidated configuration form, refer to the following guides based +on what features you intend to use: + +|Object storage type|Supported by consolidated configuration?| +|-------------------|----------------------------------------| +| [Backups](../../raketasks/backup_restore.md#uploading-backups-to-a-remote-cloud-storage)|No| +| [Job artifacts](../job_artifacts.md#using-object-storage) including archived job logs | Yes | +| [LFS objects](../lfs/index.md#storing-lfs-objects-in-remote-object-storage) | Yes | +| [Uploads](../uploads.md#using-object-storage) | Yes | +| [Container Registry](../packages/container_registry.md#use-object-storage) (optional feature) | No | +| [Merge request diffs](../merge_request_diffs.md#using-object-storage) | Yes | +| [Mattermost](https://docs.mattermost.com/administration/config-settings.html#file-storage)| No | +| [Packages](../packages/index.md#using-object-storage) (optional feature) | Yes | +| [Dependency Proxy](../packages/dependency_proxy.md#using-object-storage) (optional feature) **(PREMIUM ONLY)** | Yes | +| [Pseudonymizer](../pseudonymizer.md#configuration) (optional feature) **(ULTIMATE ONLY)** | No | +| [Autoscale runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional for improved performance) | No | +| [Terraform state files](../terraform_state.md#using-object-storage) | Yes | Using separate buckets for each data type is the recommended approach for GitLab. @@ -1758,6 +1761,22 @@ work. </a> </div> +## Configure Advanced Search **(STARTER ONLY)** + +You can leverage Elasticsearch and [enable Advanced Search](../../integration/elasticsearch.md) +for faster, more advanced code search across your entire GitLab instance. + +Elasticsearch cluster design and requirements are dependent on your specific +data. For recommended best practices about how to set up your Elasticsearch +cluster alongside your instance, read how to +[choose the optimal cluster configuration](../../integration/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration). + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + ## Configure NFS (optional) [Object storage](#configure-the-object-storage), along with [Gitaly](#configure-gitaly) diff --git a/doc/administration/reference_architectures/index.md b/doc/administration/reference_architectures/index.md index 3964b8daeb7..8816d0eecf4 100644 --- a/doc/administration/reference_architectures/index.md +++ b/doc/administration/reference_architectures/index.md @@ -105,7 +105,7 @@ is the least complex to setup. This provides a point-in-time recovery of a prede > - Supported tiers: [GitLab Starter, Premium, and Ultimate](https://about.gitlab.com/pricing/) This requires separating out GitLab into multiple application nodes with an added -[load balancer](../high_availability/load_balancer.md). The load balancer will distribute traffic +[load balancer](../load_balancer.md). The load balancer will distribute traffic across GitLab application nodes. Meanwhile, each application node connects to a shared file server and database systems on the back end. This way, if one of the application servers fails, the workflow is not interrupted. diff --git a/doc/administration/reply_by_email_postfix_setup.md b/doc/administration/reply_by_email_postfix_setup.md index f950134889d..f4fcbefa403 100644 --- a/doc/administration/reply_by_email_postfix_setup.md +++ b/doc/administration/reply_by_email_postfix_setup.md @@ -71,7 +71,7 @@ The instructions make the assumption that you will be using the email address `i sudo postfix start ``` -1. Send the new `incoming` user a dummy email to test SMTP, by entering the following into the SMTP prompt: +1. Send the new `incoming` user an email to test SMTP, by entering the following into the SMTP prompt: ```plaintext ehlo localhost @@ -112,7 +112,7 @@ The instructions make the assumption that you will be using the email address `i q ``` -1. Log out of the `incoming` account and go back to being `root`: +1. Sign out of the `incoming` account, and go back to being `root`: ```shell logout @@ -164,7 +164,7 @@ Courier, which we will install later to add IMAP authentication, requires mailbo try the above steps again, substituting `heirloom-mailx` for the `mail` command._ -1. Log out of the `incoming` account and go back to being `root`: +1. Sign out of the `incoming` account, and go back to being `root`: ```shell logout @@ -251,7 +251,7 @@ Courier, which we will install later to add IMAP authentication, requires mailbo If you get a `Connection refused` error instead, make sure your firewall is set up to allow inbound traffic on port 25. - 1. Send the `incoming` user a dummy email to test SMTP, by entering the following into the SMTP prompt: + 1. Send the `incoming` user an email to test SMTP, by entering the following into the SMTP prompt: ```plaintext ehlo gitlab.example.com @@ -288,7 +288,7 @@ Courier, which we will install later to add IMAP authentication, requires mailbo q ``` - 1. Log out of the `incoming` account and go back to being `root`: + 1. Sign out of the `incoming` account, and go back to being `root`: ```shell logout diff --git a/doc/administration/repository_storage_types.md b/doc/administration/repository_storage_types.md index 1c036cfe2ac..b272c4b463e 100644 --- a/doc/administration/repository_storage_types.md +++ b/doc/administration/repository_storage_types.md @@ -83,7 +83,7 @@ The "Gitaly relative path" is shown there, for example: This is the path under `/var/opt/gitlab/git-data/repositories/` on a default Omnibus installation. -In a [Rails console](troubleshooting/debug.md#starting-a-rails-console-session), +In a [Rails console](operations/rails_console.md#starting-a-rails-console-session), get this information using either the numeric project ID or the full path: ```ruby @@ -95,7 +95,7 @@ Project.find_by_full_path('group/project').disk_path To translate from a hashed storage path to a project name: -1. Start a [Rails console](troubleshooting/debug.md#starting-a-rails-console-session). +1. Start a [Rails console](operations/rails_console.md#starting-a-rails-console-session). 1. Run the following: ```ruby diff --git a/doc/administration/server_hooks.md b/doc/administration/server_hooks.md index 54b2cd43265..b5e975d397f 100644 --- a/doc/administration/server_hooks.md +++ b/doc/administration/server_hooks.md @@ -156,22 +156,6 @@ NOTE: **Note:** While other environment variables can be passed to server hooks, your application should not rely on them as they can change. -## Transition to Go - -> - Introduced in GitLab 13.2 using feature flags. -> - In GitLab 13.4, `update` Ruby [implementation removed](https://gitlab.com/gitlab-org/gitaly/-/merge_requests/2501). -> - In GitLab 13.4, `post-receive` Go implementation [made default](https://gitlab.com/gitlab-org/gitaly/-/merge_requests/2502). - -The following server hooks have been re-implemented in Go: - -- `pre-receive`, with the Go implementation used by default. To use the Ruby implementation instead, - [disable](feature_flags.md#enable-or-disable-the-feature) the `:gitaly_go_preceive_hook` feature - flag. -- `update`, with Go implementation always used. No Ruby implementation is available. -- `post-receive`, with the Go implementation used by default. To use the Ruby implementation - instead, [disable](feature_flags.md#enable-or-disable-the-feature) the - `:gitaly_go_postreceive_hook` feature flag. - ## Custom error messages > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/5073) in GitLab 8.10. diff --git a/doc/administration/smime_signing_email.md b/doc/administration/smime_signing_email.md index b1e7e349978..6ded7fdd7d0 100644 --- a/doc/administration/smime_signing_email.md +++ b/doc/administration/smime_signing_email.md @@ -75,8 +75,8 @@ extensions), which contain the following in a single encrypted file: - Intermediate certificates (if any) - Private key -In order to export the required files in PEM encoding from the PKCS#12 file, -the `openssl` command can be used: +To export the required files in PEM encoding from the PKCS#12 file, the +`openssl` command can be used: ```shell #-- Extract private key in PEM encoding (no password, unencrypted) diff --git a/doc/administration/snippets/index.md b/doc/administration/snippets/index.md index 95de3b8c183..5e61d20c683 100644 --- a/doc/administration/snippets/index.md +++ b/doc/administration/snippets/index.md @@ -20,8 +20,8 @@ abuse of the feature. The default value is **52428800 Bytes** (50 MB). The content size limit will be applied when a snippet is created or updated. -In order not to break any existing snippets, the limit doesn't have any -effect on them until a snippet is edited again and the content changes. +This limit doesn't affect existing snippets until they're updated and their +content changes. ### Snippets size limit configuration diff --git a/doc/administration/terraform_state.md b/doc/administration/terraform_state.md index 76e54acce16..55e166d2bf7 100644 --- a/doc/administration/terraform_state.md +++ b/doc/administration/terraform_state.md @@ -1,3 +1,9 @@ +--- +stage: Configure +group: Configure +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Terraform state administration (alpha) > [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/2673) in GitLab 12.10. diff --git a/doc/administration/troubleshooting/debug.md b/doc/administration/troubleshooting/debug.md index a1b4df9b94e..ef95bdc8602 100644 --- a/doc/administration/troubleshooting/debug.md +++ b/doc/administration/troubleshooting/debug.md @@ -5,27 +5,15 @@ in production. ## Starting a Rails console session -Troubleshooting and debugging your GitLab instance often requires a -[Rails console](https://guides.rubyonrails.org/command_line.html#rails-console). +Troubleshooting and debugging your GitLab instance often requires a Rails console. + +Your type of GitLab installation determines how +[to start a rails console](../operations/rails_console.md). See also: - [GitLab Rails Console Cheat Sheet](gitlab_rails_cheat_sheet.md). - [Navigating GitLab via Rails console](navigating_gitlab_via_rails_console.md). -**For Omnibus installations** - -```shell -sudo gitlab-rails console -``` - -**For installations from source** - -```shell -sudo -u git -H bundle exec rails console -e production -``` - -Kubernetes: the console is in the task-runner pod, refer to our [Kubernetes cheat sheet](kubernetes_cheat_sheet.md#gitlab-specific-kubernetes-information) for details. - ### Enabling Active Record logging You can enable output of Active Record debug logging in the Rails console @@ -98,7 +86,7 @@ sudo -u git -H bundle exec rails runner -e production /path/to/script.rb A common problem is that mails are not being sent for some reason. Suppose you configured an SMTP server, but you're not seeing mail delivered. Here's how to check the settings: -1. Run a [Rails console](#starting-a-rails-console-session). +1. Run a [Rails console](../operations/rails_console.md#starting-a-rails-console-session). 1. Look at the ActionMailer `delivery_method` to make sure it matches what you intended. If you configured SMTP, it should say `:smtp`. If you're using @@ -238,7 +226,7 @@ separate Rails process to debug the issue: 1. Log in to your GitLab account. 1. Copy the URL that is causing problems (e.g. `https://gitlab.com/ABC`). 1. Create a Personal Access Token for your user (Profile Settings -> Access Tokens). -1. Bring up the [GitLab Rails console.](#starting-a-rails-console-session) +1. Bring up the [GitLab Rails console.](../operations/rails_console.md#starting-a-rails-console-session) 1. At the Rails console, run: ```ruby diff --git a/doc/administration/troubleshooting/elasticsearch.md b/doc/administration/troubleshooting/elasticsearch.md index e13261e3074..6de5bb71d75 100644 --- a/doc/administration/troubleshooting/elasticsearch.md +++ b/doc/administration/troubleshooting/elasticsearch.md @@ -164,7 +164,7 @@ Troubleshooting search result issues is rather straight forward on Elasticsearch The first step is to confirm GitLab is using Elasticsearch for the search function. To do this: -1. Confirm the integration is enabled in **Admin Area > Settings > Integrations**. +1. Confirm the integration is enabled in **Admin Area > Settings > General**. 1. Confirm searches utilize Elasticsearch by accessing the rails console (`sudo gitlab-rails console`) and running the following commands: @@ -206,7 +206,7 @@ The best place to start is to determine if the issue is with creating an empty i If it is, check on the Elasticsearch side to determine if the `gitlab-production` (the name for the GitLab index) exists. If it exists, manually delete it on the Elasticsearch side and attempt to recreate it from the -[`recreate_index`](../../integration/elasticsearch.md#gitlab-elasticsearch-rake-tasks) +[`recreate_index`](../../integration/elasticsearch.md#gitlab-advanced-search-rake-tasks) Rake task. If you still encounter issues, try creating an index manually on the Elasticsearch @@ -225,8 +225,8 @@ during the indexing of projects. If errors do occur, they will either stem from If the indexing process does not present errors, you will want to check the status of the indexed projects. You can do this via the following Rake tasks: -- [`sudo gitlab-rake gitlab:elastic:index_projects_status`](../../integration/elasticsearch.md#gitlab-elasticsearch-rake-tasks) (shows the overall status) -- [`sudo gitlab-rake gitlab:elastic:projects_not_indexed`](../../integration/elasticsearch.md#gitlab-elasticsearch-rake-tasks) (shows specific projects that are not indexed) +- [`sudo gitlab-rake gitlab:elastic:index_projects_status`](../../integration/elasticsearch.md#gitlab-advanced-search-rake-tasks) (shows the overall status) +- [`sudo gitlab-rake gitlab:elastic:projects_not_indexed`](../../integration/elasticsearch.md#gitlab-advanced-search-rake-tasks) (shows specific projects that are not indexed) If: diff --git a/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md b/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md index 9a23a115765..dbda889d370 100644 --- a/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md +++ b/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md @@ -222,11 +222,12 @@ namespace = Namespace.find_by_full_path("") ```ruby # ID will be the webhook_id -WebHookLog.where(web_hook_id: ID).each_slice(ID) do |slice| - slice.each(&:destroy) -end +hook=WebHook.find(ID) + +WebHooks::DestroyService.new(current_user).execute(hook) -WebHook.find(ID).destroy +#In case the service gets timeout consider removing webhook_logs +hook.web_hook_logs.limit(BATCH_SIZE).delete_all ``` ### Bulk update service integration password for _all_ projects @@ -285,6 +286,16 @@ GitlabShellWorker.perform_in(0, :remove_repository, p.repository_storage, p.wiki p.create_wiki ### creates the wiki project on the filesystem ``` +## Issue boards + +### In case of issue boards not loading properly and it's getting time out. We need to call the Issue Rebalancing service to fix this + +```ruby +p=Project.find_by_full_path('PROJECT PATH') + +IssueRebalancingService.new(p.issues.take).execute +``` + ## Imports / Exports ```ruby @@ -411,6 +422,9 @@ user.skip_reconfirmation! # Active users on the instance, now User.active.count +# Users taking a seat on the instance +License.current.current_active_users_count + # The historical max on the instance as of the past year ::HistoricalData.max_historical_user_count ``` @@ -506,6 +520,54 @@ group = Group.find_by_path_or_name('group-name') group.project_creation_level=0 ``` +## SCIM + +### Fixing bad SCIM identities + +```ruby +def delete_bad_scim(email, group_path) + output = "" + u = User.find_by_email(email) + uid = u.id + g = Group.find_by_full_path(group_path) + saml_prov_id = SamlProvider.find_by(group_id: g.id).id + saml = Identity.where(user_id: uid, saml_provider_id: saml_prov_id) + scim = ScimIdentity.where(user_id: uid , group_id: g.id) + if saml[0] + saml_eid = saml[0].extern_uid + output += "%s," % [email] + output += "SAML: %s," % [saml_eid] + if scim[0] + scim_eid = scim[0].extern_uid + output += "SCIM: %s" % [scim_eid] + if saml_eid == scim_eid + output += " Identities matched, not deleted \n" + else + scim[0].destroy + output += " Deleted \n" + end + else + output = "ERROR No SCIM identify found for: [%s]\n" % [email] + puts output + return 1 + end + else + output = "ERROR No SAML identify found for: [%s]\n" % [email] + puts output + return 1 + end + puts output + return 0 +end + +# In case of multiple emails +emails = [email1, email2] + +emails.each do |e| + delete_bad_scim(e,'GROUPPATH') +end +``` + ## Routes ### Remove redirecting routes @@ -525,8 +587,8 @@ conflicting_permanent_redirects.destroy_all ### Close a merge request properly (if merged but still marked as open) ```ruby -p = Project.find_by_full_path('') -m = project.merge_requests.find_by(iid: ) +p = Project.find_by_full_path('<full/path/to/project>') +m = p.merge_requests.find_by(iid: <iid>) u = User.find_by_username('') MergeRequests::PostMergeService.new(p, u).execute(m) ``` diff --git a/doc/administration/troubleshooting/kubernetes_cheat_sheet.md b/doc/administration/troubleshooting/kubernetes_cheat_sheet.md index 01532032b49..7c5a9e0d79f 100644 --- a/doc/administration/troubleshooting/kubernetes_cheat_sheet.md +++ b/doc/administration/troubleshooting/kubernetes_cheat_sheet.md @@ -84,8 +84,7 @@ and they will assist you with any issues you are having. ## GitLab-specific Kubernetes information -- Minimal config that can be used to test a Kubernetes Helm chart can be found - [here](https://gitlab.com/gitlab-org/charts/gitlab/-/issues/620). +- Minimal configuration that can be used to [test a Kubernetes Helm chart](https://gitlab.com/gitlab-org/charts/gitlab/-/issues/620). - Tailing logs of a separate pod. An example for a Webservice pod: @@ -190,7 +189,7 @@ and they will assist you with any issues you are having. be possible to use [Updating GitLab using the Helm Chart](https://docs.gitlab.com/charts/index.html#updating-gitlab-using-the-helm-chart) for upgrades. -- How to apply changes to GitLab config: +- How to apply changes to GitLab configuration: - Modify the `gitlab.yaml` file. - Run the following command to apply changes: @@ -255,7 +254,7 @@ to those documents for details. helm install gitlab -f <path-to-yaml-file> gitlab/gitlab ``` - If you want to modify some GitLab settings, you can use the above-mentioned config + If you want to modify some GitLab settings, you can use the above-mentioned configuration as a base and create your own YAML file. - Monitor the installation progress via `helm status gitlab` and `minikube dashboard`. diff --git a/doc/administration/troubleshooting/log_parsing.md b/doc/administration/troubleshooting/log_parsing.md index dcd1df2f423..7914628a756 100644 --- a/doc/administration/troubleshooting/log_parsing.md +++ b/doc/administration/troubleshooting/log_parsing.md @@ -3,11 +3,11 @@ We recommend using log aggregation and search tools like Kibana and Splunk whenever possible, but if they are not available you can still quickly parse [GitLab logs](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/26311) in JSON format -(the default since GitLab 12.0) using [`jq`](https://stedolan.github.io/jq/). +(the default in GitLab 12.0 and later) using [`jq`](https://stedolan.github.io/jq/). ## What is JQ? -As noted in its [manual](https://stedolan.github.io/jq/manual/), jq is a command-line JSON processor. The following examples +As noted in its [manual](https://stedolan.github.io/jq/manual/), `jq` is a command-line JSON processor. The following examples include use cases targeted for parsing GitLab log files. ## Parsing Logs diff --git a/doc/administration/troubleshooting/navigating_gitlab_via_rails_console.md b/doc/administration/troubleshooting/navigating_gitlab_via_rails_console.md index 571973c12d9..a1485249b0e 100644 --- a/doc/administration/troubleshooting/navigating_gitlab_via_rails_console.md +++ b/doc/administration/troubleshooting/navigating_gitlab_via_rails_console.md @@ -3,7 +3,7 @@ At the heart of GitLab is a web application [built using the Ruby on Rails framework](https://about.gitlab.com/blog/2018/10/29/why-we-use-rails-to-build-gitlab/). Thanks to this, we also get access to the amazing tools built right into Rails. -In this guide, we'll introduce the [Rails console](debug.md#starting-a-rails-console-session) +In this guide, we'll introduce the [Rails console](../operations/rails_console.md#starting-a-rails-console-session) and the basics of interacting with your GitLab instance from the command line. CAUTION: **Caution:** @@ -20,20 +20,10 @@ Rails experience is helpful to have but not a must. ## Starting a Rails console session -Omnibus GitLab comes with a convenient wrapper command which automatically loads -the production GitLab environment: +Your type of GitLab installation determines how +[to start a rails console](../operations/rails_console.md). -```shell -sudo gitlab-rails console -``` - -For source installations, you'll have to instead run: - -```shell -sudo -u git -H bundle exec rails console -e production -``` - -Further code examples will all take place inside the Rails console and also +The following code examples will all take place inside the Rails console and also assume an Omnibus GitLab installation. ## Active Record objects diff --git a/doc/administration/troubleshooting/sidekiq.md b/doc/administration/troubleshooting/sidekiq.md index 404e806c5d9..b7762f8ac3e 100644 --- a/doc/administration/troubleshooting/sidekiq.md +++ b/doc/administration/troubleshooting/sidekiq.md @@ -7,7 +7,7 @@ may be filling up. Users will notice when this happens because new branches may not show up and merge requests may not be updated. The following are some troubleshooting steps that will help you diagnose the bottleneck. -NOTE **Note:** +NOTE: **Note:** GitLab administrators/users should consider working through these debug steps with GitLab Support so the backtraces can be analyzed by our team. It may reveal a bug or necessary improvement in GitLab. diff --git a/doc/administration/troubleshooting/ssl.md b/doc/administration/troubleshooting/ssl.md index e6c081e1eea..4d4b9755fa9 100644 --- a/doc/administration/troubleshooting/ssl.md +++ b/doc/administration/troubleshooting/ssl.md @@ -23,7 +23,7 @@ After configuring a GitLab instance with an internal CA certificate, you might n More details here: https://curl.haxx.se/docs/sslcerts.html ``` -- Testing via the [rails console](debug.md#starting-a-rails-console-session) also fails: +- Testing via the [rails console](../operations/rails_console.md#starting-a-rails-console-session) also fails: ```ruby uri = URI.parse("https://gitlab.domain.tld") @@ -205,6 +205,6 @@ Some of these errors come from the Excon Ruby gem, and could be generated in cir where GitLab is configured to initiate an HTTPS session to a remote server that is serving just HTTP. -One scenario is that you're using [object storage](../high_availability/object_storage.md) +One scenario is that you're using [object storage](../object_storage.md) which is not served under HTTPS. GitLab is misconfigured and attempts a TLS handshake, but the object storage will respond with plain HTTP. diff --git a/doc/administration/troubleshooting/tracing_correlation_id.md b/doc/administration/troubleshooting/tracing_correlation_id.md index 03c342595a3..ae9ebd90951 100644 --- a/doc/administration/troubleshooting/tracing_correlation_id.md +++ b/doc/administration/troubleshooting/tracing_correlation_id.md @@ -4,7 +4,7 @@ type: reference # Finding relevant log entries with a correlation ID -Since GitLab 11.6, a unique request tracking ID, known as the "correlation ID" has been +In GitLab 11.6 and later, a unique request tracking ID, known as the "correlation ID" has been logged by the GitLab instance for most requests. Each individual request to GitLab gets its own correlation ID, which then gets logged in each GitLab component's logs for that request. This makes it easier to trace behavior in a @@ -122,5 +122,5 @@ If you have done some horizontal scaling in your GitLab infrastructure, then you will need to search across _all_ of your GitLab nodes. You can do this with some sort of log aggregation software like Loki, ELK, Splunk, or others. -You can use a tool like Ansible or PSSH (parellel SSH) that can execute identical commands across your servers in +You can use a tool like Ansible or PSSH (parallel SSH) that can execute identical commands across your servers in parallel, or craft your own solution. diff --git a/doc/administration/uploads.md b/doc/administration/uploads.md index 71a41719003..91de089c45e 100644 --- a/doc/administration/uploads.md +++ b/doc/administration/uploads.md @@ -2,6 +2,35 @@ Uploads represent all user data that may be sent to GitLab as a single file. As an example, avatars and notes' attachments are uploads. Uploads are integral to GitLab functionality, and therefore cannot be disabled. +## Upload parameters + +> - [Changed](https://gitlab.com/gitlab-org/gitlab/-/issues/214785) in GitLab 13.5. +> - It's [deployed behind a feature flag](../user/feature_flags.md), enabled by default. +> - It's enabled on GitLab.com. +> - It's recommended for production use. +> - For GitLab self-managed instances, GitLab administrators can opt to disable it. **(CORE ONLY)** + +In 13.5 and later, upload parameters are passed [between Workhorse and GitLab Rails](../development/architecture.md#simplified-component-overview) differently than they +were before. + +This change is deployed behind a feature flag that is **enabled by default**. + +If you experience any issues with upload, +[GitLab administrators with access to the GitLab Rails console](./feature_flags.md) +can opt to disable it. + +To enable it: + +```ruby +Feature.enable(:upload_middleware_jwt_params_handler) +``` + +To disable it: + +```ruby +Feature.disable(:upload_middleware_jwt_params_handler) +``` + ## Using local storage NOTE: **Note:** @@ -58,7 +87,7 @@ This configuration relies on valid AWS credentials to be configured already. [Read more about using object storage with GitLab](object_storage.md). NOTE: **Note:** -We recommend using the [consolidated object storage settings](object_storage.md#consolidated-object-storage-configuration). The following instructions apply to the original config format. +We recommend using the [consolidated object storage settings](object_storage.md#consolidated-object-storage-configuration). The following instructions apply to the original configuration format. ## Object Storage Settings @@ -68,9 +97,9 @@ For source installations the following settings are nested under `uploads:` and |---------|-------------|---------| | `enabled` | Enable/disable object storage | `false` | | `remote_directory` | The bucket name where Uploads will be stored| | -| `direct_upload` | Set to true to remove Puma from the Upload path. Workhorse handles the actual Artifact Upload to Object Storage while Puma does minimal processing to keep track of the upload. There is no need for local shared storage. The option may be removed if support for a single storage type for all files is introduced. Read more on [direct upload](../development/uploads.md#direct-upload). | `false` | -| `background_upload` | Set to false to disable automatic upload. Option may be removed once upload is direct to S3 (if `direct_upload` is set to `true` it will override `background_upload`) | `true` | -| `proxy_download` | Set to true to enable proxying all files served. Option allows to reduce egress traffic as this allows clients to download directly from remote storage instead of proxying all data | `false` | +| `direct_upload` | Set to `true` to remove Puma from the Upload path. Workhorse handles the actual Artifact Upload to Object Storage while Puma does minimal processing to keep track of the upload. There is no need for local shared storage. The option may be removed if support for a single storage type for all files is introduced. Read more on [direct upload](../development/uploads.md#direct-upload). | `false` | +| `background_upload` | Set to `false` to disable automatic upload. Option may be removed once upload is direct to S3 (if `direct_upload` is set to `true` it will override `background_upload`) | `true` | +| `proxy_download` | Set to `true` to enable proxying all files served. Option allows to reduce egress traffic as this allows clients to download directly from remote storage instead of proxying all data | `false` | | `connection` | Various connection options described below | | ### Connection settings diff --git a/doc/api/README.md b/doc/api/README.md index 53df4114a71..3f7dae055e2 100644 --- a/doc/api/README.md +++ b/doc/api/README.md @@ -83,7 +83,11 @@ There are several ways to authenticate with the GitLab API: 1. [OAuth2 tokens](#oauth2-tokens) 1. [Personal access tokens](../user/profile/personal_access_tokens.md) -1. [Project access tokens](../user/project/settings/project_access_tokens.md) **(CORE ONLY)** +1. [Project access tokens](../user/project/settings/project_access_tokens.md) + +NOTE: **Note:** +Project access tokens are supported for self-managed instances on Core and above. They are also supported on GitLab.com Bronze and above. + 1. [Session cookie](#session-cookie) 1. [GitLab CI/CD job token](#gitlab-ci-job-token) **(Specific endpoints only)** @@ -158,9 +162,22 @@ for example, without needing to explicitly pass an access token. With a few API endpoints you can use a [GitLab CI/CD job token](../user/project/new_ci_build_permissions_model.md#job-token) to authenticate with the API: +- Packages: + - [Composer Repository](../user/packages/composer_repository/index.md) + - [Conan Repository](../user/packages/conan_repository/index.md) + - [Container Registry](../user/packages/container_registry/index.md) (`$CI_REGISTRY_PASSWORD` is actually `$CI_JOB_TOKEN`, but this may change in the future) + - [Go Proxy](../user/packages/go_proxy/index.md) + - [Maven Repository](../user/packages/maven_repository/index.md#authenticate-with-a-ci-job-token) + - [NPM Repository](../user/packages/npm_registry/index.md#authenticating-with-a-ci-job-token) + - [Nuget Repository](../user/packages/nuget_repository/index.md) + - [PyPI Repository](../user/packages/pypi_repository/index.md#using-gitlab-ci-with-pypi-packages) + - [Generic packages](../user/packages/generic_packages/index.md#publish-a-generic-package-by-using-cicd) - [Get job artifacts](job_artifacts.md#get-job-artifacts) -- [Pipeline triggers](pipeline_triggers.md) +- [Pipeline triggers](pipeline_triggers.md) (via `token=` parameter) - [Release creation](releases/index.md#create-a-release) +- [Terraform plan](../user/infrastructure/index.md) + +The token is valid as long as the job is running. ### Impersonation tokens @@ -297,21 +314,22 @@ The following table gives an overview of how the API functions generally behave. The following table shows the possible return codes for API requests. -| Return values | Description | -| ------------- | ----------- | -| `200 OK` | The `GET`, `PUT` or `DELETE` request was successful, the resource(s) itself is returned as JSON. | -| `204 No Content` | The server has successfully fulfilled the request and that there is no additional content to send in the response payload body. | -| `201 Created` | The `POST` request was successful and the resource is returned as JSON. | -| `304 Not Modified` | Indicates that the resource has not been modified since the last request. | -| `400 Bad Request` | A required attribute of the API request is missing, e.g., the title of an issue is not given. | -| `401 Unauthorized` | The user is not authenticated, a valid [user token](#authentication) is necessary. | -| `403 Forbidden` | The request is not allowed, e.g., the user is not allowed to delete a project. | -| `404 Not Found` | A resource could not be accessed, e.g., an ID for a resource could not be found. | +| Return values | Description | +| ------------------------ | ----------- | +| `200 OK` | The `GET`, `PUT` or `DELETE` request was successful, the resource(s) itself is returned as JSON. | +| `204 No Content` | The server has successfully fulfilled the request and that there is no additional content to send in the response payload body. | +| `201 Created` | The `POST` request was successful and the resource is returned as JSON. | +| `304 Not Modified` | Indicates that the resource has not been modified since the last request. | +| `400 Bad Request` | A required attribute of the API request is missing, e.g., the title of an issue is not given. | +| `401 Unauthorized` | The user is not authenticated, a valid [user token](#authentication) is necessary. | +| `403 Forbidden` | The request is not allowed, e.g., the user is not allowed to delete a project. | +| `404 Not Found` | A resource could not be accessed, e.g., an ID for a resource could not be found. | | `405 Method Not Allowed` | The request is not supported. | -| `409 Conflict` | A conflicting resource already exists, e.g., creating a project with a name that already exists. | -| `412` | Indicates the request was denied. May happen if the `If-Unmodified-Since` header is provided when trying to delete a resource, which was modified in between. | -| `422 Unprocessable` | The entity could not be processed. | -| `500 Server Error` | While handling the request something went wrong server-side. | +| `409 Conflict` | A conflicting resource already exists, e.g., creating a project with a name that already exists. | +| `412` | Indicates the request was denied. May happen if the `If-Unmodified-Since` header is provided when trying to delete a resource, which was modified in between. | +| `422 Unprocessable` | The entity could not be processed. | +| `429 Too Many Requests` | The user exceeded the [application rate limits](../administration/instance_limits.md#rate-limits). | +| `500 Server Error` | While handling the request, something went wrong server-side. | ## Pagination @@ -360,22 +378,22 @@ curl --head --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.exampl The response will then be: ```http -HTTP/1.1 200 OK -Cache-Control: no-cache -Content-Length: 1103 -Content-Type: application/json -Date: Mon, 18 Jan 2016 09:43:18 GMT -Link: <https://gitlab.example.com/api/v4/projects/8/issues/8/notes?page=1&per_page=3>; rel="prev", <https://gitlab.example.com/api/v4/projects/8/issues/8/notes?page=3&per_page=3>; rel="next", <https://gitlab.example.com/api/v4/projects/8/issues/8/notes?page=1&per_page=3>; rel="first", <https://gitlab.example.com/api/v4/projects/8/issues/8/notes?page=3&per_page=3>; rel="last" -Status: 200 OK -Vary: Origin -X-Next-Page: 3 -X-Page: 2 -X-Per-Page: 3 -X-Prev-Page: 1 -X-Request-Id: 732ad4ee-9870-4866-a199-a9db0cde3c86 -X-Runtime: 0.108688 -X-Total: 8 -X-Total-Pages: 3 +HTTP/2 200 OK +cache-control: no-cache +content-length: 1103 +content-type: application/json +date: Mon, 18 Jan 2016 09:43:18 GMT +link: <https://gitlab.example.com/api/v4/projects/8/issues/8/notes?page=1&per_page=3>; rel="prev", <https://gitlab.example.com/api/v4/projects/8/issues/8/notes?page=3&per_page=3>; rel="next", <https://gitlab.example.com/api/v4/projects/8/issues/8/notes?page=1&per_page=3>; rel="first", <https://gitlab.example.com/api/v4/projects/8/issues/8/notes?page=3&per_page=3>; rel="last" +status: 200 OK +vary: Origin +x-next-page: 3 +x-page: 2 +x-per-page: 3 +x-prev-page: 1 +x-request-id: 732ad4ee-9870-4866-a199-a9db0cde3c86 +x-runtime: 0.108688 +x-total: 8 +x-total-pages: 3 ``` #### Other pagination headers @@ -384,12 +402,12 @@ GitLab also returns the following additional pagination headers: | Header | Description | | --------------- | --------------------------------------------- | -| `X-Total` | The total number of items | -| `X-Total-Pages` | The total number of pages | -| `X-Per-Page` | The number of items per page | -| `X-Page` | The index of the current page (starting at 1) | -| `X-Next-Page` | The index of the next page | -| `X-Prev-Page` | The index of the previous page | +| `x-total` | The total number of items | +| `x-total-pages` | The total number of pages | +| `x-per-page` | The number of items per page | +| `x-page` | The index of the current page (starting at 1) | +| `x-next-page` | The index of the next page | +| `X-prev-page` | The index of the previous page | NOTE: **Note:** For GitLab.com users, [some pagination headers may not be returned](../user/gitlab_com/index.md#pagination-response-headers). @@ -588,7 +606,7 @@ Such errors appear in two cases: - A required attribute of the API request is missing, e.g., the title of an issue is not given -- An attribute did not pass the validation, e.g., user bio is too long +- An attribute did not pass the validation, e.g., the user bio is too long When an attribute is missing, you will get something like: diff --git a/doc/api/admin_sidekiq_queues.md b/doc/api/admin_sidekiq_queues.md index 4a2456d6f4a..22488d053b4 100644 --- a/doc/api/admin_sidekiq_queues.md +++ b/doc/api/admin_sidekiq_queues.md @@ -29,6 +29,7 @@ DELETE /admin/sidekiq/queues/:queue_name | `root_namespace` | string | no | The root namespace of the project | | `subscription_plan` | string | no | The subscription plan of the root namespace (GitLab.com only) | | `caller_id` | string | no | The endpoint or background job that schedule the job (for example: `ProjectsController#create`, `/api/:version/projects/:id`, `PostReceive`) | +| `feature_category` | string | no | The feature category of the background job (for example: `issue_tracking` or `code_review`) | At least one attribute, other than `queue_name`, is required. diff --git a/doc/api/api_resources.md b/doc/api/api_resources.md index 898aa713331..199b244b2c3 100644 --- a/doc/api/api_resources.md +++ b/doc/api/api_resources.md @@ -9,7 +9,7 @@ Available resources for the [GitLab API](README.md) can be grouped in the follow See also: - [V3 to V4](v3_to_v4.md). -- Adding [deploy keys for multiple projects](deploy_key_multiple_projects.md). +- Adding [deploy keys for multiple projects](deploy_keys.md#adding-deploy-keys-to-multiple-projects). - [API Resources for various templates](#templates-api-resources). ## Project resources @@ -38,6 +38,7 @@ The following API resources are available in the project context: | [Issues Statistics](issues_statistics.md) | `/projects/:id/issues_statistics` (also available for groups and standalone) | | [Issue boards](boards.md) | `/projects/:id/boards` | | [Issue links](issue_links.md) **(STARTER)** | `/projects/:id/issues/.../links` | +| [Iterations](iterations.md) **(STARTER)** | `/projects/:id/iterations` (also available for groups) | | [Jobs](jobs.md) | `/projects/:id/jobs`, `/projects/:id/pipelines/.../jobs` | | [Labels](labels.md) | `/projects/:id/labels` | | [Managed licenses](managed_licenses.md) **(ULTIMATE)** | `/projects/:id/managed_licenses` | @@ -80,7 +81,7 @@ The following API resources are available in the project context: | [Vulnerability exports](vulnerability_exports.md) **(ULTIMATE)** | `/projects/:id/vulnerability_exports` | | [Project vulnerabilities](project_vulnerabilities.md) **(ULTIMATE)** | `/projects/:id/vulnerabilities` | | [Vulnerability findings](vulnerability_findings.md) **(ULTIMATE)** | `/projects/:id/vulnerability_findings` | -| [Wikis](wikis.md) | `/projects/:id/wikis` | +| [Project wikis](wikis.md) | `/projects/:id/wikis` | ## Group resources @@ -97,6 +98,7 @@ The following API resources are available in the group context: | [Groups](groups.md) | `/groups`, `/groups/.../subgroups` | | [Group badges](group_badges.md) | `/groups/:id/badges` | | [Group issue boards](group_boards.md) | `/groups/:id/boards` | +| [Group iterations](group_iterations.md) **(STARTER)** | `/groups/:id/iterations` (also available for projects) | | [Group labels](group_labels.md) | `/groups/:id/labels` | | [Group-level variables](group_level_variables.md) | `/groups/:id/variables` | | [Group milestones](group_milestones.md) | `/groups/:id/milestones` | @@ -108,6 +110,7 @@ The following API resources are available in the group context: | [Notification settings](notification_settings.md) | `/groups/:id/notification_settings` (also available for projects and standalone) | | [Resource label events](resource_label_events.md) | `/groups/:id/epics/.../resource_label_events` (also available for projects) | | [Search](search.md) | `/groups/:id/search` (also available for projects and standalone) | +| [Group wikis](group_wikis.md) **(PREMIUM)** | `/groups/:id/wikis` | ## Standalone resources diff --git a/doc/api/commits.md b/doc/api/commits.md index da95e9a943f..66b34d4bc75 100644 --- a/doc/api/commits.md +++ b/doc/api/commits.md @@ -614,7 +614,7 @@ Example response: ## Commit status -Since GitLab 8.1, this is the new commit status API. +In GitLab 8.1 and later, this is the new commit status API. ### List the statuses of a commit diff --git a/doc/api/container_registry.md b/doc/api/container_registry.md index e4687994017..3a7ebf9a2aa 100644 --- a/doc/api/container_registry.md +++ b/doc/api/container_registry.md @@ -23,9 +23,8 @@ GET /projects/:id/registry/repositories | Attribute | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) accessible by the authenticated user. | -| `tags` | boolean | no | If the parameter is included as true, each repository will include an array of `"tags"` in the response. | -| `name` | string | no | Returns a list of repositories with a name that matches the value. ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/29763) in GitLab 13.0). | -| `tags_count` | boolean | no | If the parameter is included as true, each repository will include `"tags_count"` in the response ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/32141) in GitLab 13.1). | +| `tags` | boolean | no | If the parameter is included as true, each repository includes an array of `"tags"` in the response. | +| `tags_count` | boolean | no | If the parameter is included as true, each repository includes `"tags_count"` in the response ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/32141) in GitLab 13.1). | ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories" @@ -41,7 +40,8 @@ Example response: "path": "group/project", "project_id": 9, "location": "gitlab.example.com:5000/group/project", - "created_at": "2019-01-10T13:38:57.391Z" + "created_at": "2019-01-10T13:38:57.391Z", + "cleanup_policy_started_at": "2020-01-10T15:40:57.391Z" }, { "id": 2, @@ -49,7 +49,8 @@ Example response: "path": "group/project/releases", "project_id": 9, "location": "gitlab.example.com:5000/group/project/releases", - "created_at": "2019-01-10T13:39:08.229Z" + "created_at": "2019-01-10T13:39:08.229Z", + "cleanup_policy_started_at": "2020-08-17T03:12:35.489Z" } ] ``` @@ -65,9 +66,8 @@ GET /groups/:id/registry/repositories | Attribute | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) accessible by the authenticated user. | -| `tags` | boolean | no | If the parameter is included as true, each repository will include an array of `"tags"` in the response. | -| `name` | string | no | Returns a list of repositories with a name that matches the value. ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/29763) in GitLab 13.0). | -| `tags_count` | boolean | no | If the parameter is included as true, each repository will include `"tags_count"` in the response ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/32141) in GitLab 13.1). | +| `tags` | boolean | no | If the parameter is included as true, each repository includes an array of `"tags"` in the response. | +| `tags_count` | boolean | no | If the parameter is included as true, each repository includes `"tags_count"` in the response ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/32141) in GitLab 13.1). | ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/2/registry/repositories?tags=1&tags_count=true" @@ -84,6 +84,7 @@ Example response: "project_id": 9, "location": "gitlab.example.com:5000/group/project", "created_at": "2019-01-10T13:38:57.391Z", + "cleanup_policy_started_at": "2020-08-17T03:12:35.489Z", "tags_count": 1, "tags": [ { @@ -100,6 +101,7 @@ Example response: "project_id": 11, "location": "gitlab.example.com:5000/group/other_project", "created_at": "2019-01-10T13:39:08.229Z", + "cleanup_policy_started_at": "2020-01-10T15:40:57.391Z", "tags_count": 3, "tags": [ { @@ -228,7 +230,7 @@ DELETE /projects/:id/registry/repositories/:repository_id/tags/:tag_name curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags/v10.0.0" ``` -This action does not delete blobs. In order to delete them and recycle disk space, +This action doesn't delete blobs. To delete them and recycle disk space, [run the garbage collection](https://docs.gitlab.com/omnibus/maintenance/README.html#removing-unused-layers-not-referenced-by-manifests). ## Delete registry repository tags in bulk @@ -248,28 +250,29 @@ DELETE /projects/:id/registry/repositories/:repository_id/tags | `repository_id` | integer | yes | The ID of registry repository. | | `name_regex` | string | no | The [re2](https://github.com/google/re2/wiki/Syntax) regex of the name to delete. To delete all tags specify `.*`. **Note:** `name_regex` is deprecated in favor of `name_regex_delete`. This field is validated. | | `name_regex_delete` | string | yes | The [re2](https://github.com/google/re2/wiki/Syntax) regex of the name to delete. To delete all tags specify `.*`. This field is validated. | -| `name_regex_keep` | string | no | The [re2](https://github.com/google/re2/wiki/Syntax) regex of the name to keep. This value will override any matches from `name_regex_delete`. This field is validated. Note: setting to `.*` will result in a no-op. | +| `name_regex_keep` | string | no | The [re2](https://github.com/google/re2/wiki/Syntax) regex of the name to keep. This value overrides any matches from `name_regex_delete`. This field is validated. Note: setting to `.*` results in a no-op. | | `keep_n` | integer | no | The amount of latest tags of given name to keep. | | `older_than` | string | no | Tags to delete that are older than the given time, written in human readable form `1h`, `1d`, `1month`. | This API call performs the following operations: -1. It orders all tags by creation date. The creation date is the time of the - manifest creation, not the time of tag push. -1. It removes only the tags matching the given `name_regex_delete` (or deprecated `name_regex`), keeping any that match `name_regex_keep`. -1. It never removes the tag named `latest`. -1. It keeps N latest matching tags (if `keep_n` is specified). -1. It only removes tags that are older than X amount of time (if `older_than` is specified). -1. It schedules the asynchronous job to be executed in the background. - -These operations are executed asynchronously and it might -take time to get executed. You can run this at most -once an hour for a given container repository. -This action does not delete blobs. In order to delete them and recycle disk space, +- It orders all tags by creation date. The creation date is the time of the + manifest creation, not the time of tag push. +- It removes only the tags matching the given `name_regex_delete` (or deprecated + `name_regex`), keeping any that match `name_regex_keep`. +- It never removes the tag named `latest`. +- It keeps N latest matching tags (if `keep_n` is specified). +- It only removes tags that are older than X amount of time (if `older_than` is + specified). +- It schedules the asynchronous job to be executed in the background. + +These operations are executed asynchronously and can take time to get executed. +You can run this at most once an hour for a given container repository. This +action doesn't delete blobs. To delete them and recycle disk space, [run the garbage collection](https://docs.gitlab.com/omnibus/maintenance/README.html#removing-unused-layers-not-referenced-by-manifests). NOTE: **Note:** -Since GitLab 12.4, individual tags are deleted. +In GitLab 12.4 and later, individual tags are deleted. For more details, see the [discussion](https://gitlab.com/gitlab-org/gitlab/-/issues/15737). Examples: diff --git a/doc/api/epics.md b/doc/api/epics.md index 91ea92c8589..5c7366c8457 100644 --- a/doc/api/epics.md +++ b/doc/api/epics.md @@ -267,6 +267,7 @@ POST /groups/:id/epics | `labels` | string | no | The comma separated list of labels | | `description` | string | no | The description of the epic. Limited to 1,048,576 characters. | | `confidential` | boolean | no | Whether the epic should be confidential | +| `created_at` | string | no | When the epic was created. Date time string, ISO 8601 formatted, for example `2016-03-11T03:45:40Z` . Requires administrator or project/group owner privileges ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/255309) in GitLab 13.5) | | `start_date_is_fixed` | boolean | no | Whether start date should be sourced from `start_date_fixed` or from milestones (since 11.3) | | `start_date_fixed` | string | no | The fixed start date of an epic (since 11.3) | | `due_date_is_fixed` | boolean | no | Whether due date should be sourced from `due_date_fixed` or from milestones (since 11.3) | @@ -349,6 +350,7 @@ PUT /groups/:id/epics/:epic_iid | `description` | string | no | The description of an epic. Limited to 1,048,576 characters. | | `confidential` | boolean | no | Whether the epic should be confidential | | `labels` | string | no | The comma separated list of labels | +| `updated_at` | string | no | When the epic was updated. Date time string, ISO 8601 formatted, for example `2016-03-11T03:45:40Z` . Requires administrator or project/group owner privileges ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/255309) in GitLab 13.5) | | `start_date_is_fixed` | boolean | no | Whether start date should be sourced from `start_date_fixed` or from milestones (since 11.3) | | `start_date_fixed` | string | no | The fixed start date of an epic (since 11.3) | | `due_date_is_fixed` | boolean | no | Whether due date should be sourced from `due_date_fixed` or from milestones (since 11.3) | @@ -422,10 +424,10 @@ DELETE /groups/:id/epics/:epic_iid curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/1/epics/5" ``` -## Create a to-do +## Create a to do -Manually creates a to-do for the current user on an epic. If -there already exists a to-do for the user on that epic, status code `304` is +Manually creates a to do for the current user on an epic. If +there already exists a to do for the user on that epic, status code `304` is returned. ```plaintext diff --git a/doc/api/experiments.md b/doc/api/experiments.md new file mode 100644 index 00000000000..66c444e54ce --- /dev/null +++ b/doc/api/experiments.md @@ -0,0 +1,40 @@ +--- +stage: Growth +group: Expansion +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Experiments API + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/262725) in GitLab 13.5. + +This API is for listing Experiments [experiment use in development of GitLab](../development/experiment_guide/index.md). + +All methods require user be a [GitLab team member](https://gitlab.com/groups/gitlab-com/-/group_members) for authorization. + +## List all experiments + +Get a list of all experiments, with its enabled status. + +```plaintext +GET /experiments +``` + +```shell +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/experiments" +``` + +Example response: + +```json +[ + { + "key": "experiment_1", + "enabled": true + }, + { + "key": "experiment_2", + "enabled": false + } +] +``` diff --git a/doc/api/feature_flag_user_lists.md b/doc/api/feature_flag_user_lists.md index 460f3727819..b44cb1fb9f2 100644 --- a/doc/api/feature_flag_user_lists.md +++ b/doc/api/feature_flag_user_lists.md @@ -4,9 +4,10 @@ group: Progressive Delivery info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# Feature flag user lists API **(PREMIUM)** +# Feature flag user lists API **(CORE)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/205409) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.10. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/205409) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.10. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/212318) to GitLab Core in 13.5. API for accessing GitLab Feature Flag User Lists. diff --git a/doc/api/feature_flags.md b/doc/api/feature_flags.md index 1088154b599..fbda873f866 100644 --- a/doc/api/feature_flags.md +++ b/doc/api/feature_flags.md @@ -4,9 +4,11 @@ group: Progressive Delivery info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# Feature Flags API **(PREMIUM)** +# Feature Flags API **(CORE)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9566) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.5. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9566) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.5. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/212318) to [GitLab Starter](https://about.gitlab.com/pricing/) in 13.4. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/212318) to [GitLab Core](https://about.gitlab.com/pricing/) in 13.5. NOTE: **Note:** This API is behind a [feature flag](../operations/feature_flags.md#enable-or-disable-feature-flag-strategies). @@ -151,7 +153,7 @@ POST /projects/:id/feature_flags | `description` | string | no | The description of the feature flag. | | `active` | boolean | no | The active state of the flag. Defaults to true. [Supported](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/38350) in GitLab 13.3 and later. | | `strategies` | JSON | no | The feature flag [strategies](../operations/feature_flags.md#feature-flag-strategies). | -| `strategies:name` | JSON | no | The strategy name. | +| `strategies:name` | JSON | no | The strategy name. Can be `default`, `gradualRolloutUserId`, `userWithId`, or `gitlabUserList`. In [GitLab 13.5](https://gitlab.com/gitlab-org/gitlab/-/issues/36380) and later, can be [`flexibleRollout`](https://unleash.github.io/docs/activation_strategy#flexiblerollout). | | `strategies:parameters` | JSON | no | The strategy parameters. | | `strategies:scopes` | JSON | no | The scopes for the strategy. | | `strategies:scopes:environment_scope` | string | no | The environment spec for the scope. | diff --git a/doc/api/feature_flags_legacy.md b/doc/api/feature_flags_legacy.md index 175261b3a7b..a7c139a02ba 100644 --- a/doc/api/feature_flags_legacy.md +++ b/doc/api/feature_flags_legacy.md @@ -4,9 +4,11 @@ group: Progressive Delivery info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# Legacy Feature Flags API **(PREMIUM)** +# Legacy Feature Flags API **(CORE)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9566) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.5. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9566) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.5. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/212318) to [GitLab Starter](https://about.gitlab.com/pricing/) in 13.4. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/212318) to [GitLab Core](https://about.gitlab.com/pricing/) in 13.5. CAUTION: **Deprecation:** This API is deprecated and [scheduled for removal in GitLab 14.0](https://gitlab.com/gitlab-org/gitlab/-/issues/213369). Use [this API](feature_flags.md) instead. diff --git a/doc/api/geo_nodes.md b/doc/api/geo_nodes.md index 8d2052f7373..064bd26ee72 100644 --- a/doc/api/geo_nodes.md +++ b/doc/api/geo_nodes.md @@ -1,7 +1,7 @@ # Geo Nodes API **(PREMIUM ONLY)** -In order to interact with Geo node endpoints, you need to authenticate yourself -as an admin. +To interact with Geo node endpoints, you need to authenticate yourself as an +admin. ## Create a new Geo node @@ -460,12 +460,12 @@ Example response: "package_files_registry_count": 10, "package_files_synced_count": 6, "package_files_failed_count": 3, - "terraform_states_count": 10, - "terraform_states_checksummed_count": 10, - "terraform_states_checksum_failed_count": 0, - "terraform_states_registry_count": 10, - "terraform_states_synced_count": 6, - "terraform_states_failed_count": 3 + "terraform_state_versions_count": 10, + "terraform_state_versions_checksummed_count": 10, + "terraform_state_versions_checksum_failed_count": 0, + "terraform_state_versions_registry_count": 10, + "terraform_state_versions_synced_count": 6, + "terraform_state_versions_failed_count": 3, "snippet_repositories_count": 10, "snippet_repositories_checksummed_count": 10, "snippet_repositories_checksum_failed_count": 0, diff --git a/doc/api/graphql/reference/gitlab_schema.graphql b/doc/api/graphql/reference/gitlab_schema.graphql index 7f5ff04bcee..a44f8f70311 100644 --- a/doc/api/graphql/reference/gitlab_schema.graphql +++ b/doc/api/graphql/reference/gitlab_schema.graphql @@ -32,7 +32,7 @@ input AddAwardEmojiInput { """ The global id of the awardable resource """ - awardableId: ID! + awardableId: AwardableID! """ A unique identifier for the client performing the mutation. @@ -77,7 +77,7 @@ input AddProjectToSecurityDashboardInput { """ ID of the project to be added to Instance Security Dashboard """ - id: ID! + id: ProjectID! } """ @@ -115,6 +115,11 @@ input AdminSidekiqQueuesDeleteJobsInput { clientMutationId: String """ + Delete jobs matching feature_category in the context metadata + """ + featureCategory: String + + """ Delete jobs matching project in the context metadata """ project: String @@ -245,6 +250,11 @@ type AlertManagementAlert implements Noteable { endedAt: Time """ + Environment for the alert + """ + environment: Environment + + """ Number of events of this alert """ eventCount: Int @@ -435,6 +445,16 @@ Values for sorting alerts """ enum AlertManagementAlertSort { """ + Created at ascending order + """ + CREATED_ASC + + """ + Created at descending order + """ + CREATED_DESC + + """ Created time by ascending order """ CREATED_TIME_ASC @@ -495,6 +515,16 @@ enum AlertManagementAlertSort { STATUS_DESC """ + Updated at ascending order + """ + UPDATED_ASC + + """ + Updated at descending order + """ + UPDATED_DESC + + """ Created time by ascending order """ UPDATED_TIME_ASC @@ -507,22 +537,22 @@ enum AlertManagementAlertSort { """ Created at ascending order """ - created_asc + created_asc @deprecated(reason: "Use CREATED_ASC. Deprecated in 13.5") """ Created at descending order """ - created_desc + created_desc @deprecated(reason: "Use CREATED_DESC. Deprecated in 13.5") """ Updated at ascending order """ - updated_asc + updated_asc @deprecated(reason: "Use UPDATED_ASC. Deprecated in 13.5") """ Updated at descending order """ - updated_desc + updated_desc @deprecated(reason: "Use UPDATED_DESC. Deprecated in 13.5") } """ @@ -772,7 +802,7 @@ input AwardEmojiAddInput { """ The global id of the awardable resource """ - awardableId: ID! + awardableId: AwardableID! """ A unique identifier for the client performing the mutation. @@ -812,7 +842,7 @@ input AwardEmojiRemoveInput { """ The global id of the awardable resource """ - awardableId: ID! + awardableId: AwardableID! """ A unique identifier for the client performing the mutation. @@ -852,7 +882,7 @@ input AwardEmojiToggleInput { """ The global id of the awardable resource """ - awardableId: ID! + awardableId: AwardableID! """ A unique identifier for the client performing the mutation. @@ -890,6 +920,11 @@ type AwardEmojiTogglePayload { toggledOn: Boolean! } +""" +Identifier of Awardable +""" +scalar AwardableID + type BaseService implements Service { """ Indicates if the service is active @@ -1035,7 +1070,7 @@ type Board { Returns the last _n_ elements from the list. """ last: Int - ): EpicConnection + ): BoardEpicConnection """ Whether or not backlog list is hidden. @@ -1053,6 +1088,31 @@ type Board { id: ID! """ + Labels of the board + """ + labels( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + ): LabelConnection + + """ Lists of the board """ lists( @@ -1077,6 +1137,11 @@ type Board { id: ID """ + Filters applied when getting issue metadata in the board list + """ + issueFilters: BoardIssueInput + + """ Returns the last _n_ elements from the list. """ last: Int @@ -1134,6 +1199,484 @@ type BoardEdge { } """ +Represents an epic on an issue board +""" +type BoardEpic implements CurrentUserTodos & Noteable { + """ + Author of the epic + """ + author: User! + + """ + Children (sub-epics) of the epic + """ + children( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Filter epics by author + """ + authorUsername: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + List items overlapping a time frame defined by startDate..endDate (if one + date is provided, both must be present). Deprecated in 13.5: Use timeframe.end + """ + endDate: Time + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + IID of the epic, e.g., "1" + """ + iid: ID + + """ + Filter epics by iid for autocomplete + """ + iidStartsWith: String + + """ + List of IIDs of epics, e.g., [1, 2] + """ + iids: [ID!] + + """ + Filter epics by labels + """ + labelName: [String!] + + """ + Returns the last _n_ elements from the list. + """ + last: Int + + """ + Filter epics by milestone title, computed from epic's issues + """ + milestoneTitle: String + + """ + Search query for epic title or description + """ + search: String + + """ + List epics by sort order + """ + sort: EpicSort + + """ + List items overlapping a time frame defined by startDate..endDate (if one + date is provided, both must be present). Deprecated in 13.5: Use + timeframe.start + """ + startDate: Time + + """ + Filter epics by state + """ + state: EpicState + + """ + List items overlapping the given timeframe + """ + timeframe: Timeframe + ): EpicConnection + + """ + Timestamp of when the epic was closed + """ + closedAt: Time + + """ + Indicates if the epic is confidential + """ + confidential: Boolean + + """ + Timestamp of when the epic was created + """ + createdAt: Time + + """ + Todos for the current user + """ + currentUserTodos( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + + """ + State of the todos + """ + state: TodoStateEnum + ): TodoConnection! + + """ + Number of open and closed descendant epics and issues + """ + descendantCounts: EpicDescendantCount + + """ + Total weight of open and closed issues in the epic and its descendants + """ + descendantWeightSum: EpicDescendantWeights + + """ + Description of the epic + """ + description: String + + """ + All discussions on this noteable + """ + discussions( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + ): DiscussionConnection! + + """ + Number of downvotes the epic has received + """ + downvotes: Int! + + """ + Due date of the epic + """ + dueDate: Time + + """ + Fixed due date of the epic + """ + dueDateFixed: Time + + """ + Inherited due date of the epic from milestones + """ + dueDateFromMilestones: Time + + """ + Indicates if the due date has been manually set + """ + dueDateIsFixed: Boolean + + """ + Group to which the epic belongs + """ + group: Group! + + """ + Indicates if the epic has children + """ + hasChildren: Boolean! + + """ + Indicates if the epic has direct issues + """ + hasIssues: Boolean! + + """ + Indicates if the epic has a parent epic + """ + hasParent: Boolean! + + """ + Current health status of the epic + """ + healthStatus: EpicHealthStatus + + """ + ID of the epic + """ + id: ID! + + """ + Internal ID of the epic + """ + iid: ID! + + """ + A list of issues associated with the epic + """ + issues( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + ): EpicIssueConnection + + """ + Labels assigned to the epic + """ + labels( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + ): LabelConnection + + """ + All notes on this noteable + """ + notes( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + ): NoteConnection! + + """ + Parent epic of the epic + """ + parent: Epic + + """ + List of participants for the epic + """ + participants( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + ): UserConnection + + """ + Internal reference of the epic. Returned in shortened format by default + """ + reference( + """ + Indicates if the reference should be returned in full + """ + full: Boolean = false + ): String! + + """ + URI path of the epic-issue relationship + """ + relationPath: String + + """ + The relative position of the epic in the epic tree + """ + relativePosition: Int + + """ + Start date of the epic + """ + startDate: Time + + """ + Fixed start date of the epic + """ + startDateFixed: Time + + """ + Inherited start date of the epic from milestones + """ + startDateFromMilestones: Time + + """ + Indicates if the start date has been manually set + """ + startDateIsFixed: Boolean + + """ + State of the epic + """ + state: EpicState! + + """ + Indicates the currently logged in user is subscribed to the epic + """ + subscribed: Boolean! + + """ + Title of the epic + """ + title: String + + """ + Timestamp of when the epic was updated + """ + updatedAt: Time + + """ + Number of upvotes the epic has received + """ + upvotes: Int! + + """ + Permissions for the current user on the resource + """ + userPermissions: EpicPermissions! + + """ + User preferences for the epic on the issue board + """ + userPreferences: BoardEpicUserPreferences + + """ + Web path of the epic + """ + webPath: String! + + """ + Web URL of the epic + """ + webUrl: String! +} + +""" +The connection type for BoardEpic. +""" +type BoardEpicConnection { + """ + A list of edges. + """ + edges: [BoardEpicEdge] + + """ + A list of nodes. + """ + nodes: [BoardEpic] + + """ + Information to aid in pagination. + """ + pageInfo: PageInfo! +} + +""" +An edge in a connection. +""" +type BoardEpicEdge { + """ + A cursor for use in pagination. + """ + cursor: String! + + """ + The item at the end of the edge. + """ + node: BoardEpic +} + +""" +Represents user preferences for a board epic +""" +type BoardEpicUserPreferences { + """ + Indicates epic should be displayed as collapsed + """ + collapsed: Boolean! +} + +""" Identifier of Board """ scalar BoardID @@ -1402,7 +1945,7 @@ input BoardListUpdateLimitMetricsInput { """ The global ID of the list. """ - listId: ID! + listId: ListID! """ The new maximum issue count limit. @@ -1479,6 +2022,11 @@ type BurnupChartDailyTotals { type CiGroup { """ + Detailed status of the group + """ + detailedStatus: DetailedStatus + + """ Jobs in group """ jobs( @@ -1551,6 +2099,11 @@ type CiGroupEdge { type CiJob { """ + Detailed status of the job + """ + detailedStatus: DetailedStatus + + """ Name of the job """ name: String @@ -1579,6 +2132,11 @@ type CiJob { """ last: Int ): CiJobConnection + + """ + Schedule for the build + """ + scheduledAt: Time } """ @@ -1623,6 +2181,11 @@ scalar CiPipelineID type CiStage { """ + Detailed status of the stage + """ + detailedStatus: DetailedStatus + + """ Group of jobs for the stage """ groups( @@ -1937,6 +2500,11 @@ Identifier of Clusters::AgentToken """ scalar ClustersAgentTokenID +""" +Identifier of Clusters::Cluster +""" +scalar ClustersClusterID + type Commit { """ Author of the commit @@ -2481,7 +3049,7 @@ input CreateAnnotationInput { """ The global id of the cluster to add an annotation to """ - clusterId: ID + clusterId: ClustersClusterID """ The path to a file defining the dashboard on which the annotation should be added @@ -2501,7 +3069,7 @@ input CreateAnnotationInput { """ The global id of the environment to add an annotation to """ - environmentId: ID + environmentId: EnvironmentID """ Timestamp indicating starting moment to which the annotation relates @@ -2530,6 +3098,71 @@ type CreateAnnotationPayload { } """ +Autogenerated input type of CreateBoard +""" +input CreateBoardInput { + """ + The ID of the user to be assigned to the board. + """ + assigneeId: String + + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + The group full path the board is associated with. + """ + groupPath: ID + + """ + The IDs of labels to be added to the board. + """ + labelIds: [ID!] + + """ + The ID of the milestone to be assigned to the board. + """ + milestoneId: ID + + """ + The board name. + """ + name: String + + """ + The project full path the board is associated with. + """ + projectPath: ID + + """ + The weight of the board. + """ + weight: Boolean +} + +""" +Autogenerated return type of CreateBoard +""" +type CreateBoardPayload { + """ + The board after mutation. + """ + board: Board + + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Errors encountered during execution of the mutation. + """ + errors: [String!]! +} + +""" Autogenerated input type of CreateBranch """ input CreateBranchInput { @@ -2636,7 +3269,7 @@ input CreateDiffNoteInput { """ The global id of the resource to add a note to """ - noteableId: ID! + noteableId: NoteableID! """ The position of this note on a diff @@ -2766,7 +3399,7 @@ input CreateImageDiffNoteInput { """ The global id of the resource to add a note to """ - noteableId: ID! + noteableId: NoteableID! """ The position of this note on a diff @@ -2795,6 +3428,121 @@ type CreateImageDiffNotePayload { } """ +Autogenerated input type of CreateIssue +""" +input CreateIssueInput { + """ + The array of user IDs to assign to the issue + """ + assigneeIds: [UserID!] + + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Indicates the issue is confidential + """ + confidential: Boolean + + """ + Timestamp when the issue was created. Available only for admins and project owners + """ + createdAt: Time + + """ + Description of the issue + """ + description: String + + """ + The ID of a discussion to resolve. Also pass `merge_request_to_resolve_discussions_of` + """ + discussionToResolve: String + + """ + Due date of the issue + """ + dueDate: ISO8601Date + + """ + The ID of an epic to associate the issue with + """ + epicId: EpicID + + """ + The desired health status + """ + healthStatus: HealthStatus + + """ + The IID (internal ID) of a project issue. Only admins and project owners can modify + """ + iid: Int + + """ + The IDs of labels to be added to the issue + """ + labelIds: [LabelID!] + + """ + Labels of the issue + """ + labels: [String!] + + """ + Indicates discussion is locked on the issue + """ + locked: Boolean + + """ + The IID of a merge request for which to resolve discussions + """ + mergeRequestToResolveDiscussionsOf: MergeRequestID + + """ + The ID of the milestone to assign to the issue. On update milestone will be removed if set to null + """ + milestoneId: MilestoneID + + """ + Project full path the issue is associated with + """ + projectPath: ID! + + """ + Title of the issue + """ + title: String! + + """ + The weight of the issue + """ + weight: Int +} + +""" +Autogenerated return type of CreateIssue +""" +type CreateIssuePayload { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Errors encountered during execution of the mutation. + """ + errors: [String!]! + + """ + The issue after mutation + """ + issue: Issue +} + +""" Autogenerated input type of CreateIteration """ input CreateIterationInput { @@ -2876,12 +3624,12 @@ input CreateNoteInput { """ The global id of the discussion this note is in reply to """ - discussionId: ID + discussionId: DiscussionID """ The global id of the resource to add a note to """ - noteableId: ID! + noteableId: NoteableID! } """ @@ -2914,14 +3662,19 @@ input CreateRequirementInput { clientMutationId: String """ - The project full path the requirement is associated with + Description of the requirement + """ + description: String + + """ + Full project path the requirement is associated with """ projectPath: ID! """ Title of the requirement """ - title: String! + title: String } """ @@ -2939,7 +3692,7 @@ type CreateRequirementPayload { errors: [String!]! """ - The requirement after mutation + Requirement after mutation """ requirement: Requirement } @@ -3002,6 +3755,11 @@ type CreateSnippetPayload { The snippet after mutation """ snippet: Snippet + + """ + Indicates whether the operation returns a record detected as spam + """ + spam: Boolean } """ @@ -3133,6 +3891,11 @@ type DastOnDemandScanCreatePayload { enum DastScanTypeEnum { """ + Active DAST scan. This scan will make active attacks against the target site. + """ + ACTIVE + + """ Passive DAST scan. This scan will not make active attacks against the target site. """ PASSIVE @@ -3163,6 +3926,16 @@ type DastScannerProfile { profileName: String """ + Indicates the type of DAST scan that will run. Either a Passive Scan or an Active Scan. + """ + scanType: DastScanTypeEnum + + """ + Indicates if debug messages should be included in DAST console output. True to include the debug messages. + """ + showDebugMessages: Boolean! + + """ The maximum number of minutes allowed for the spider to traverse the site """ spiderTimeout: Int @@ -3171,6 +3944,13 @@ type DastScannerProfile { The maximum number of seconds allowed for the site under test to respond to a request """ targetTimeout: Int + + """ + Indicates if the AJAX spider should be used to crawl the target site. True to + run the AJAX spider in addition to the traditional spider, and false to run + only the traditional spider. + """ + useAjaxSpider: Boolean! } """ @@ -3213,6 +3993,16 @@ input DastScannerProfileCreateInput { profileName: String! """ + Indicates the type of DAST scan that will run. Either a Passive Scan or an Active Scan. + """ + scanType: DastScanTypeEnum = PASSIVE + + """ + Indicates if debug messages should be included in DAST console output. True to include the debug messages. + """ + showDebugMessages: Boolean = false + + """ The maximum number of minutes allowed for the spider to traverse the site. """ spiderTimeout: Int @@ -3221,6 +4011,13 @@ input DastScannerProfileCreateInput { The maximum number of seconds allowed for the site under test to respond to a request. """ targetTimeout: Int + + """ + Indicates if the AJAX spider should be used to crawl the target site. True to + run the AJAX spider in addition to the traditional spider, and false to run + only the traditional spider. + """ + useAjaxSpider: Boolean = false } """ @@ -3328,6 +4125,16 @@ input DastScannerProfileUpdateInput { profileName: String! """ + Indicates the type of DAST scan that will run. Either a Passive Scan or an Active Scan. + """ + scanType: DastScanTypeEnum + + """ + Indicates if debug messages should be included in DAST console output. True to include the debug messages. + """ + showDebugMessages: Boolean + + """ The maximum number of minutes allowed for the spider to traverse the site. """ spiderTimeout: Int! @@ -3336,6 +4143,13 @@ input DastScannerProfileUpdateInput { The maximum number of seconds allowed for the site under test to respond to a request. """ targetTimeout: Int! + + """ + Indicates if the AJAX spider should be used to crawl the target site. True to + run the AJAX spider in addition to the traditional spider, and false to run + only the traditional spider. + """ + useAjaxSpider: Boolean } """ @@ -3596,6 +4410,66 @@ enum DastSiteProfileValidationStatusEnum { } """ +Autogenerated input type of DastSiteTokenCreate +""" +input DastSiteTokenCreateInput { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + The project the site token belongs to. + """ + fullPath: ID! + + """ + The URL of the target to be validated. + """ + targetUrl: String +} + +""" +Autogenerated return type of DastSiteTokenCreate +""" +type DastSiteTokenCreatePayload { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Errors encountered during execution of the mutation. + """ + errors: [String!]! + + """ + ID of the site token. + """ + id: DastSiteTokenID + + """ + The current validation status of the target. + """ + status: DastSiteProfileValidationStatusEnum + + """ + Token string. + """ + token: String +} + +""" +Identifier of DastSiteToken +""" +scalar DastSiteTokenID + +""" +Date represented in ISO 8601 +""" +scalar Date + +""" Autogenerated input type of DeleteAnnotation """ input DeleteAnnotationInput { @@ -3920,6 +4794,11 @@ A collection of designs """ type DesignCollection { """ + Copy state of the design collection + """ + copyState: DesignCollectionCopyState + + """ Find a specific design """ design( @@ -4047,6 +4926,26 @@ type DesignCollection { } """ +Copy state of a DesignCollection +""" +enum DesignCollectionCopyState { + """ + The DesignCollection encountered an error during a copy + """ + ERROR + + """ + The DesignCollection is being copied + """ + IN_PROGRESS + + """ + The DesignCollection has no copy in progress + """ + READY +} + +""" The connection type for Design. """ type DesignConnection { @@ -4471,6 +5370,41 @@ input DestroyBoardInput { } """ +Autogenerated input type of DestroyBoardList +""" +input DestroyBoardListInput { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Global ID of the list to destroy. Only label lists are accepted. + """ + listId: ListID! +} + +""" +Autogenerated return type of DestroyBoardList +""" +type DestroyBoardListPayload { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Errors encountered during execution of the mutation. + """ + errors: [String!]! + + """ + The list after mutation. + """ + list: BoardList +} + +""" Autogenerated return type of DestroyBoard """ type DestroyBoardPayload { @@ -4502,7 +5436,7 @@ input DestroyNoteInput { """ The global id of the note to destroy """ - id: ID! + id: NoteID! } """ @@ -4562,44 +5496,49 @@ type DestroySnippetPayload { type DetailedStatus { """ - Path of the details for the pipeline status + Action information for the status. This includes method, button title, icon, path, and title + """ + action: StatusAction + + """ + Path of the details for the status """ - detailsPath: String! + detailsPath: String """ - Favicon of the pipeline status + Favicon of the status """ - favicon: String! + favicon: String """ - Group of the pipeline status + Group of the status """ - group: String! + group: String """ - Indicates if the pipeline status has further details + Indicates if the status has further details """ - hasDetails: Boolean! + hasDetails: Boolean """ - Icon of the pipeline status + Icon of the status """ - icon: String! + icon: String """ - Label of the pipeline status + Label of the status """ - label: String! + label: String """ - Text of the pipeline status + Text of the status """ - text: String! + text: String """ - Tooltip associated with the pipeline status + Tooltip associated with the status """ - tooltip: String! + tooltip: String } input DiffImagePositionInput { @@ -4915,6 +5854,11 @@ type DiscussionEdge { } """ +Identifier of Discussion +""" +scalar DiscussionID + +""" Autogenerated input type of DiscussionToggleResolve """ input DiscussionToggleResolveInput { @@ -4926,7 +5870,7 @@ input DiscussionToggleResolveInput { """ The global id of the discussion """ - id: ID! + id: DiscussionID! """ Will resolve the discussion when true, and unresolve the discussion when false @@ -4971,7 +5915,7 @@ input DismissVulnerabilityInput { """ ID of the vulnerability to be dismissed """ - id: ID! + id: VulnerabilityID! } """ @@ -5045,7 +5989,7 @@ type Environment { id: ID! """ - The most severe open alert for the environment. If multiple alerts have equal severity, the most recent is returned. + The most severe open alert for the environment. If multiple alerts have equal severity, the most recent is returned """ latestOpenedMostSevereAlert: AlertManagementAlert @@ -5065,6 +6009,12 @@ type Environment { name: String! """ + The path to the environment. Will always return null if + `expose_environment_path_in_alert_details` feature flag is disabled + """ + path: String + + """ State of the environment, for example: available/stopped """ state: String! @@ -5106,6 +6056,11 @@ type EnvironmentEdge { } """ +Identifier of Environment +""" +scalar EnvironmentID + +""" Represents an epic """ type Epic implements CurrentUserTodos & Noteable { @@ -5134,8 +6089,8 @@ type Epic implements CurrentUserTodos & Noteable { before: String """ - List items within a time frame where items.end_date is between startDate and - endDate parameters (startDate parameter must be present) + List items overlapping a time frame defined by startDate..endDate (if one + date is provided, both must be present). Deprecated in 13.5: Use timeframe.end """ endDate: Time @@ -5185,8 +6140,9 @@ type Epic implements CurrentUserTodos & Noteable { sort: EpicSort """ - List items within a time frame where items.start_date is between startDate - and endDate parameters (endDate parameter must be present) + List items overlapping a time frame defined by startDate..endDate (if one + date is provided, both must be present). Deprecated in 13.5: Use + timeframe.start """ startDate: Time @@ -5194,10 +6150,15 @@ type Epic implements CurrentUserTodos & Noteable { Filter epics by state """ state: EpicState + + """ + List items overlapping the given timeframe + """ + timeframe: Timeframe ): EpicConnection """ - Timestamp of the epic's closure + Timestamp of when the epic was closed """ closedAt: Time @@ -5207,7 +6168,7 @@ type Epic implements CurrentUserTodos & Noteable { confidential: Boolean """ - Timestamp of the epic's creation + Timestamp of when the epic was created """ createdAt: Time @@ -5502,7 +6463,7 @@ type Epic implements CurrentUserTodos & Noteable { title: String """ - Timestamp of the epic's last activity + Timestamp of when the epic was updated """ updatedAt: Time @@ -5967,6 +6928,11 @@ type EpicIssue implements CurrentUserTodos & Noteable { severity: IssuableSeverity """ + Timestamp of when the issue SLA expires. + """ + slaDueAt: Time + + """ State of the issue """ state: IssueState! @@ -6233,17 +7199,17 @@ input EpicTreeNodeFieldsInputType { """ The id of the epic_issue or issue that the actual epic or issue is switched with """ - adjacentReferenceId: ID + adjacentReferenceId: EpicTreeSortingID """ The id of the epic_issue or epic that is being moved """ - id: ID! + id: EpicTreeSortingID! """ ID of the new parent epic """ - newParentId: ID + newParentId: EpicID """ The type of the switch, after or before allowed @@ -6258,7 +7224,7 @@ input EpicTreeReorderInput { """ The id of the base epic of the tree """ - baseEpicId: ID! + baseEpicId: EpicID! """ A unique identifier for the client performing the mutation. @@ -6287,6 +7253,11 @@ type EpicTreeReorderPayload { } """ +Identifier of EpicTreeSorting +""" +scalar EpicTreeSortingID + +""" Epic ID wildcard values """ enum EpicWildcardId { @@ -6328,6 +7299,36 @@ type GeoNode { internalUrl: String """ + Find merge request diff registries on this Geo node + """ + mergeRequestDiffRegistries( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Filters registries by their ID + """ + ids: [ID!] + + """ + Returns the last _n_ elements from the list. + """ + last: Int + ): MergeRequestDiffRegistryConnection + + """ The interval (in days) in which the repository verification is valid. Once expired, it will be reverified """ minimumReverificationInterval: Int @@ -6418,10 +7419,9 @@ type GeoNode { syncObjectStorage: Boolean """ - Find terraform state registries on this Geo node. Available only when feature - flag `geo_terraform_state_replication` is enabled + Find terraform state version registries on this Geo node """ - terraformStateRegistries( + terraformStateVersionRegistries( """ Returns the elements in the list that come after the specified cursor. """ @@ -6446,7 +7446,7 @@ type GeoNode { Returns the last _n_ elements from the list. """ last: Int - ): TerraformStateRegistryConnection + ): TerraformStateVersionRegistryConnection """ The user-facing URL for this Geo node @@ -6493,6 +7493,16 @@ type GrafanaIntegration { type Group { """ + Size limit for repositories in the namespace in bytes + """ + actualRepositorySizeLimit: Float + + """ + Additional storage purchased for the root namespace in bytes + """ + additionalPurchasedStorageSize: Float + + """ Indicates whether Auto DevOps is enabled for all projects within this group """ autoDevopsEnabled: Boolean @@ -6507,9 +7517,9 @@ type Group { """ board( """ - Find a board by its ID + The board's ID """ - id: ID + id: BoardID! ): Board """ @@ -6543,6 +7553,11 @@ type Group { ): BoardConnection """ + Includes at least one project where the repository size exceeds the limit + """ + containsLockedProjects: Boolean! + + """ Description of the namespace """ description: String @@ -6567,8 +7582,8 @@ type Group { authorUsername: String """ - List items within a time frame where items.end_date is between startDate and - endDate parameters (startDate parameter must be present) + List items overlapping a time frame defined by startDate..endDate (if one + date is provided, both must be present). Deprecated in 13.5: Use timeframe.end """ endDate: Time @@ -6608,8 +7623,9 @@ type Group { sort: EpicSort """ - List items within a time frame where items.start_date is between startDate - and endDate parameters (endDate parameter must be present) + List items overlapping a time frame defined by startDate..endDate (if one + date is provided, both must be present). Deprecated in 13.5: Use + timeframe.start """ startDate: Time @@ -6617,6 +7633,11 @@ type Group { Filter epics by state """ state: EpicState + + """ + List items overlapping the given timeframe + """ + timeframe: Timeframe ): Epic """ @@ -6639,8 +7660,8 @@ type Group { before: String """ - List items within a time frame where items.end_date is between startDate and - endDate parameters (startDate parameter must be present) + List items overlapping a time frame defined by startDate..endDate (if one + date is provided, both must be present). Deprecated in 13.5: Use timeframe.end """ endDate: Time @@ -6690,8 +7711,9 @@ type Group { sort: EpicSort """ - List items within a time frame where items.start_date is between startDate - and endDate parameters (endDate parameter must be present) + List items overlapping a time frame defined by startDate..endDate (if one + date is provided, both must be present). Deprecated in 13.5: Use + timeframe.start """ startDate: Time @@ -6699,6 +7721,11 @@ type Group { Filter epics by state """ state: EpicState + + """ + List items overlapping the given timeframe + """ + timeframe: Timeframe ): EpicConnection """ @@ -6762,7 +7789,7 @@ type Group { isTemporaryStorageIncreaseEnabled: Boolean! """ - Issues of the group + Issues for projects in this group """ issues( """ @@ -6781,6 +7808,16 @@ type Group { assigneeUsername: String """ + Usernames of users assigned to the issue + """ + assigneeUsernames: [String!] + + """ + Username of the author of the issue + """ + authorUsername: String + + """ Returns the elements in the list that come before the specified cursor. """ before: String @@ -6821,7 +7858,7 @@ type Group { iids: [String!] """ - Include issues belonging to subgroups. + Include issues belonging to subgroups """ includeSubgroups: Boolean = false @@ -6891,8 +7928,8 @@ type Group { before: String """ - List items within a time frame where items.end_date is between startDate and - endDate parameters (startDate parameter must be present) + List items overlapping a time frame defined by startDate..endDate (if one + date is provided, both must be present). Deprecated in 13.5: Use timeframe.end """ endDate: Time @@ -6922,8 +7959,9 @@ type Group { last: Int """ - List items within a time frame where items.start_date is between startDate - and endDate parameters (endDate parameter must be present) + List items overlapping a time frame defined by startDate..endDate (if one + date is provided, both must be present). Deprecated in 13.5: Use + timeframe.start """ startDate: Time @@ -6933,6 +7971,11 @@ type Group { state: IterationState """ + List items overlapping the given timeframe + """ + timeframe: Timeframe + + """ Fuzzy search by title """ title: String @@ -6989,6 +8032,91 @@ type Group { mentionsDisabled: Boolean """ + Merge requests for projects in this group + """ + mergeRequests( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Username of the assignee + """ + assigneeUsername: String + + """ + Username of the author + """ + authorUsername: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Array of IIDs of merge requests, for example `[1, 2]` + """ + iids: [String!] + + """ + Include merge requests belonging to subgroups + """ + includeSubgroups: Boolean = false + + """ + Array of label names. All resolved merge requests will have all of these labels. + """ + labels: [String!] + + """ + Returns the last _n_ elements from the list. + """ + last: Int + + """ + Merge requests merged after this date + """ + mergedAfter: Time + + """ + Merge requests merged before this date + """ + mergedBefore: Time + + """ + Title of the milestone + """ + milestoneTitle: String + + """ + Sort merge requests by this criteria + """ + sort: MergeRequestSort = created_desc + + """ + Array of source branch names. All resolved merge requests will have one of these branches as their source. + """ + sourceBranches: [String!] + + """ + A merge request state. If provided, all resolved merge requests will have this state. + """ + state: MergeRequestState + + """ + Array of target branch names. All resolved merge requests will have one of these branches as their target. + """ + targetBranches: [String!] + ): MergeRequestConnection + + """ Milestones of the group """ milestones( @@ -7003,8 +8131,13 @@ type Group { before: String """ - List items within a time frame where items.end_date is between startDate and - endDate parameters (startDate parameter must be present) + A date that the milestone contains + """ + containingDate: Time + + """ + List items overlapping a time frame defined by startDate..endDate (if one + date is provided, both must be present). Deprecated in 13.5: Use timeframe.end """ endDate: Time @@ -7029,8 +8162,14 @@ type Group { last: Int """ - List items within a time frame where items.start_date is between startDate - and endDate parameters (endDate parameter must be present) + A search string for the title + """ + searchTitle: String + + """ + List items overlapping a time frame defined by startDate..endDate (if one + date is provided, both must be present). Deprecated in 13.5: Use + timeframe.start """ startDate: Time @@ -7038,6 +8177,16 @@ type Group { Filter milestones by state """ state: MilestoneStateEnum + + """ + List items overlapping the given timeframe + """ + timeframe: Timeframe + + """ + The title of the milestone + """ + title: String ): MilestoneConnection """ @@ -7106,6 +8255,11 @@ type Group { ): ProjectConnection! """ + Number of projects in the root namespace where the repository size exceeds the limit + """ + repositorySizeExcessProjectCount: Int! + + """ Indicates if users can request access to namespace """ requestAccessEnabled: Boolean @@ -7186,6 +8340,16 @@ type Group { ): TimelogConnection! """ + Total repository size of all projects in the root namespace in bytes + """ + totalRepositorySize: Float + + """ + Total excess repository size of all projects in the root namespace in bytes + """ + totalRepositorySizeExcess: Float + + """ Time before two-factor authentication is enforced """ twoFactorGracePeriod: Int @@ -7962,6 +9126,11 @@ type Issue implements CurrentUserTodos & Noteable { severity: IssuableSeverity """ + Timestamp of when the issue SLA expires. + """ + slaDueAt: Time + + """ State of the issue """ state: IssueState! @@ -8088,6 +9257,31 @@ Identifier of Issue scalar IssueID """ +Autogenerated input type of IssueMove +""" +input IssueMoveInput { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + The IID of the issue to mutate + """ + iid: String! + + """ + The project the issue to mutate is in + """ + projectPath: ID! + + """ + The project to move the issue to + """ + targetProjectPath: ID! +} + +""" Autogenerated input type of IssueMoveList """ input IssueMoveListInput { @@ -8158,6 +9352,26 @@ type IssueMoveListPayload { } """ +Autogenerated return type of IssueMove +""" +type IssueMovePayload { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Errors encountered during execution of the mutation. + """ + errors: [String!]! + + """ + The issue after mutation + """ + issue: Issue +} + +""" Check permissions for the current user on a issue """ type IssuePermissions { @@ -8404,7 +9618,7 @@ input IssueSetIterationInput { """ The iteration to assign to the issue. """ - iterationId: ID + iterationId: IterationID """ The project the issue to mutate is in @@ -8617,6 +9831,16 @@ Values for sorting issues """ enum IssueSort { """ + Created at ascending order + """ + CREATED_ASC + + """ + Created at descending order + """ + CREATED_DESC + + """ Due date by ascending order """ DUE_DATE_ASC @@ -8657,11 +9881,41 @@ enum IssueSort { PRIORITY_DESC """ + Published issues shown last + """ + PUBLISHED_ASC + + """ + Published issues shown first + """ + PUBLISHED_DESC + + """ Relative position by ascending order """ RELATIVE_POSITION_ASC """ + Severity from less critical to more critical + """ + SEVERITY_ASC + + """ + Severity from more critical to less critical + """ + SEVERITY_DESC + + """ + Updated at ascending order + """ + UPDATED_ASC + + """ + Updated at descending order + """ + UPDATED_DESC + + """ Weight by ascending order """ WEIGHT_ASC @@ -8674,22 +9928,22 @@ enum IssueSort { """ Created at ascending order """ - created_asc + created_asc @deprecated(reason: "Use CREATED_ASC. Deprecated in 13.5") """ Created at descending order """ - created_desc + created_desc @deprecated(reason: "Use CREATED_DESC. Deprecated in 13.5") """ Updated at ascending order """ - updated_asc + updated_asc @deprecated(reason: "Use UPDATED_ASC. Deprecated in 13.5") """ Updated at descending order """ - updated_desc + updated_desc @deprecated(reason: "Use UPDATED_DESC. Deprecated in 13.5") } """ @@ -8703,6 +9957,21 @@ enum IssueState { } """ +Values for issue state events +""" +enum IssueStateEvent { + """ + Closes the issue + """ + CLOSE + + """ + Reopens the issue + """ + REOPEN +} + +""" Represents total number of issues for the represented statuses """ type IssueStatusCountsType { @@ -9215,6 +10484,11 @@ The connection type for Label. """ type LabelConnection { """ + Total count of collection + """ + count: Int! + + """ A list of edges. """ edges: [LabelEdge] @@ -9251,6 +10525,11 @@ Identifier of Label scalar LabelID """ +Identifier of List +""" +scalar ListID + +""" List limit metric setting """ enum ListLimitMetric { @@ -9319,6 +10598,26 @@ enum MeasurementIdentifier { PIPELINES """ + Pipeline count with canceled status + """ + PIPELINES_CANCELED + + """ + Pipeline count with failed status + """ + PIPELINES_FAILED + + """ + Pipeline count with skipped status + """ + PIPELINES_SKIPPED + + """ + Pipeline count with success status + """ + PIPELINES_SUCCEEDED + + """ Project count """ PROJECTS @@ -10019,6 +11318,86 @@ type MergeRequestCreatePayload { } """ +Represents the Geo sync and verification state of a Merge Request diff +""" +type MergeRequestDiffRegistry { + """ + Timestamp when the MergeRequestDiffRegistry was created + """ + createdAt: Time + + """ + ID of the MergeRequestDiffRegistry + """ + id: ID! + + """ + Error message during sync of the MergeRequestDiffRegistry + """ + lastSyncFailure: String + + """ + Timestamp of the most recent successful sync of the MergeRequestDiffRegistry + """ + lastSyncedAt: Time + + """ + ID of the Merge Request diff + """ + mergeRequestDiffId: ID! + + """ + Timestamp after which the MergeRequestDiffRegistry should be resynced + """ + retryAt: Time + + """ + Number of consecutive failed sync attempts of the MergeRequestDiffRegistry + """ + retryCount: Int + + """ + Sync state of the MergeRequestDiffRegistry + """ + state: RegistryState +} + +""" +The connection type for MergeRequestDiffRegistry. +""" +type MergeRequestDiffRegistryConnection { + """ + A list of edges. + """ + edges: [MergeRequestDiffRegistryEdge] + + """ + A list of nodes. + """ + nodes: [MergeRequestDiffRegistry] + + """ + Information to aid in pagination. + """ + pageInfo: PageInfo! +} + +""" +An edge in a connection. +""" +type MergeRequestDiffRegistryEdge { + """ + A cursor for use in pagination. + """ + cursor: String! + + """ + The item at the end of the edge. + """ + node: MergeRequestDiffRegistry +} + +""" An edge in a connection. """ type MergeRequestEdge { @@ -10034,6 +11413,11 @@ type MergeRequestEdge { } """ +Identifier of MergeRequest +""" +scalar MergeRequestID + +""" Check permissions for the current user on a merge request """ type MergeRequestPermissions { @@ -10245,7 +11629,7 @@ input MergeRequestSetMilestoneInput { """ The milestone to assign to the merge request. """ - milestoneId: ID + milestoneId: MilestoneID """ The project the merge request to mutate is in @@ -10368,6 +11752,16 @@ Values for sorting merge requests """ enum MergeRequestSort { """ + Created at ascending order + """ + CREATED_ASC + + """ + Created at descending order + """ + CREATED_DESC + + """ Label priority by ascending order """ LABEL_PRIORITY_ASC @@ -10408,24 +11802,34 @@ enum MergeRequestSort { PRIORITY_DESC """ + Updated at ascending order + """ + UPDATED_ASC + + """ + Updated at descending order + """ + UPDATED_DESC + + """ Created at ascending order """ - created_asc + created_asc @deprecated(reason: "Use CREATED_ASC. Deprecated in 13.5") """ Created at descending order """ - created_desc + created_desc @deprecated(reason: "Use CREATED_DESC. Deprecated in 13.5") """ Updated at ascending order """ - updated_asc + updated_asc @deprecated(reason: "Use UPDATED_ASC. Deprecated in 13.5") """ Updated at descending order """ - updated_desc + updated_desc @deprecated(reason: "Use UPDATED_DESC. Deprecated in 13.5") } """ @@ -10783,11 +12187,13 @@ type Mutation { configureSast(input: ConfigureSastInput!): ConfigureSastPayload createAlertIssue(input: CreateAlertIssueInput!): CreateAlertIssuePayload createAnnotation(input: CreateAnnotationInput!): CreateAnnotationPayload + createBoard(input: CreateBoardInput!): CreateBoardPayload createBranch(input: CreateBranchInput!): CreateBranchPayload createClusterAgent(input: CreateClusterAgentInput!): CreateClusterAgentPayload createDiffNote(input: CreateDiffNoteInput!): CreateDiffNotePayload createEpic(input: CreateEpicInput!): CreateEpicPayload createImageDiffNote(input: CreateImageDiffNoteInput!): CreateImageDiffNotePayload + createIssue(input: CreateIssueInput!): CreateIssuePayload createIteration(input: CreateIterationInput!): CreateIterationPayload createNote(input: CreateNoteInput!): CreateNotePayload createRequirement(input: CreateRequirementInput!): CreateRequirementPayload @@ -10800,11 +12206,13 @@ type Mutation { dastSiteProfileCreate(input: DastSiteProfileCreateInput!): DastSiteProfileCreatePayload dastSiteProfileDelete(input: DastSiteProfileDeleteInput!): DastSiteProfileDeletePayload dastSiteProfileUpdate(input: DastSiteProfileUpdateInput!): DastSiteProfileUpdatePayload + dastSiteTokenCreate(input: DastSiteTokenCreateInput!): DastSiteTokenCreatePayload deleteAnnotation(input: DeleteAnnotationInput!): DeleteAnnotationPayload designManagementDelete(input: DesignManagementDeleteInput!): DesignManagementDeletePayload designManagementMove(input: DesignManagementMoveInput!): DesignManagementMovePayload designManagementUpload(input: DesignManagementUploadInput!): DesignManagementUploadPayload destroyBoard(input: DestroyBoardInput!): DestroyBoardPayload + destroyBoardList(input: DestroyBoardListInput!): DestroyBoardListPayload destroyNote(input: DestroyNoteInput!): DestroyNotePayload destroySnippet(input: DestroySnippetInput!): DestroySnippetPayload @@ -10812,10 +12220,11 @@ type Mutation { Toggles the resolved state of a discussion """ discussionToggleResolve(input: DiscussionToggleResolveInput!): DiscussionToggleResolvePayload - dismissVulnerability(input: DismissVulnerabilityInput!): DismissVulnerabilityPayload + dismissVulnerability(input: DismissVulnerabilityInput!): DismissVulnerabilityPayload @deprecated(reason: "Use vulnerabilityDismiss. Deprecated in 13.5") epicAddIssue(input: EpicAddIssueInput!): EpicAddIssuePayload epicSetSubscription(input: EpicSetSubscriptionInput!): EpicSetSubscriptionPayload epicTreeReorder(input: EpicTreeReorderInput!): EpicTreeReorderPayload + issueMove(input: IssueMoveInput!): IssueMovePayload issueMoveList(input: IssueMoveListInput!): IssueMoveListPayload issueSetAssignees(input: IssueSetAssigneesInput!): IssueSetAssigneesPayload issueSetConfidential(input: IssueSetConfidentialInput!): IssueSetConfidentialPayload @@ -10847,6 +12256,7 @@ type Mutation { pipelineRetry(input: PipelineRetryInput!): PipelineRetryPayload removeAwardEmoji(input: RemoveAwardEmojiInput!): RemoveAwardEmojiPayload @deprecated(reason: "Use awardEmojiRemove. Deprecated in 13.2") removeProjectFromSecurityDashboard(input: RemoveProjectFromSecurityDashboardInput!): RemoveProjectFromSecurityDashboardPayload + revertVulnerabilityToDetected(input: RevertVulnerabilityToDetectedInput!): RevertVulnerabilityToDetectedPayload @deprecated(reason: "Use vulnerabilityRevertToDetected. Deprecated in 13.5") runDastScan(input: RunDASTScanInput!): RunDASTScanPayload @deprecated(reason: "Use DastOnDemandScanCreate. Deprecated in 13.4") todoMarkDone(input: TodoMarkDoneInput!): TodoMarkDonePayload todoRestore(input: TodoRestoreInput!): TodoRestorePayload @@ -10855,6 +12265,7 @@ type Mutation { toggleAwardEmoji(input: ToggleAwardEmojiInput!): ToggleAwardEmojiPayload @deprecated(reason: "Use awardEmojiToggle. Deprecated in 13.2") updateAlertStatus(input: UpdateAlertStatusInput!): UpdateAlertStatusPayload updateBoard(input: UpdateBoardInput!): UpdateBoardPayload + updateBoardEpicUserPreferences(input: UpdateBoardEpicUserPreferencesInput!): UpdateBoardEpicUserPreferencesPayload updateBoardList(input: UpdateBoardListInput!): UpdateBoardListPayload updateContainerExpirationPolicy(input: UpdateContainerExpirationPolicyInput!): UpdateContainerExpirationPolicyPayload updateEpic(input: UpdateEpicInput!): UpdateEpicPayload @@ -10875,7 +12286,10 @@ type Mutation { updateNote(input: UpdateNoteInput!): UpdateNotePayload updateRequirement(input: UpdateRequirementInput!): UpdateRequirementPayload updateSnippet(input: UpdateSnippetInput!): UpdateSnippetPayload + vulnerabilityConfirm(input: VulnerabilityConfirmInput!): VulnerabilityConfirmPayload + vulnerabilityDismiss(input: VulnerabilityDismissInput!): VulnerabilityDismissPayload vulnerabilityResolve(input: VulnerabilityResolveInput!): VulnerabilityResolvePayload + vulnerabilityRevertToDetected(input: VulnerabilityRevertToDetectedInput!): VulnerabilityRevertToDetectedPayload } """ @@ -10900,6 +12314,21 @@ enum MutationOperationMode { type Namespace { """ + Size limit for repositories in the namespace in bytes + """ + actualRepositorySizeLimit: Float + + """ + Additional storage purchased for the root namespace in bytes + """ + additionalPurchasedStorageSize: Float + + """ + Includes at least one project where the repository size exceeds the limit + """ + containsLockedProjects: Boolean! + + """ Description of the namespace """ description: String @@ -10990,6 +12419,11 @@ type Namespace { ): ProjectConnection! """ + Number of projects in the root namespace where the repository size exceeds the limit + """ + repositorySizeExcessProjectCount: Int! + + """ Indicates if users can request access to namespace """ requestAccessEnabled: Boolean @@ -11010,6 +12444,16 @@ type Namespace { temporaryStorageIncreaseEndsOn: Time """ + Total repository size of all projects in the root namespace in bytes + """ + totalRepositorySize: Float + + """ + Total excess repository size of all projects in the root namespace in bytes + """ + totalRepositorySizeExcess: Float + + """ Visibility of the namespace """ visibility: String @@ -11051,6 +12495,11 @@ type NamespaceEdge { } """ +Identifier of Namespace +""" +scalar NamespaceID + +""" Autogenerated input type of NamespaceIncreaseStorageTemporarily """ input NamespaceIncreaseStorageTemporarilyInput { @@ -11062,7 +12511,7 @@ input NamespaceIncreaseStorageTemporarilyInput { """ The global id of the namespace to mutate """ - id: ID! + id: NamespaceID! } """ @@ -11259,6 +12708,11 @@ type NoteEdge { node: Note } +""" +Identifier of Note +""" +scalar NoteID + type NotePermissions { """ Indicates the user can perform `admin_note` on this resource @@ -11339,6 +12793,11 @@ interface Noteable { } """ +Identifier of Noteable +""" +scalar NoteableID + +""" Represents a package """ type Package { @@ -11409,7 +12868,7 @@ type PackageEdge { } """ -Represents the sync and verification state of a package file +Represents the Geo sync and verification state of a package file """ type PackageFileRegistry { """ @@ -11490,37 +12949,47 @@ type PackageFileRegistryEdge { enum PackageTypeEnum { """ - Packages from the composer package manager + Packages from the Composer package manager """ COMPOSER """ - Packages from the conan package manager + Packages from the Conan package manager """ CONAN """ - Packages from the generic package manager + Packages from the Debian package manager + """ + DEBIAN + + """ + Packages from the Generic package manager """ GENERIC """ - Packages from the maven package manager + Packages from the Golang package manager + """ + GOLANG + + """ + Packages from the Maven package manager """ MAVEN """ - Packages from the npm package manager + Packages from the NPM package manager """ NPM """ - Packages from the nuget package manager + Packages from the Nuget package manager """ NUGET """ - Packages from the pypi package manager + Packages from the PyPI package manager """ PYPI } @@ -11854,10 +13323,20 @@ enum PipelineStatusEnum { type Project { """ + Size limit for the repository in bytes + """ + actualRepositorySizeLimit: Float + + """ A single Alert Management alert of the project """ alertManagementAlert( """ + Username of a user assigned to the issue + """ + assigneeUsername: String + + """ IID of the alert. For example, "1" """ iid: String @@ -11883,6 +13362,11 @@ type Project { """ alertManagementAlertStatusCounts( """ + Username of a user assigned to the issue + """ + assigneeUsername: String + + """ Search criteria for filtering alerts. This will search on title, description, service, monitoring_tool. """ search: String @@ -11898,6 +13382,11 @@ type Project { after: String """ + Username of a user assigned to the issue + """ + assigneeUsername: String + + """ Returns the elements in the list that come before the specified cursor. """ before: String @@ -11959,9 +13448,9 @@ type Project { """ board( """ - Find a board by its ID + The board's ID """ - id: ID + id: BoardID! ): Board """ @@ -12249,6 +13738,16 @@ type Project { assigneeUsername: String """ + Usernames of users assigned to the issue + """ + assigneeUsernames: [String!] + + """ + Username of the author of the issue + """ + authorUsername: String + + """ Issues closed after this date """ closedAfter: Time @@ -12339,6 +13838,16 @@ type Project { assigneeUsername: String """ + Usernames of users assigned to the issue + """ + assigneeUsernames: [String!] + + """ + Username of the author of the issue + """ + authorUsername: String + + """ Issues closed after this date """ closedAfter: Time @@ -12419,6 +13928,16 @@ type Project { assigneeUsername: String """ + Usernames of users assigned to the issue + """ + assigneeUsernames: [String!] + + """ + Username of the author of the issue + """ + authorUsername: String + + """ Returns the elements in the list that come before the specified cursor. """ before: String @@ -12529,8 +14048,8 @@ type Project { before: String """ - List items within a time frame where items.end_date is between startDate and - endDate parameters (startDate parameter must be present) + List items overlapping a time frame defined by startDate..endDate (if one + date is provided, both must be present). Deprecated in 13.5: Use timeframe.end """ endDate: Time @@ -12560,8 +14079,9 @@ type Project { last: Int """ - List items within a time frame where items.start_date is between startDate - and endDate parameters (endDate parameter must be present) + List items overlapping a time frame defined by startDate..endDate (if one + date is provided, both must be present). Deprecated in 13.5: Use + timeframe.start """ startDate: Time @@ -12571,6 +14091,11 @@ type Project { state: IterationState """ + List items overlapping the given timeframe + """ + timeframe: Timeframe + + """ Fuzzy search by title """ title: String @@ -12778,8 +14303,13 @@ type Project { before: String """ - List items within a time frame where items.end_date is between startDate and - endDate parameters (startDate parameter must be present) + A date that the milestone contains + """ + containingDate: Time + + """ + List items overlapping a time frame defined by startDate..endDate (if one + date is provided, both must be present). Deprecated in 13.5: Use timeframe.end """ endDate: Time @@ -12804,8 +14334,14 @@ type Project { last: Int """ - List items within a time frame where items.start_date is between startDate - and endDate parameters (endDate parameter must be present) + A search string for the title + """ + searchTitle: String + + """ + List items overlapping a time frame defined by startDate..endDate (if one + date is provided, both must be present). Deprecated in 13.5: Use + timeframe.start """ startDate: Time @@ -12813,6 +14349,16 @@ type Project { Filter milestones by state """ state: MilestoneStateEnum + + """ + List items overlapping the given timeframe + """ + timeframe: Timeframe + + """ + The title of the milestone + """ + title: String ): MilestoneConnection """ @@ -13012,12 +14558,17 @@ type Project { repository: Repository """ + Size of repository that exceeds the limit in bytes + """ + repositorySizeExcess: Float + + """ Indicates if users can request member access to the project """ requestAccessEnabled: Boolean """ - Find a single requirement. Available only when feature flag `requirements_management` is enabled. + Find a single requirement """ requirement( """ @@ -13057,7 +14608,7 @@ type Project { requirementStatesCount: RequirementStatesCount """ - Find requirements. Available only when feature flag `requirements_management` is enabled. + Find requirements """ requirements( """ @@ -13257,6 +14808,31 @@ type Project { tagList: String """ + Terraform states associated with the project + """ + terraformStates( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + ): TerraformStateConnection + + """ Permissions for the current user on the resource """ userPermissions: ProjectPermissions! @@ -13468,6 +15044,11 @@ type ProjectEdge { } """ +Identifier of Project +""" +scalar ProjectID + +""" Represents a Project Membership """ type ProjectMember implements MemberInterface { @@ -14001,9 +15582,44 @@ type Query { Search query for project name, path, or description """ search: String + + """ + Include namespace in project search + """ + searchNamespaces: Boolean + + """ + Sort order of results + """ + sort: String ): ProjectConnection """ + Supported runner platforms + """ + runnerPlatforms( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + ): RunnerPlatformConnection + + """ Find Snippets visible to the current user """ snippets( @@ -14249,6 +15865,16 @@ type Query { """ startDate: ISO8601Date! ): VulnerabilitiesCountByDayAndSeverityConnection @deprecated(reason: "Use `vulnerabilitiesCountByDay`. Deprecated in 13.3") + + """ + Find a vulnerability + """ + vulnerability( + """ + The Global ID of the Vulnerability + """ + id: VulnerabilityID! + ): Vulnerability } """ @@ -14725,7 +16351,7 @@ input RemoveAwardEmojiInput { """ The global id of the awardable resource """ - awardableId: ID! + awardableId: AwardableID! """ A unique identifier for the client performing the mutation. @@ -14770,7 +16396,7 @@ input RemoveProjectFromSecurityDashboardInput { """ ID of the project to remove from the Instance Security Dashboard """ - id: ID! + id: ProjectID! } """ @@ -14840,6 +16466,16 @@ type Requirement { createdAt: Time! """ + Description of the requirement + """ + description: String + + """ + The GitLab Flavored Markdown rendering of `description` + """ + descriptionHtml: String + + """ ID of the requirement """ id: ID! @@ -14850,6 +16486,11 @@ type Requirement { iid: ID! """ + Indicates if latest test report was created by user + """ + lastTestReportManuallyCreated: Boolean + + """ Latest requirement test report state """ lastTestReportState: TestReportState @@ -14900,6 +16541,11 @@ type Requirement { title: String """ + The GitLab Flavored Markdown rendering of `title` + """ + titleHtml: String + + """ Timestamp of when the requirement was last updated """ updatedAt: Time! @@ -15020,6 +16666,41 @@ interface ResolvableInterface { resolvedBy: User } +""" +Autogenerated input type of RevertVulnerabilityToDetected +""" +input RevertVulnerabilityToDetectedInput { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + ID of the vulnerability to be reverted + """ + id: VulnerabilityID! +} + +""" +Autogenerated return type of RevertVulnerabilityToDetected +""" +type RevertVulnerabilityToDetectedPayload { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Errors encountered during execution of the mutation. + """ + errors: [String!]! + + """ + The vulnerability after revert + """ + vulnerability: Vulnerability +} + type RootStorageStatistics { """ The CI artifacts size in bytes @@ -15037,6 +16718,11 @@ type RootStorageStatistics { packagesSize: Float! """ + The CI pipeline artifacts size in bytes + """ + pipelineArtifactsSize: Float! + + """ The Git repository size in bytes """ repositorySize: Float! @@ -15107,6 +16793,125 @@ type RunDASTScanPayload { pipelineUrl: String } +type RunnerArchitecture { + """ + Download location for the runner for the platform architecture + """ + downloadLocation: String! + + """ + Name of the runner platform architecture + """ + name: String! +} + +""" +The connection type for RunnerArchitecture. +""" +type RunnerArchitectureConnection { + """ + A list of edges. + """ + edges: [RunnerArchitectureEdge] + + """ + A list of nodes. + """ + nodes: [RunnerArchitecture] + + """ + Information to aid in pagination. + """ + pageInfo: PageInfo! +} + +""" +An edge in a connection. +""" +type RunnerArchitectureEdge { + """ + A cursor for use in pagination. + """ + cursor: String! + + """ + The item at the end of the edge. + """ + node: RunnerArchitecture +} + +type RunnerPlatform { + """ + Runner architectures supported for the platform + """ + architectures( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + ): RunnerArchitectureConnection + + """ + Human readable name of the runner platform + """ + humanReadableName: String! + + """ + Name slug of the runner platform + """ + name: String! +} + +""" +The connection type for RunnerPlatform. +""" +type RunnerPlatformConnection { + """ + A list of edges. + """ + edges: [RunnerPlatformEdge] + + """ + A list of nodes. + """ + nodes: [RunnerPlatform] + + """ + Information to aid in pagination. + """ + pageInfo: PageInfo! +} + +""" +An edge in a connection. +""" +type RunnerPlatformEdge { + """ + A cursor for use in pagination. + """ + cursor: String! + + """ + The item at the end of the edge. + """ + node: RunnerPlatform +} + """ Represents a CI configuration of SAST """ @@ -15273,6 +17078,26 @@ type SastCiConfigurationAnalyzersEntityEdge { } """ +Represents the analyzers entity in SAST CI configuration +""" +input SastCiConfigurationAnalyzersEntityInput { + """ + State of the analyzer + """ + enabled: Boolean! + + """ + Name of analyzer + """ + name: String! + + """ + List of variables for the analyzer + """ + variables: [SastCiConfigurationEntityInput!] +} + +""" Represents an entity in SAST CI configuration """ type SastCiConfigurationEntity { @@ -15397,6 +17222,11 @@ Represents a CI configuration of SAST """ input SastCiConfigurationInput { """ + List of analyzers and related variables for the SAST configuration + """ + analyzers: [SastCiConfigurationAnalyzersEntityInput!] + + """ List of global entities related to SAST configuration """ global: [SastCiConfigurationEntityInput!] @@ -15521,6 +17351,11 @@ Represents summary of a security report """ type SecurityReportSummary { """ + Aggregated counts for the api_fuzzing scan + """ + apiFuzzing: SecurityReportSummarySection + + """ Aggregated counts for the container_scanning scan """ containerScanning: SecurityReportSummarySection @@ -15600,6 +17435,7 @@ type SecurityReportSummarySection { The type of the security scanner """ enum SecurityScannerType { + API_FUZZING CONTAINER_SCANNING COVERAGE_FUZZING DAST @@ -16183,7 +18019,32 @@ type Snippet implements Noteable { """ Snippet blobs """ - blobs: [SnippetBlob!]! + blobs( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + + """ + Paths of the blobs + """ + paths: [String!] + ): SnippetBlobConnection """ Timestamp this snippet was created @@ -16407,6 +18268,41 @@ input SnippetBlobActionInputType { } """ +The connection type for SnippetBlob. +""" +type SnippetBlobConnection { + """ + A list of edges. + """ + edges: [SnippetBlobEdge] + + """ + A list of nodes. + """ + nodes: [SnippetBlob] + + """ + Information to aid in pagination. + """ + pageInfo: PageInfo! +} + +""" +An edge in a connection. +""" +type SnippetBlobEdge { + """ + A cursor for use in pagination. + """ + cursor: String! + + """ + The item at the end of the edge. + """ + node: SnippetBlob +} + +""" Represents how the blob content should be displayed """ type SnippetBlobViewer { @@ -16520,22 +18416,69 @@ enum Sort { """ Created at ascending order """ - created_asc + CREATED_ASC + + """ + Created at descending order + """ + CREATED_DESC + + """ + Updated at ascending order + """ + UPDATED_ASC + + """ + Updated at descending order + """ + UPDATED_DESC + + """ + Created at ascending order + """ + created_asc @deprecated(reason: "Use CREATED_ASC. Deprecated in 13.5") """ Created at descending order """ - created_desc + created_desc @deprecated(reason: "Use CREATED_DESC. Deprecated in 13.5") """ Updated at ascending order """ - updated_asc + updated_asc @deprecated(reason: "Use UPDATED_ASC. Deprecated in 13.5") """ Updated at descending order """ - updated_desc + updated_desc @deprecated(reason: "Use UPDATED_DESC. Deprecated in 13.5") +} + +type StatusAction { + """ + Title for the button, for example: Retry this job + """ + buttonTitle: String + + """ + Icon used in the action button + """ + icon: String + + """ + Method for the action, for example: :post + """ + method: String + + """ + Path for the action + """ + path: String + + """ + Title for the action, for example: Retry + """ + title: String } type Submodule implements Entry { @@ -16630,64 +18573,131 @@ type TaskCompletionStatus { count: Int! } +type TerraformState { + """ + Timestamp the Terraform state was created + """ + createdAt: Time! + + """ + ID of the Terraform state + """ + id: ID! + + """ + Timestamp the Terraform state was locked + """ + lockedAt: Time + + """ + The user currently holding a lock on the Terraform state + """ + lockedByUser: User + + """ + Name of the Terraform state + """ + name: String! + + """ + Timestamp the Terraform state was updated + """ + updatedAt: Time! +} + +""" +The connection type for TerraformState. +""" +type TerraformStateConnection { + """ + A list of edges. + """ + edges: [TerraformStateEdge] + + """ + A list of nodes. + """ + nodes: [TerraformState] + + """ + Information to aid in pagination. + """ + pageInfo: PageInfo! +} + """ -Represents the sync and verification state of a terraform state +An edge in a connection. """ -type TerraformStateRegistry { +type TerraformStateEdge { """ - Timestamp when the TerraformStateRegistry was created + A cursor for use in pagination. + """ + cursor: String! + + """ + The item at the end of the edge. + """ + node: TerraformState +} + +""" +Represents the Geo sync and verification state of a terraform state version +""" +type TerraformStateVersionRegistry { + """ + Timestamp when the TerraformStateVersionRegistry was created """ createdAt: Time """ - ID of the TerraformStateRegistry + ID of the TerraformStateVersionRegistry """ id: ID! """ - Error message during sync of the TerraformStateRegistry + Error message during sync of the TerraformStateVersionRegistry """ lastSyncFailure: String """ - Timestamp of the most recent successful sync of the TerraformStateRegistry + Timestamp of the most recent successful sync of the TerraformStateVersionRegistry """ lastSyncedAt: Time """ - Timestamp after which the TerraformStateRegistry should be resynced + Timestamp after which the TerraformStateVersionRegistry should be resynced """ retryAt: Time """ - Number of consecutive failed sync attempts of the TerraformStateRegistry + Number of consecutive failed sync attempts of the TerraformStateVersionRegistry """ retryCount: Int """ - Sync state of the TerraformStateRegistry + Sync state of the TerraformStateVersionRegistry """ state: RegistryState """ - ID of the TerraformState + ID of the terraform state version """ - terraformStateId: ID! + terraformStateVersionId: ID! } """ -The connection type for TerraformStateRegistry. +The connection type for TerraformStateVersionRegistry. """ -type TerraformStateRegistryConnection { +type TerraformStateVersionRegistryConnection { """ A list of edges. """ - edges: [TerraformStateRegistryEdge] + edges: [TerraformStateVersionRegistryEdge] """ A list of nodes. """ - nodes: [TerraformStateRegistry] + nodes: [TerraformStateVersionRegistry] """ Information to aid in pagination. @@ -16698,7 +18708,7 @@ type TerraformStateRegistryConnection { """ An edge in a connection. """ -type TerraformStateRegistryEdge { +type TerraformStateVersionRegistryEdge { """ A cursor for use in pagination. """ @@ -16707,7 +18717,7 @@ type TerraformStateRegistryEdge { """ The item at the end of the edge. """ - node: TerraformStateRegistry + node: TerraformStateVersionRegistry } """ @@ -16790,6 +18800,21 @@ interface TimeboxBurnupTimeSeriesInterface { burnupTimeSeries: [BurnupChartDailyTotals!] } +""" +A time-frame defined as a closed inclusive range of two dates +""" +input Timeframe { + """ + The end of the range + """ + end: Date! + + """ + The start of the range + """ + start: Date! +} + type Timelog { """ Timestamp of when the time tracked was spent at. Deprecated in 12.10: Use `spentAt` @@ -16953,6 +18978,11 @@ type TodoEdge { } """ +Identifier of Todo +""" +scalar TodoID + +""" Autogenerated input type of TodoMarkDone """ input TodoMarkDoneInput { @@ -16964,7 +18994,7 @@ input TodoMarkDoneInput { """ The global id of the todo to mark as done """ - id: ID! + id: TodoID! } """ @@ -16999,7 +19029,7 @@ input TodoRestoreInput { """ The global id of the todo to restore """ - id: ID! + id: TodoID! } """ @@ -17014,7 +19044,7 @@ input TodoRestoreManyInput { """ The global ids of the todos to restore (a maximum of 50 is supported at once) """ - ids: [ID!]! + ids: [TodoID!]! } """ @@ -17141,7 +19171,7 @@ input ToggleAwardEmojiInput { """ The global id of the awardable resource """ - awardableId: ID! + awardableId: AwardableID! """ A unique identifier for the client performing the mutation. @@ -17407,13 +19437,58 @@ type UpdateAlertStatusPayload { } """ +Autogenerated input type of UpdateBoardEpicUserPreferences +""" +input UpdateBoardEpicUserPreferencesInput { + """ + The board global ID + """ + boardId: BoardID! + + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Whether the epic should be collapsed in the board + """ + collapsed: Boolean! + + """ + ID of an epic to set preferences for + """ + epicId: EpicID! +} + +""" +Autogenerated return type of UpdateBoardEpicUserPreferences +""" +type UpdateBoardEpicUserPreferencesPayload { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + User preferences for the epic in the board after mutation + """ + epicUserPreferences: BoardEpicUserPreferences + + """ + Errors encountered during execution of the mutation. + """ + errors: [String!]! +} + +""" Autogenerated input type of UpdateBoard """ input UpdateBoardInput { """ The id of user to be assigned to the board. """ - assigneeId: ID + assigneeId: UserID """ A unique identifier for the client performing the mutation. @@ -17433,12 +19508,22 @@ input UpdateBoardInput { """ The board global id. """ - id: ID! + id: BoardID! + + """ + The IDs of labels to be added to the board. + """ + labelIds: [LabelID!] + + """ + Labels of the issue + """ + labels: [String!] """ The id of milestone to be assigned to the board. """ - milestoneId: ID + milestoneId: MilestoneID """ Name of the board @@ -17710,7 +19795,7 @@ input UpdateImageDiffNoteInput { """ The global id of the note to update """ - id: ID! + id: NoteID! """ The position of this note on a diff @@ -17743,7 +19828,7 @@ Autogenerated input type of UpdateIssue """ input UpdateIssueInput { """ - The IDs of labels to be added to the issue. + The IDs of labels to be added to the issue """ addLabelIds: [ID!] @@ -17765,7 +19850,7 @@ input UpdateIssueInput { """ Due date of the issue """ - dueDate: Time + dueDate: ISO8601Date """ The ID of the parent epic. NULL when removing the association @@ -17788,7 +19873,7 @@ input UpdateIssueInput { locked: Boolean """ - The ID of the milestone to be assigned, milestone will be removed if set to null. + The ID of the milestone to assign to the issue. On update milestone will be removed if set to null """ milestoneId: ID @@ -17798,14 +19883,24 @@ input UpdateIssueInput { projectPath: ID! """ - The IDs of labels to be removed from the issue. + The IDs of labels to be removed from the issue """ removeLabelIds: [ID!] """ + Close or reopen an issue + """ + stateEvent: IssueStateEvent + + """ Title of the issue """ title: String + + """ + The weight of the issue + """ + weight: Int } """ @@ -17910,7 +20005,7 @@ input UpdateNoteInput { """ The global id of the note to update """ - id: ID! + id: NoteID! } """ @@ -17943,6 +20038,11 @@ input UpdateRequirementInput { clientMutationId: String """ + Description of the requirement + """ + description: String + + """ The iid of the requirement to update """ iid: String! @@ -17953,7 +20053,7 @@ input UpdateRequirementInput { lastTestReportState: TestReportState """ - The project full path the requirement is associated with + Full project path the requirement is associated with """ projectPath: ID! @@ -17983,7 +20083,7 @@ type UpdateRequirementPayload { errors: [String!]! """ - The requirement after mutation + Requirement after mutation """ requirement: Requirement } @@ -18041,6 +20141,11 @@ type UpdateSnippetPayload { The snippet after mutation """ snippet: Snippet + + """ + Indicates whether the operation returns a record detected as spam + """ + spam: Boolean } scalar Upload @@ -18056,6 +20161,11 @@ type User { after: String """ + Username of the author + """ + authorUsername: String + + """ Returns the elements in the list that come before the specified cursor. """ before: String @@ -18136,6 +20246,11 @@ type User { after: String """ + Username of the assignee + """ + assigneeUsername: String + + """ Returns the elements in the list that come before the specified cursor. """ before: String @@ -18666,7 +20781,7 @@ type VulnerabilitiesCountByDayEdge { """ Represents a vulnerability """ -type Vulnerability { +type Vulnerability implements Noteable { """ Description of the vulnerability """ @@ -18678,6 +20793,31 @@ type Vulnerability { detectedAt: Time! """ + All discussions on this noteable + """ + discussions( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + ): DiscussionConnection! + + """ GraphQL ID of the vulnerability """ id: ID! @@ -18723,6 +20863,31 @@ type Vulnerability { location: VulnerabilityLocation """ + All notes on this noteable + """ + notes( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + ): NoteConnection! + + """ Primary identifier of the vulnerability. """ primaryIdentifier: VulnerabilityIdentifier @@ -18735,7 +20900,7 @@ type Vulnerability { """ Type of the security report that found the vulnerability (SAST, DEPENDENCY_SCANNING, CONTAINER_SCANNING, DAST, SECRET_DETECTION, - COVERAGE_FUZZING) + COVERAGE_FUZZING, API_FUZZING) """ reportType: VulnerabilityReportType @@ -18755,7 +20920,7 @@ type Vulnerability { severity: VulnerabilitySeverity """ - State of the vulnerability (DETECTED, DISMISSED, RESOLVED, CONFIRMED) + State of the vulnerability (DETECTED, CONFIRMED, RESOLVED, DISMISSED) """ state: VulnerabilityState @@ -18781,6 +20946,41 @@ type Vulnerability { } """ +Autogenerated input type of VulnerabilityConfirm +""" +input VulnerabilityConfirmInput { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + ID of the vulnerability to be confirmed + """ + id: VulnerabilityID! +} + +""" +Autogenerated return type of VulnerabilityConfirm +""" +type VulnerabilityConfirmPayload { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Errors encountered during execution of the mutation. + """ + errors: [String!]! + + """ + The vulnerability after state change + """ + vulnerability: Vulnerability +} + +""" The connection type for Vulnerability. """ type VulnerabilityConnection { @@ -18801,6 +21001,46 @@ type VulnerabilityConnection { } """ +Autogenerated input type of VulnerabilityDismiss +""" +input VulnerabilityDismissInput { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Reason why vulnerability should be dismissed + """ + comment: String + + """ + ID of the vulnerability to be dismissed + """ + id: VulnerabilityID! +} + +""" +Autogenerated return type of VulnerabilityDismiss +""" +type VulnerabilityDismissPayload { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Errors encountered during execution of the mutation. + """ + errors: [String!]! + + """ + The vulnerability after dismissal + """ + vulnerability: Vulnerability +} + +""" An edge in a connection. """ type VulnerabilityEdge { @@ -19123,6 +21363,7 @@ type VulnerabilityPermissions { The type of the security scan that found the vulnerability """ enum VulnerabilityReportType { + API_FUZZING CONTAINER_SCANNING COVERAGE_FUZZING DAST @@ -19141,7 +21382,7 @@ input VulnerabilityResolveInput { clientMutationId: String """ - ID of the vulnerability to be resolveed + ID of the vulnerability to be resolved """ id: VulnerabilityID! } @@ -19167,6 +21408,41 @@ type VulnerabilityResolvePayload { } """ +Autogenerated input type of VulnerabilityRevertToDetected +""" +input VulnerabilityRevertToDetectedInput { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + ID of the vulnerability to be reverted + """ + id: VulnerabilityID! +} + +""" +Autogenerated return type of VulnerabilityRevertToDetected +""" +type VulnerabilityRevertToDetectedPayload { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Errors encountered during execution of the mutation. + """ + errors: [String!]! + + """ + The vulnerability after revert + """ + vulnerability: Vulnerability +} + +""" Represents a vulnerability scanner """ type VulnerabilityScanner { @@ -19278,6 +21554,26 @@ Vulnerability sort values """ enum VulnerabilitySort { """ + Detection timestamp in ascending order + """ + detected_asc + + """ + Detection timestamp in descending order + """ + detected_desc + + """ + Report Type in ascending order + """ + report_type_asc + + """ + Report Type in descending order + """ + report_type_desc + + """ Severity in ascending order """ severity_asc @@ -19286,6 +21582,26 @@ enum VulnerabilitySort { Severity in descending order """ severity_desc + + """ + State in ascending order + """ + state_asc + + """ + State in descending order + """ + state_desc + + """ + Title in ascending order + """ + title_asc + + """ + Title in descending order + """ + title_desc } """ diff --git a/doc/api/graphql/reference/gitlab_schema.json b/doc/api/graphql/reference/gitlab_schema.json index 173415ca164..6914ba29c57 100644 --- a/doc/api/graphql/reference/gitlab_schema.json +++ b/doc/api/graphql/reference/gitlab_schema.json @@ -111,7 +111,7 @@ "name": null, "ofType": { "kind": "SCALAR", - "name": "ID", + "name": "AwardableID", "ofType": null } }, @@ -227,7 +227,7 @@ "name": null, "ofType": { "kind": "SCALAR", - "name": "ID", + "name": "ProjectID", "ofType": null } }, @@ -382,6 +382,16 @@ "defaultValue": null }, { + "name": "featureCategory", + "description": "Delete jobs matching feature_category in the context metadata", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { "name": "queueName", "description": "The name of the queue to delete jobs from", "type": { @@ -667,6 +677,20 @@ "deprecationReason": null }, { + "name": "environment", + "description": "Environment for the alert", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "Environment", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "eventCount", "description": "Number of events of this alert", "args": [ @@ -1227,23 +1251,47 @@ { "name": "updated_desc", "description": "Updated at descending order", + "isDeprecated": true, + "deprecationReason": "Use UPDATED_DESC. Deprecated in 13.5" + }, + { + "name": "updated_asc", + "description": "Updated at ascending order", + "isDeprecated": true, + "deprecationReason": "Use UPDATED_ASC. Deprecated in 13.5" + }, + { + "name": "created_desc", + "description": "Created at descending order", + "isDeprecated": true, + "deprecationReason": "Use CREATED_DESC. Deprecated in 13.5" + }, + { + "name": "created_asc", + "description": "Created at ascending order", + "isDeprecated": true, + "deprecationReason": "Use CREATED_ASC. Deprecated in 13.5" + }, + { + "name": "UPDATED_DESC", + "description": "Updated at descending order", "isDeprecated": false, "deprecationReason": null }, { - "name": "updated_asc", + "name": "UPDATED_ASC", "description": "Updated at ascending order", "isDeprecated": false, "deprecationReason": null }, { - "name": "created_desc", + "name": "CREATED_DESC", "description": "Created at descending order", "isDeprecated": false, "deprecationReason": null }, { - "name": "created_asc", + "name": "CREATED_ASC", "description": "Created at ascending order", "isDeprecated": false, "deprecationReason": null @@ -1969,7 +2017,7 @@ "name": null, "ofType": { "kind": "SCALAR", - "name": "ID", + "name": "AwardableID", "ofType": null } }, @@ -2085,7 +2133,7 @@ "name": null, "ofType": { "kind": "SCALAR", - "name": "ID", + "name": "AwardableID", "ofType": null } }, @@ -2201,7 +2249,7 @@ "name": null, "ofType": { "kind": "SCALAR", - "name": "ID", + "name": "AwardableID", "ofType": null } }, @@ -2322,6 +2370,16 @@ "possibleTypes": null }, { + "kind": "SCALAR", + "name": "AwardableID", + "description": "Identifier of Awardable", + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { "kind": "OBJECT", "name": "BaseService", "description": null, @@ -2764,7 +2822,7 @@ ], "type": { "kind": "OBJECT", - "name": "EpicConnection", + "name": "BoardEpicConnection", "ofType": null }, "isDeprecated": false, @@ -2817,6 +2875,59 @@ "deprecationReason": null }, { + "name": "labels", + "description": "Labels of the board", + "args": [ + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "LabelConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "lists", "description": "Lists of the board", "args": [ @@ -2831,6 +2942,16 @@ "defaultValue": null }, { + "name": "issueFilters", + "description": "Filters applied when getting issue metadata in the board list", + "type": { + "kind": "INPUT_OBJECT", + "name": "BoardIssueInput", + "ofType": null + }, + "defaultValue": null + }, + { "name": "after", "description": "Returns the elements in the list that come after the specified cursor.", "type": { @@ -3042,6 +3163,1273 @@ "possibleTypes": null }, { + "kind": "OBJECT", + "name": "BoardEpic", + "description": "Represents an epic on an issue board", + "fields": [ + { + "name": "author", + "description": "Author of the epic", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "User", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "children", + "description": "Children (sub-epics) of the epic", + "args": [ + { + "name": "startDate", + "description": "List items overlapping a time frame defined by startDate..endDate (if one date is provided, both must be present). Deprecated in 13.5: Use timeframe.start", + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "endDate", + "description": "List items overlapping a time frame defined by startDate..endDate (if one date is provided, both must be present). Deprecated in 13.5: Use timeframe.end", + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "timeframe", + "description": "List items overlapping the given timeframe", + "type": { + "kind": "INPUT_OBJECT", + "name": "Timeframe", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "iid", + "description": "IID of the epic, e.g., \"1\"", + "type": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "iids", + "description": "List of IIDs of epics, e.g., [1, 2]", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + } + }, + "defaultValue": null + }, + { + "name": "state", + "description": "Filter epics by state", + "type": { + "kind": "ENUM", + "name": "EpicState", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "search", + "description": "Search query for epic title or description", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "sort", + "description": "List epics by sort order", + "type": { + "kind": "ENUM", + "name": "EpicSort", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "authorUsername", + "description": "Filter epics by author", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "labelName", + "description": "Filter epics by labels", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + }, + "defaultValue": null + }, + { + "name": "milestoneTitle", + "description": "Filter epics by milestone title, computed from epic's issues", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "iidStartsWith", + "description": "Filter epics by iid for autocomplete", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "EpicConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "closedAt", + "description": "Timestamp of when the epic was closed", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "confidential", + "description": "Indicates if the epic is confidential", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "createdAt", + "description": "Timestamp of when the epic was created", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "currentUserTodos", + "description": "Todos for the current user", + "args": [ + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "state", + "description": "State of the todos", + "type": { + "kind": "ENUM", + "name": "TodoStateEnum", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "TodoConnection", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "descendantCounts", + "description": "Number of open and closed descendant epics and issues", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "EpicDescendantCount", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "descendantWeightSum", + "description": "Total weight of open and closed issues in the epic and its descendants", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "EpicDescendantWeights", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "description", + "description": "Description of the epic", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "discussions", + "description": "All discussions on this noteable", + "args": [ + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "DiscussionConnection", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "downvotes", + "description": "Number of downvotes the epic has received", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "dueDate", + "description": "Due date of the epic", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "dueDateFixed", + "description": "Fixed due date of the epic", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "dueDateFromMilestones", + "description": "Inherited due date of the epic from milestones", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "dueDateIsFixed", + "description": "Indicates if the due date has been manually set", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "group", + "description": "Group to which the epic belongs", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "Group", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "hasChildren", + "description": "Indicates if the epic has children", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "hasIssues", + "description": "Indicates if the epic has direct issues", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "hasParent", + "description": "Indicates if the epic has a parent epic", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "healthStatus", + "description": "Current health status of the epic", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "EpicHealthStatus", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "id", + "description": "ID of the epic", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "iid", + "description": "Internal ID of the epic", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "issues", + "description": "A list of issues associated with the epic", + "args": [ + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "EpicIssueConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "labels", + "description": "Labels assigned to the epic", + "args": [ + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "LabelConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "notes", + "description": "All notes on this noteable", + "args": [ + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "NoteConnection", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "parent", + "description": "Parent epic of the epic", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "Epic", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "participants", + "description": "List of participants for the epic", + "args": [ + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "UserConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "reference", + "description": "Internal reference of the epic. Returned in shortened format by default", + "args": [ + { + "name": "full", + "description": "Indicates if the reference should be returned in full", + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "defaultValue": "false" + } + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "relationPath", + "description": "URI path of the epic-issue relationship", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "relativePosition", + "description": "The relative position of the epic in the epic tree", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "startDate", + "description": "Start date of the epic", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "startDateFixed", + "description": "Fixed start date of the epic", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "startDateFromMilestones", + "description": "Inherited start date of the epic from milestones", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "startDateIsFixed", + "description": "Indicates if the start date has been manually set", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "state", + "description": "State of the epic", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "ENUM", + "name": "EpicState", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "subscribed", + "description": "Indicates the currently logged in user is subscribed to the epic", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "title", + "description": "Title of the epic", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "updatedAt", + "description": "Timestamp of when the epic was updated", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "upvotes", + "description": "Number of upvotes the epic has received", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "userPermissions", + "description": "Permissions for the current user on the resource", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "EpicPermissions", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "userPreferences", + "description": "User preferences for the epic on the issue board", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "BoardEpicUserPreferences", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "webPath", + "description": "Web path of the epic", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "webUrl", + "description": "Web URL of the epic", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + { + "kind": "INTERFACE", + "name": "Noteable", + "ofType": null + }, + { + "kind": "INTERFACE", + "name": "CurrentUserTodos", + "ofType": null + } + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "BoardEpicConnection", + "description": "The connection type for BoardEpic.", + "fields": [ + { + "name": "edges", + "description": "A list of edges.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "BoardEpicEdge", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "nodes", + "description": "A list of nodes.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "BoardEpic", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "pageInfo", + "description": "Information to aid in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "PageInfo", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "BoardEpicEdge", + "description": "An edge in a connection.", + "fields": [ + { + "name": "cursor", + "description": "A cursor for use in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "node", + "description": "The item at the end of the edge.", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "BoardEpic", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "BoardEpicUserPreferences", + "description": "Represents user preferences for a board epic", + "fields": [ + { + "name": "collapsed", + "description": "Indicates epic should be displayed as collapsed", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { "kind": "SCALAR", "name": "BoardID", "description": "Identifier of Board", @@ -3718,7 +5106,7 @@ "name": null, "ofType": { "kind": "SCALAR", - "name": "ID", + "name": "ListID", "ofType": null } }, @@ -4000,6 +5388,20 @@ "description": null, "fields": [ { + "name": "detailedStatus", + "description": "Detailed status of the group", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "DetailedStatus", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "jobs", "description": "Jobs in group", "args": [ @@ -4206,6 +5608,20 @@ "description": null, "fields": [ { + "name": "detailedStatus", + "description": "Detailed status of the job", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "DetailedStatus", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "name", "description": "Name of the job", "args": [ @@ -4271,6 +5687,20 @@ }, "isDeprecated": false, "deprecationReason": null + }, + { + "name": "scheduledAt", + "description": "Schedule for the build", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null } ], "inputFields": null, @@ -4408,6 +5838,20 @@ "description": null, "fields": [ { + "name": "detailedStatus", + "description": "Detailed status of the stage", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "DetailedStatus", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "groups", "description": "Group of jobs for the stage", "args": [ @@ -5330,6 +6774,16 @@ "possibleTypes": null }, { + "kind": "SCALAR", + "name": "ClustersClusterID", + "description": "Identifier of Clusters::Cluster", + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { "kind": "OBJECT", "name": "Commit", "description": null, @@ -6695,7 +8149,7 @@ "description": "The global id of the environment to add an annotation to", "type": { "kind": "SCALAR", - "name": "ID", + "name": "EnvironmentID", "ofType": null }, "defaultValue": null @@ -6705,7 +8159,7 @@ "description": "The global id of the cluster to add an annotation to", "type": { "kind": "SCALAR", - "name": "ID", + "name": "ClustersClusterID", "ofType": null }, "defaultValue": null @@ -6846,6 +8300,172 @@ }, { "kind": "INPUT_OBJECT", + "name": "CreateBoardInput", + "description": "Autogenerated input type of CreateBoard", + "fields": null, + "inputFields": [ + { + "name": "projectPath", + "description": "The project full path the board is associated with.", + "type": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "groupPath", + "description": "The group full path the board is associated with.", + "type": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "name", + "description": "The board name.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "assigneeId", + "description": "The ID of the user to be assigned to the board.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "milestoneId", + "description": "The ID of the milestone to be assigned to the board.", + "type": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "weight", + "description": "The weight of the board.", + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "labelIds", + "description": "The IDs of labels to be added to the board.", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + } + }, + "defaultValue": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "CreateBoardPayload", + "description": "Autogenerated return type of CreateBoard", + "fields": [ + { + "name": "board", + "description": "The board after mutation.", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "Board", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "errors", + "description": "Errors encountered during execution of the mutation.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "INPUT_OBJECT", "name": "CreateBranchInput", "description": "Autogenerated input type of CreateBranch", "fields": null, @@ -7104,7 +8724,7 @@ "name": null, "ofType": { "kind": "SCALAR", - "name": "ID", + "name": "NoteableID", "ofType": null } }, @@ -7452,7 +9072,7 @@ "name": null, "ofType": { "kind": "SCALAR", - "name": "ID", + "name": "NoteableID", "ofType": null } }, @@ -7580,6 +9200,296 @@ }, { "kind": "INPUT_OBJECT", + "name": "CreateIssueInput", + "description": "Autogenerated input type of CreateIssue", + "fields": null, + "inputFields": [ + { + "name": "description", + "description": "Description of the issue", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "dueDate", + "description": "Due date of the issue", + "type": { + "kind": "SCALAR", + "name": "ISO8601Date", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "confidential", + "description": "Indicates the issue is confidential", + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "locked", + "description": "Indicates discussion is locked on the issue", + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "projectPath", + "description": "Project full path the issue is associated with", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "iid", + "description": "The IID (internal ID) of a project issue. Only admins and project owners can modify", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "title", + "description": "Title of the issue", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "milestoneId", + "description": "The ID of the milestone to assign to the issue. On update milestone will be removed if set to null", + "type": { + "kind": "SCALAR", + "name": "MilestoneID", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "labels", + "description": "Labels of the issue", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + }, + "defaultValue": null + }, + { + "name": "labelIds", + "description": "The IDs of labels to be added to the issue", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "LabelID", + "ofType": null + } + } + }, + "defaultValue": null + }, + { + "name": "createdAt", + "description": "Timestamp when the issue was created. Available only for admins and project owners", + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "mergeRequestToResolveDiscussionsOf", + "description": "The IID of a merge request for which to resolve discussions", + "type": { + "kind": "SCALAR", + "name": "MergeRequestID", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "discussionToResolve", + "description": "The ID of a discussion to resolve. Also pass `merge_request_to_resolve_discussions_of`", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "assigneeIds", + "description": "The array of user IDs to assign to the issue", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "UserID", + "ofType": null + } + } + }, + "defaultValue": null + }, + { + "name": "healthStatus", + "description": "The desired health status", + "type": { + "kind": "ENUM", + "name": "HealthStatus", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "weight", + "description": "The weight of the issue", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "epicId", + "description": "The ID of an epic to associate the issue with", + "type": { + "kind": "SCALAR", + "name": "EpicID", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "CreateIssuePayload", + "description": "Autogenerated return type of CreateIssue", + "fields": [ + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "errors", + "description": "Errors encountered during execution of the mutation.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "issue", + "description": "The issue after mutation", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "Issue", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "INPUT_OBJECT", "name": "CreateIterationInput", "description": "Autogenerated input type of CreateIteration", "fields": null, @@ -7740,7 +9650,7 @@ "name": null, "ofType": { "kind": "SCALAR", - "name": "ID", + "name": "NoteableID", "ofType": null } }, @@ -7775,7 +9685,7 @@ "description": "The global id of the discussion this note is in reply to", "type": { "kind": "SCALAR", - "name": "ID", + "name": "DiscussionID", "ofType": null }, "defaultValue": null @@ -7872,19 +9782,25 @@ "name": "title", "description": "Title of the requirement", "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "description", + "description": "Description of the requirement", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null }, "defaultValue": null }, { "name": "projectPath", - "description": "The project full path the requirement is associated with", + "description": "Full project path the requirement is associated with", "type": { "kind": "NON_NULL", "name": null, @@ -7958,7 +9874,7 @@ }, { "name": "requirement", - "description": "The requirement after mutation", + "description": "Requirement after mutation", "args": [ ], @@ -8141,6 +10057,20 @@ }, "isDeprecated": false, "deprecationReason": null + }, + { + "name": "spam", + "description": "Indicates whether the operation returns a record detected as spam", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null } ], "inputFields": null, @@ -8373,6 +10303,11 @@ "possibleTypes": [ { "kind": "OBJECT", + "name": "BoardEpic", + "ofType": null + }, + { + "kind": "OBJECT", "name": "Design", "ofType": null }, @@ -8537,6 +10472,12 @@ "description": "Passive DAST scan. This scan will not make active attacks against the target site.", "isDeprecated": false, "deprecationReason": null + }, + { + "name": "ACTIVE", + "description": "Active DAST scan. This scan will make active attacks against the target site.", + "isDeprecated": false, + "deprecationReason": null } ], "possibleTypes": null @@ -8611,6 +10552,38 @@ "deprecationReason": null }, { + "name": "scanType", + "description": "Indicates the type of DAST scan that will run. Either a Passive Scan or an Active Scan.", + "args": [ + + ], + "type": { + "kind": "ENUM", + "name": "DastScanTypeEnum", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "showDebugMessages", + "description": "Indicates if debug messages should be included in DAST console output. True to include the debug messages.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "spiderTimeout", "description": "The maximum number of minutes allowed for the spider to traverse the site", "args": [ @@ -8637,6 +10610,24 @@ }, "isDeprecated": false, "deprecationReason": null + }, + { + "name": "useAjaxSpider", + "description": "Indicates if the AJAX spider should be used to crawl the target site. True to run the AJAX spider in addition to the traditional spider, and false to run only the traditional spider.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null } ], "inputFields": null, @@ -8768,6 +10759,36 @@ "defaultValue": null }, { + "name": "scanType", + "description": "Indicates the type of DAST scan that will run. Either a Passive Scan or an Active Scan.", + "type": { + "kind": "ENUM", + "name": "DastScanTypeEnum", + "ofType": null + }, + "defaultValue": "PASSIVE" + }, + { + "name": "useAjaxSpider", + "description": "Indicates if the AJAX spider should be used to crawl the target site. True to run the AJAX spider in addition to the traditional spider, and false to run only the traditional spider.", + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "defaultValue": "false" + }, + { + "name": "showDebugMessages", + "description": "Indicates if debug messages should be included in DAST console output. True to include the debug messages.", + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "defaultValue": "false" + }, + { "name": "clientMutationId", "description": "A unique identifier for the client performing the mutation.", "type": { @@ -9097,6 +11118,36 @@ "defaultValue": null }, { + "name": "scanType", + "description": "Indicates the type of DAST scan that will run. Either a Passive Scan or an Active Scan.", + "type": { + "kind": "ENUM", + "name": "DastScanTypeEnum", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "useAjaxSpider", + "description": "Indicates if the AJAX spider should be used to crawl the target site. True to run the AJAX spider in addition to the traditional spider, and false to run only the traditional spider.", + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "showDebugMessages", + "description": "Indicates if debug messages should be included in DAST console output. True to include the debug messages.", + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "defaultValue": null + }, + { "name": "clientMutationId", "description": "A unique identifier for the client performing the mutation.", "type": { @@ -9841,6 +11892,166 @@ }, { "kind": "INPUT_OBJECT", + "name": "DastSiteTokenCreateInput", + "description": "Autogenerated input type of DastSiteTokenCreate", + "fields": null, + "inputFields": [ + { + "name": "fullPath", + "description": "The project the site token belongs to.", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "targetUrl", + "description": "The URL of the target to be validated.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "DastSiteTokenCreatePayload", + "description": "Autogenerated return type of DastSiteTokenCreate", + "fields": [ + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "errors", + "description": "Errors encountered during execution of the mutation.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "id", + "description": "ID of the site token.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "DastSiteTokenID", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "status", + "description": "The current validation status of the target.", + "args": [ + + ], + "type": { + "kind": "ENUM", + "name": "DastSiteProfileValidationStatusEnum", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "token", + "description": "Token string.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "SCALAR", + "name": "DastSiteTokenID", + "description": "Identifier of DastSiteToken", + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "SCALAR", + "name": "Date", + "description": "Date represented in ISO 8601", + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "INPUT_OBJECT", "name": "DeleteAnnotationInput", "description": "Autogenerated input type of DeleteAnnotation", "fields": null, @@ -10790,6 +13001,20 @@ "description": "A collection of designs", "fields": [ { + "name": "copyState", + "description": "Copy state of the design collection", + "args": [ + + ], + "type": { + "kind": "ENUM", + "name": "DesignCollectionCopyState", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "design", "description": "Find a specific design", "args": [ @@ -11107,6 +13332,35 @@ "possibleTypes": null }, { + "kind": "ENUM", + "name": "DesignCollectionCopyState", + "description": "Copy state of a DesignCollection", + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": [ + { + "name": "READY", + "description": "The DesignCollection has no copy in progress", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "IN_PROGRESS", + "description": "The DesignCollection is being copied", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "ERROR", + "description": "The DesignCollection encountered an error during a copy", + "isDeprecated": false, + "deprecationReason": null + } + ], + "possibleTypes": null + }, + { "kind": "OBJECT", "name": "DesignConnection", "description": "The connection type for Design.", @@ -12358,6 +14612,108 @@ "possibleTypes": null }, { + "kind": "INPUT_OBJECT", + "name": "DestroyBoardListInput", + "description": "Autogenerated input type of DestroyBoardList", + "fields": null, + "inputFields": [ + { + "name": "listId", + "description": "Global ID of the list to destroy. Only label lists are accepted.", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ListID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "DestroyBoardListPayload", + "description": "Autogenerated return type of DestroyBoardList", + "fields": [ + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "errors", + "description": "Errors encountered during execution of the mutation.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "list", + "description": "The list after mutation.", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "BoardList", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { "kind": "OBJECT", "name": "DestroyBoardPayload", "description": "Autogenerated return type of DestroyBoard", @@ -12438,7 +14794,7 @@ "name": null, "ofType": { "kind": "SCALAR", - "name": "ID", + "name": "NoteID", "ofType": null } }, @@ -12634,145 +14990,127 @@ "description": null, "fields": [ { + "name": "action", + "description": "Action information for the status. This includes method, button title, icon, path, and title", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "StatusAction", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "detailsPath", - "description": "Path of the details for the pipeline status", + "description": "Path of the details for the status", "args": [ ], "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } + "kind": "SCALAR", + "name": "String", + "ofType": null }, "isDeprecated": false, "deprecationReason": null }, { "name": "favicon", - "description": "Favicon of the pipeline status", + "description": "Favicon of the status", "args": [ ], "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } + "kind": "SCALAR", + "name": "String", + "ofType": null }, "isDeprecated": false, "deprecationReason": null }, { "name": "group", - "description": "Group of the pipeline status", + "description": "Group of the status", "args": [ ], "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } + "kind": "SCALAR", + "name": "String", + "ofType": null }, "isDeprecated": false, "deprecationReason": null }, { "name": "hasDetails", - "description": "Indicates if the pipeline status has further details", + "description": "Indicates if the status has further details", "args": [ ], "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Boolean", - "ofType": null - } + "kind": "SCALAR", + "name": "Boolean", + "ofType": null }, "isDeprecated": false, "deprecationReason": null }, { "name": "icon", - "description": "Icon of the pipeline status", + "description": "Icon of the status", "args": [ ], "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } + "kind": "SCALAR", + "name": "String", + "ofType": null }, "isDeprecated": false, "deprecationReason": null }, { "name": "label", - "description": "Label of the pipeline status", + "description": "Label of the status", "args": [ ], "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } + "kind": "SCALAR", + "name": "String", + "ofType": null }, "isDeprecated": false, "deprecationReason": null }, { "name": "text", - "description": "Text of the pipeline status", + "description": "Text of the status", "args": [ ], "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } + "kind": "SCALAR", + "name": "String", + "ofType": null }, "isDeprecated": false, "deprecationReason": null }, { "name": "tooltip", - "description": "Tooltip associated with the pipeline status", + "description": "Tooltip associated with the status", "args": [ ], "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } + "kind": "SCALAR", + "name": "String", + "ofType": null }, "isDeprecated": false, "deprecationReason": null @@ -13744,6 +16082,16 @@ "possibleTypes": null }, { + "kind": "SCALAR", + "name": "DiscussionID", + "description": "Identifier of Discussion", + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { "kind": "INPUT_OBJECT", "name": "DiscussionToggleResolveInput", "description": "Autogenerated input type of DiscussionToggleResolve", @@ -13757,7 +16105,7 @@ "name": null, "ofType": { "kind": "SCALAR", - "name": "ID", + "name": "DiscussionID", "ofType": null } }, @@ -13873,7 +16221,7 @@ "name": null, "ofType": { "kind": "SCALAR", - "name": "ID", + "name": "VulnerabilityID", "ofType": null } }, @@ -14160,7 +16508,7 @@ }, { "name": "latestOpenedMostSevereAlert", - "description": "The most severe open alert for the environment. If multiple alerts have equal severity, the most recent is returned.", + "description": "The most severe open alert for the environment. If multiple alerts have equal severity, the most recent is returned", "args": [ ], @@ -14218,6 +16566,20 @@ "deprecationReason": null }, { + "name": "path", + "description": "The path to the environment. Will always return null if `expose_environment_path_in_alert_details` feature flag is disabled", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "state", "description": "State of the environment, for example: available/stopped", "args": [ @@ -14356,6 +16718,16 @@ "possibleTypes": null }, { + "kind": "SCALAR", + "name": "EnvironmentID", + "description": "Identifier of Environment", + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { "kind": "OBJECT", "name": "Epic", "description": "Represents an epic", @@ -14384,7 +16756,7 @@ "args": [ { "name": "startDate", - "description": "List items within a time frame where items.start_date is between startDate and endDate parameters (endDate parameter must be present)", + "description": "List items overlapping a time frame defined by startDate..endDate (if one date is provided, both must be present). Deprecated in 13.5: Use timeframe.start", "type": { "kind": "SCALAR", "name": "Time", @@ -14394,7 +16766,7 @@ }, { "name": "endDate", - "description": "List items within a time frame where items.end_date is between startDate and endDate parameters (startDate parameter must be present)", + "description": "List items overlapping a time frame defined by startDate..endDate (if one date is provided, both must be present). Deprecated in 13.5: Use timeframe.end", "type": { "kind": "SCALAR", "name": "Time", @@ -14403,6 +16775,16 @@ "defaultValue": null }, { + "name": "timeframe", + "description": "List items overlapping the given timeframe", + "type": { + "kind": "INPUT_OBJECT", + "name": "Timeframe", + "ofType": null + }, + "defaultValue": null + }, + { "name": "iid", "description": "IID of the epic, e.g., \"1\"", "type": { @@ -14559,7 +16941,7 @@ }, { "name": "closedAt", - "description": "Timestamp of the epic's closure", + "description": "Timestamp of when the epic was closed", "args": [ ], @@ -14587,7 +16969,7 @@ }, { "name": "createdAt", - "description": "Timestamp of the epic's creation", + "description": "Timestamp of when the epic was created", "args": [ ], @@ -15354,7 +17736,7 @@ }, { "name": "updatedAt", - "description": "Timestamp of the epic's last activity", + "description": "Timestamp of when the epic was updated", "args": [ ], @@ -16627,6 +19009,20 @@ "deprecationReason": null }, { + "name": "slaDueAt", + "description": "Timestamp of when the issue SLA expires.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "state", "description": "State of the issue", "args": [ @@ -17433,7 +19829,7 @@ "name": null, "ofType": { "kind": "SCALAR", - "name": "ID", + "name": "EpicTreeSortingID", "ofType": null } }, @@ -17444,7 +19840,7 @@ "description": "The id of the epic_issue or issue that the actual epic or issue is switched with", "type": { "kind": "SCALAR", - "name": "ID", + "name": "EpicTreeSortingID", "ofType": null }, "defaultValue": null @@ -17464,7 +19860,7 @@ "description": "ID of the new parent epic", "type": { "kind": "SCALAR", - "name": "ID", + "name": "EpicID", "ofType": null }, "defaultValue": null @@ -17488,7 +19884,7 @@ "name": null, "ofType": { "kind": "SCALAR", - "name": "ID", + "name": "EpicID", "ofType": null } }, @@ -17577,6 +19973,16 @@ "possibleTypes": null }, { + "kind": "SCALAR", + "name": "EpicTreeSortingID", + "description": "Identifier of EpicTreeSorting", + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { "kind": "ENUM", "name": "EpicWildcardId", "description": "Epic ID wildcard values", @@ -17689,6 +20095,77 @@ "deprecationReason": null }, { + "name": "mergeRequestDiffRegistries", + "description": "Find merge request diff registries on this Geo node", + "args": [ + { + "name": "ids", + "description": "Filters registries by their ID", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + } + }, + "defaultValue": null + }, + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "MergeRequestDiffRegistryConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "minimumReverificationInterval", "description": "The interval (in days) in which the repository verification is valid. Once expired, it will be reverified", "args": [ @@ -17919,8 +20396,8 @@ "deprecationReason": null }, { - "name": "terraformStateRegistries", - "description": "Find terraform state registries on this Geo node. Available only when feature flag `geo_terraform_state_replication` is enabled", + "name": "terraformStateVersionRegistries", + "description": "Find terraform state version registries on this Geo node", "args": [ { "name": "ids", @@ -17983,7 +20460,7 @@ ], "type": { "kind": "OBJECT", - "name": "TerraformStateRegistryConnection", + "name": "TerraformStateVersionRegistryConnection", "ofType": null }, "isDeprecated": false, @@ -18152,6 +20629,34 @@ "description": null, "fields": [ { + "name": "actualRepositorySizeLimit", + "description": "Size limit for repositories in the namespace in bytes", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Float", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "additionalPurchasedStorageSize", + "description": "Additional storage purchased for the root namespace in bytes", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Float", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "autoDevopsEnabled", "description": "Indicates whether Auto DevOps is enabled for all projects within this group", "args": [ @@ -18185,11 +20690,15 @@ "args": [ { "name": "id", - "description": "Find a board by its ID", + "description": "The board's ID", "type": { - "kind": "SCALAR", - "name": "ID", - "ofType": null + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "BoardID", + "ofType": null + } }, "defaultValue": null } @@ -18266,6 +20775,24 @@ "deprecationReason": null }, { + "name": "containsLockedProjects", + "description": "Includes at least one project where the repository size exceeds the limit", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "description", "description": "Description of the namespace", "args": [ @@ -18313,7 +20840,7 @@ "args": [ { "name": "startDate", - "description": "List items within a time frame where items.start_date is between startDate and endDate parameters (endDate parameter must be present)", + "description": "List items overlapping a time frame defined by startDate..endDate (if one date is provided, both must be present). Deprecated in 13.5: Use timeframe.start", "type": { "kind": "SCALAR", "name": "Time", @@ -18323,7 +20850,7 @@ }, { "name": "endDate", - "description": "List items within a time frame where items.end_date is between startDate and endDate parameters (startDate parameter must be present)", + "description": "List items overlapping a time frame defined by startDate..endDate (if one date is provided, both must be present). Deprecated in 13.5: Use timeframe.end", "type": { "kind": "SCALAR", "name": "Time", @@ -18332,6 +20859,16 @@ "defaultValue": null }, { + "name": "timeframe", + "description": "List items overlapping the given timeframe", + "type": { + "kind": "INPUT_OBJECT", + "name": "Timeframe", + "ofType": null + }, + "defaultValue": null + }, + { "name": "iid", "description": "IID of the epic, e.g., \"1\"", "type": { @@ -18452,7 +20989,7 @@ "args": [ { "name": "startDate", - "description": "List items within a time frame where items.start_date is between startDate and endDate parameters (endDate parameter must be present)", + "description": "List items overlapping a time frame defined by startDate..endDate (if one date is provided, both must be present). Deprecated in 13.5: Use timeframe.start", "type": { "kind": "SCALAR", "name": "Time", @@ -18462,7 +20999,7 @@ }, { "name": "endDate", - "description": "List items within a time frame where items.end_date is between startDate and endDate parameters (startDate parameter must be present)", + "description": "List items overlapping a time frame defined by startDate..endDate (if one date is provided, both must be present). Deprecated in 13.5: Use timeframe.end", "type": { "kind": "SCALAR", "name": "Time", @@ -18471,6 +21008,16 @@ "defaultValue": null }, { + "name": "timeframe", + "description": "List items overlapping the given timeframe", + "type": { + "kind": "INPUT_OBJECT", + "name": "Timeframe", + "ofType": null + }, + "defaultValue": null + }, + { "name": "iid", "description": "IID of the epic, e.g., \"1\"", "type": { @@ -18790,7 +21337,7 @@ }, { "name": "issues", - "description": "Issues of the group", + "description": "Issues for projects in this group", "args": [ { "name": "iid", @@ -18849,6 +21396,16 @@ "defaultValue": null }, { + "name": "authorUsername", + "description": "Username of the author of the issue", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { "name": "assigneeUsername", "description": "Username of a user assigned to the issue", "type": { @@ -18859,6 +21416,24 @@ "defaultValue": null }, { + "name": "assigneeUsernames", + "description": "Usernames of users assigned to the issue", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + }, + "defaultValue": null + }, + { "name": "assigneeId", "description": "ID of a user assigned to the issues, \"none\" and \"any\" values supported", "type": { @@ -18992,7 +21567,7 @@ }, { "name": "includeSubgroups", - "description": "Include issues belonging to subgroups.", + "description": "Include issues belonging to subgroups", "type": { "kind": "SCALAR", "name": "Boolean", @@ -19055,7 +21630,7 @@ "args": [ { "name": "startDate", - "description": "List items within a time frame where items.start_date is between startDate and endDate parameters (endDate parameter must be present)", + "description": "List items overlapping a time frame defined by startDate..endDate (if one date is provided, both must be present). Deprecated in 13.5: Use timeframe.start", "type": { "kind": "SCALAR", "name": "Time", @@ -19065,7 +21640,7 @@ }, { "name": "endDate", - "description": "List items within a time frame where items.end_date is between startDate and endDate parameters (startDate parameter must be present)", + "description": "List items overlapping a time frame defined by startDate..endDate (if one date is provided, both must be present). Deprecated in 13.5: Use timeframe.end", "type": { "kind": "SCALAR", "name": "Time", @@ -19074,6 +21649,16 @@ "defaultValue": null }, { + "name": "timeframe", + "description": "List items overlapping the given timeframe", + "type": { + "kind": "INPUT_OBJECT", + "name": "Timeframe", + "ofType": null + }, + "defaultValue": null + }, + { "name": "state", "description": "Filter iterations by state", "type": { @@ -19291,12 +21876,217 @@ "deprecationReason": null }, { + "name": "mergeRequests", + "description": "Merge requests for projects in this group", + "args": [ + { + "name": "iids", + "description": "Array of IIDs of merge requests, for example `[1, 2]`", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + }, + "defaultValue": null + }, + { + "name": "sourceBranches", + "description": "Array of source branch names. All resolved merge requests will have one of these branches as their source.", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + }, + "defaultValue": null + }, + { + "name": "targetBranches", + "description": "Array of target branch names. All resolved merge requests will have one of these branches as their target.", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + }, + "defaultValue": null + }, + { + "name": "state", + "description": "A merge request state. If provided, all resolved merge requests will have this state.", + "type": { + "kind": "ENUM", + "name": "MergeRequestState", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "labels", + "description": "Array of label names. All resolved merge requests will have all of these labels.", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + }, + "defaultValue": null + }, + { + "name": "mergedAfter", + "description": "Merge requests merged after this date", + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "mergedBefore", + "description": "Merge requests merged before this date", + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "milestoneTitle", + "description": "Title of the milestone", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "sort", + "description": "Sort merge requests by this criteria", + "type": { + "kind": "ENUM", + "name": "MergeRequestSort", + "ofType": null + }, + "defaultValue": "created_desc" + }, + { + "name": "includeSubgroups", + "description": "Include merge requests belonging to subgroups", + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "defaultValue": "false" + }, + { + "name": "assigneeUsername", + "description": "Username of the assignee", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "authorUsername", + "description": "Username of the author", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "MergeRequestConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "milestones", "description": "Milestones of the group", "args": [ { "name": "startDate", - "description": "List items within a time frame where items.start_date is between startDate and endDate parameters (endDate parameter must be present)", + "description": "List items overlapping a time frame defined by startDate..endDate (if one date is provided, both must be present). Deprecated in 13.5: Use timeframe.start", "type": { "kind": "SCALAR", "name": "Time", @@ -19306,7 +22096,7 @@ }, { "name": "endDate", - "description": "List items within a time frame where items.end_date is between startDate and endDate parameters (startDate parameter must be present)", + "description": "List items overlapping a time frame defined by startDate..endDate (if one date is provided, both must be present). Deprecated in 13.5: Use timeframe.end", "type": { "kind": "SCALAR", "name": "Time", @@ -19315,6 +22105,16 @@ "defaultValue": null }, { + "name": "timeframe", + "description": "List items overlapping the given timeframe", + "type": { + "kind": "INPUT_OBJECT", + "name": "Timeframe", + "ofType": null + }, + "defaultValue": null + }, + { "name": "ids", "description": "Array of global milestone IDs, e.g., \"gid://gitlab/Milestone/1\"", "type": { @@ -19343,6 +22143,36 @@ "defaultValue": null }, { + "name": "title", + "description": "The title of the milestone", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "searchTitle", + "description": "A search string for the title", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "containingDate", + "description": "A date that the milestone contains", + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "defaultValue": null + }, + { "name": "includeDescendants", "description": "Also return milestones in all subgroups and subprojects", "type": { @@ -19563,6 +22393,24 @@ "deprecationReason": null }, { + "name": "repositorySizeExcessProjectCount", + "description": "Number of projects in the root namespace where the repository size exceeds the limit", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "requestAccessEnabled", "description": "Indicates if users can request access to namespace", "args": [ @@ -19758,6 +22606,34 @@ "deprecationReason": null }, { + "name": "totalRepositorySize", + "description": "Total repository size of all projects in the root namespace in bytes", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Float", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "totalRepositorySizeExcess", + "description": "Total excess repository size of all projects in the root namespace in bytes", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Float", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "twoFactorGracePeriod", "description": "Time before two-factor authentication is enforced", "args": [ @@ -21905,6 +24781,20 @@ "deprecationReason": null }, { + "name": "slaDueAt", + "description": "Timestamp of when the issue SLA expires.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "state", "description": "State of the issue", "args": [ @@ -22335,6 +25225,69 @@ }, { "kind": "INPUT_OBJECT", + "name": "IssueMoveInput", + "description": "Autogenerated input type of IssueMove", + "fields": null, + "inputFields": [ + { + "name": "projectPath", + "description": "The project the issue to mutate is in", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "iid", + "description": "The IID of the issue to mutate", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "targetProjectPath", + "description": "The project to move the issue to", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "INPUT_OBJECT", "name": "IssueMoveListInput", "description": "Autogenerated input type of IssueMoveList", "fields": null, @@ -22515,6 +25468,73 @@ }, { "kind": "OBJECT", + "name": "IssueMovePayload", + "description": "Autogenerated return type of IssueMove", + "fields": [ + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "errors", + "description": "Errors encountered during execution of the mutation.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "issue", + "description": "The issue after mutation", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "Issue", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", "name": "IssuePermissions", "description": "Check permissions for the current user on a issue", "fields": [ @@ -23243,7 +26263,7 @@ "description": "The iteration to assign to the issue.\n", "type": { "kind": "SCALAR", - "name": "ID", + "name": "IterationID", "ofType": null }, "defaultValue": null @@ -23861,23 +26881,47 @@ { "name": "updated_desc", "description": "Updated at descending order", + "isDeprecated": true, + "deprecationReason": "Use UPDATED_DESC. Deprecated in 13.5" + }, + { + "name": "updated_asc", + "description": "Updated at ascending order", + "isDeprecated": true, + "deprecationReason": "Use UPDATED_ASC. Deprecated in 13.5" + }, + { + "name": "created_desc", + "description": "Created at descending order", + "isDeprecated": true, + "deprecationReason": "Use CREATED_DESC. Deprecated in 13.5" + }, + { + "name": "created_asc", + "description": "Created at ascending order", + "isDeprecated": true, + "deprecationReason": "Use CREATED_ASC. Deprecated in 13.5" + }, + { + "name": "UPDATED_DESC", + "description": "Updated at descending order", "isDeprecated": false, "deprecationReason": null }, { - "name": "updated_asc", + "name": "UPDATED_ASC", "description": "Updated at ascending order", "isDeprecated": false, "deprecationReason": null }, { - "name": "created_desc", + "name": "CREATED_DESC", "description": "Created at descending order", "isDeprecated": false, "deprecationReason": null }, { - "name": "created_asc", + "name": "CREATED_ASC", "description": "Created at ascending order", "isDeprecated": false, "deprecationReason": null @@ -23937,6 +26981,18 @@ "deprecationReason": null }, { + "name": "SEVERITY_ASC", + "description": "Severity from less critical to more critical", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "SEVERITY_DESC", + "description": "Severity from more critical to less critical", + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "WEIGHT_ASC", "description": "Weight by ascending order", "isDeprecated": false, @@ -23947,6 +27003,18 @@ "description": "Weight by descending order", "isDeprecated": false, "deprecationReason": null + }, + { + "name": "PUBLISHED_ASC", + "description": "Published issues shown last", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "PUBLISHED_DESC", + "description": "Published issues shown first", + "isDeprecated": false, + "deprecationReason": null } ], "possibleTypes": null @@ -23987,6 +27055,29 @@ "possibleTypes": null }, { + "kind": "ENUM", + "name": "IssueStateEvent", + "description": "Values for issue state events", + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": [ + { + "name": "REOPEN", + "description": "Reopens the issue", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "CLOSE", + "description": "Closes the issue", + "isDeprecated": false, + "deprecationReason": null + } + ], + "possibleTypes": null + }, + { "kind": "OBJECT", "name": "IssueStatusCountsType", "description": "Represents total number of issues for the represented statuses", @@ -25555,6 +28646,24 @@ "description": "The connection type for Label.", "fields": [ { + "name": "count", + "description": "Total count of collection", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "edges", "description": "A list of edges.", "args": [ @@ -25672,6 +28781,16 @@ "possibleTypes": null }, { + "kind": "SCALAR", + "name": "ListID", + "description": "Identifier of List", + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { "kind": "ENUM", "name": "ListLimitMetric", "description": "List limit metric setting", @@ -25845,6 +28964,30 @@ "description": "Pipeline count", "isDeprecated": false, "deprecationReason": null + }, + { + "name": "PIPELINES_SUCCEEDED", + "description": "Pipeline count with success status", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "PIPELINES_FAILED", + "description": "Pipeline count with failed status", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "PIPELINES_CANCELED", + "description": "Pipeline count with canceled status", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "PIPELINES_SKIPPED", + "description": "Pipeline count with skipped status", + "isDeprecated": false, + "deprecationReason": null } ], "possibleTypes": null @@ -27832,6 +30975,251 @@ }, { "kind": "OBJECT", + "name": "MergeRequestDiffRegistry", + "description": "Represents the Geo sync and verification state of a Merge Request diff", + "fields": [ + { + "name": "createdAt", + "description": "Timestamp when the MergeRequestDiffRegistry was created", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "id", + "description": "ID of the MergeRequestDiffRegistry", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "lastSyncFailure", + "description": "Error message during sync of the MergeRequestDiffRegistry", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "lastSyncedAt", + "description": "Timestamp of the most recent successful sync of the MergeRequestDiffRegistry", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "mergeRequestDiffId", + "description": "ID of the Merge Request diff", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "retryAt", + "description": "Timestamp after which the MergeRequestDiffRegistry should be resynced", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "retryCount", + "description": "Number of consecutive failed sync attempts of the MergeRequestDiffRegistry", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "state", + "description": "Sync state of the MergeRequestDiffRegistry", + "args": [ + + ], + "type": { + "kind": "ENUM", + "name": "RegistryState", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "MergeRequestDiffRegistryConnection", + "description": "The connection type for MergeRequestDiffRegistry.", + "fields": [ + { + "name": "edges", + "description": "A list of edges.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "MergeRequestDiffRegistryEdge", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "nodes", + "description": "A list of nodes.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "MergeRequestDiffRegistry", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "pageInfo", + "description": "Information to aid in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "PageInfo", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "MergeRequestDiffRegistryEdge", + "description": "An edge in a connection.", + "fields": [ + { + "name": "cursor", + "description": "A cursor for use in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "node", + "description": "The item at the end of the edge.", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "MergeRequestDiffRegistry", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", "name": "MergeRequestEdge", "description": "An edge in a connection.", "fields": [ @@ -27876,6 +31264,16 @@ "possibleTypes": null }, { + "kind": "SCALAR", + "name": "MergeRequestID", + "description": "Identifier of MergeRequest", + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { "kind": "OBJECT", "name": "MergeRequestPermissions", "description": "Check permissions for the current user on a merge request", @@ -28515,7 +31913,7 @@ "description": "The milestone to assign to the merge request.\n", "type": { "kind": "SCALAR", - "name": "ID", + "name": "MilestoneID", "ofType": null }, "defaultValue": null @@ -28873,23 +32271,47 @@ { "name": "updated_desc", "description": "Updated at descending order", + "isDeprecated": true, + "deprecationReason": "Use UPDATED_DESC. Deprecated in 13.5" + }, + { + "name": "updated_asc", + "description": "Updated at ascending order", + "isDeprecated": true, + "deprecationReason": "Use UPDATED_ASC. Deprecated in 13.5" + }, + { + "name": "created_desc", + "description": "Created at descending order", + "isDeprecated": true, + "deprecationReason": "Use CREATED_DESC. Deprecated in 13.5" + }, + { + "name": "created_asc", + "description": "Created at ascending order", + "isDeprecated": true, + "deprecationReason": "Use CREATED_ASC. Deprecated in 13.5" + }, + { + "name": "UPDATED_DESC", + "description": "Updated at descending order", "isDeprecated": false, "deprecationReason": null }, { - "name": "updated_asc", + "name": "UPDATED_ASC", "description": "Updated at ascending order", "isDeprecated": false, "deprecationReason": null }, { - "name": "created_desc", + "name": "CREATED_DESC", "description": "Created at descending order", "isDeprecated": false, "deprecationReason": null }, { - "name": "created_asc", + "name": "CREATED_ASC", "description": "Created at ascending order", "isDeprecated": false, "deprecationReason": null @@ -30437,6 +33859,33 @@ "deprecationReason": null }, { + "name": "createBoard", + "description": null, + "args": [ + { + "name": "input", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "CreateBoardInput", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "CreateBoardPayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "createBranch", "description": null, "args": [ @@ -30572,6 +34021,33 @@ "deprecationReason": null }, { + "name": "createIssue", + "description": null, + "args": [ + { + "name": "input", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "CreateIssueInput", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "CreateIssuePayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "createIteration", "description": null, "args": [ @@ -30896,6 +34372,33 @@ "deprecationReason": null }, { + "name": "dastSiteTokenCreate", + "description": null, + "args": [ + { + "name": "input", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "DastSiteTokenCreateInput", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "DastSiteTokenCreatePayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "deleteAnnotation", "description": null, "args": [ @@ -31031,6 +34534,33 @@ "deprecationReason": null }, { + "name": "destroyBoardList", + "description": null, + "args": [ + { + "name": "input", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "DestroyBoardListInput", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "DestroyBoardListPayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "destroyNote", "description": null, "args": [ @@ -31135,8 +34665,8 @@ "name": "DismissVulnerabilityPayload", "ofType": null }, - "isDeprecated": false, - "deprecationReason": null + "isDeprecated": true, + "deprecationReason": "Use vulnerabilityDismiss. Deprecated in 13.5" }, { "name": "epicAddIssue", @@ -31220,6 +34750,33 @@ "deprecationReason": null }, { + "name": "issueMove", + "description": null, + "args": [ + { + "name": "input", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "IssueMoveInput", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "IssueMovePayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "issueMoveList", "description": null, "args": [ @@ -31949,6 +35506,33 @@ "deprecationReason": null }, { + "name": "revertVulnerabilityToDetected", + "description": null, + "args": [ + { + "name": "input", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "RevertVulnerabilityToDetectedInput", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "RevertVulnerabilityToDetectedPayload", + "ofType": null + }, + "isDeprecated": true, + "deprecationReason": "Use vulnerabilityRevertToDetected. Deprecated in 13.5" + }, + { "name": "runDastScan", "description": null, "args": [ @@ -32165,6 +35749,33 @@ "deprecationReason": null }, { + "name": "updateBoardEpicUserPreferences", + "description": null, + "args": [ + { + "name": "input", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "UpdateBoardEpicUserPreferencesInput", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "UpdateBoardEpicUserPreferencesPayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "updateBoardList", "description": null, "args": [ @@ -32408,6 +36019,60 @@ "deprecationReason": null }, { + "name": "vulnerabilityConfirm", + "description": null, + "args": [ + { + "name": "input", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "VulnerabilityConfirmInput", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "VulnerabilityConfirmPayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "vulnerabilityDismiss", + "description": null, + "args": [ + { + "name": "input", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "VulnerabilityDismissInput", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "VulnerabilityDismissPayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "vulnerabilityResolve", "description": null, "args": [ @@ -32433,6 +36098,33 @@ }, "isDeprecated": false, "deprecationReason": null + }, + { + "name": "vulnerabilityRevertToDetected", + "description": null, + "args": [ + { + "name": "input", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "VulnerabilityRevertToDetectedInput", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "VulnerabilityRevertToDetectedPayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null } ], "inputFields": null, @@ -32477,6 +36169,52 @@ "description": null, "fields": [ { + "name": "actualRepositorySizeLimit", + "description": "Size limit for repositories in the namespace in bytes", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Float", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "additionalPurchasedStorageSize", + "description": "Additional storage purchased for the root namespace in bytes", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Float", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "containsLockedProjects", + "description": "Includes at least one project where the repository size exceeds the limit", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "description", "description": "Description of the namespace", "args": [ @@ -32724,6 +36462,24 @@ "deprecationReason": null }, { + "name": "repositorySizeExcessProjectCount", + "description": "Number of projects in the root namespace where the repository size exceeds the limit", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "requestAccessEnabled", "description": "Indicates if users can request access to namespace", "args": [ @@ -32780,6 +36536,34 @@ "deprecationReason": null }, { + "name": "totalRepositorySize", + "description": "Total repository size of all projects in the root namespace in bytes", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Float", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "totalRepositorySizeExcess", + "description": "Total excess repository size of all projects in the root namespace in bytes", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Float", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "visibility", "description": "Visibility of the namespace", "args": [ @@ -32914,6 +36698,16 @@ "possibleTypes": null }, { + "kind": "SCALAR", + "name": "NamespaceID", + "description": "Identifier of Namespace", + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { "kind": "INPUT_OBJECT", "name": "NamespaceIncreaseStorageTemporarilyInput", "description": "Autogenerated input type of NamespaceIncreaseStorageTemporarily", @@ -32927,7 +36721,7 @@ "name": null, "ofType": { "kind": "SCALAR", - "name": "ID", + "name": "NamespaceID", "ofType": null } }, @@ -33535,6 +37329,16 @@ "possibleTypes": null }, { + "kind": "SCALAR", + "name": "NoteID", + "description": "Identifier of Note", + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { "kind": "OBJECT", "name": "NotePermissions", "description": null, @@ -33768,6 +37572,11 @@ }, { "kind": "OBJECT", + "name": "BoardEpic", + "ofType": null + }, + { + "kind": "OBJECT", "name": "Design", "ofType": null }, @@ -33795,10 +37604,25 @@ "kind": "OBJECT", "name": "Snippet", "ofType": null + }, + { + "kind": "OBJECT", + "name": "Vulnerability", + "ofType": null } ] }, { + "kind": "SCALAR", + "name": "NoteableID", + "description": "Identifier of Noteable", + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { "kind": "OBJECT", "name": "Package", "description": "Represents a package", @@ -34030,7 +37854,7 @@ { "kind": "OBJECT", "name": "PackageFileRegistry", - "description": "Represents the sync and verification state of a package file", + "description": "Represents the Geo sync and verification state of a package file", "fields": [ { "name": "createdAt", @@ -34282,43 +38106,55 @@ "enumValues": [ { "name": "MAVEN", - "description": "Packages from the maven package manager", + "description": "Packages from the Maven package manager", "isDeprecated": false, "deprecationReason": null }, { "name": "NPM", - "description": "Packages from the npm package manager", + "description": "Packages from the NPM package manager", "isDeprecated": false, "deprecationReason": null }, { "name": "CONAN", - "description": "Packages from the conan package manager", + "description": "Packages from the Conan package manager", "isDeprecated": false, "deprecationReason": null }, { "name": "NUGET", - "description": "Packages from the nuget package manager", + "description": "Packages from the Nuget package manager", "isDeprecated": false, "deprecationReason": null }, { "name": "PYPI", - "description": "Packages from the pypi package manager", + "description": "Packages from the PyPI package manager", "isDeprecated": false, "deprecationReason": null }, { "name": "COMPOSER", - "description": "Packages from the composer package manager", + "description": "Packages from the Composer package manager", "isDeprecated": false, "deprecationReason": null }, { "name": "GENERIC", - "description": "Packages from the generic package manager", + "description": "Packages from the Generic package manager", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "GOLANG", + "description": "Packages from the Golang package manager", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "DEBIAN", + "description": "Packages from the Debian package manager", "isDeprecated": false, "deprecationReason": null } @@ -35391,6 +39227,20 @@ "description": null, "fields": [ { + "name": "actualRepositorySizeLimit", + "description": "Size limit for the repository in bytes", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Float", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "alertManagementAlert", "description": "A single Alert Management alert of the project", "args": [ @@ -35441,6 +39291,16 @@ "ofType": null }, "defaultValue": null + }, + { + "name": "assigneeUsername", + "description": "Username of a user assigned to the issue", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null } ], "type": { @@ -35464,6 +39324,16 @@ "ofType": null }, "defaultValue": null + }, + { + "name": "assigneeUsername", + "description": "Username of a user assigned to the issue", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null } ], "type": { @@ -35527,6 +39397,16 @@ "defaultValue": null }, { + "name": "assigneeUsername", + "description": "Username of a user assigned to the issue", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { "name": "after", "description": "Returns the elements in the list that come after the specified cursor.", "type": { @@ -35637,11 +39517,15 @@ "args": [ { "name": "id", - "description": "Find a board by its ID", + "description": "The board's ID", "type": { - "kind": "SCALAR", - "name": "ID", - "ofType": null + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "BoardID", + "ofType": null + } }, "defaultValue": null } @@ -36366,6 +40250,16 @@ "defaultValue": null }, { + "name": "authorUsername", + "description": "Username of the author of the issue", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { "name": "assigneeUsername", "description": "Username of a user assigned to the issue", "type": { @@ -36376,6 +40270,24 @@ "defaultValue": null }, { + "name": "assigneeUsernames", + "description": "Usernames of users assigned to the issue", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + }, + "defaultValue": null + }, + { "name": "assigneeId", "description": "ID of a user assigned to the issues, \"none\" and \"any\" values supported", "type": { @@ -36577,6 +40489,16 @@ "defaultValue": null }, { + "name": "authorUsername", + "description": "Username of the author of the issue", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { "name": "assigneeUsername", "description": "Username of a user assigned to the issue", "type": { @@ -36587,6 +40509,24 @@ "defaultValue": null }, { + "name": "assigneeUsernames", + "description": "Usernames of users assigned to the issue", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + }, + "defaultValue": null + }, + { "name": "assigneeId", "description": "ID of a user assigned to the issues, \"none\" and \"any\" values supported", "type": { @@ -36754,6 +40694,16 @@ "defaultValue": null }, { + "name": "authorUsername", + "description": "Username of the author of the issue", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { "name": "assigneeUsername", "description": "Username of a user assigned to the issue", "type": { @@ -36764,6 +40714,24 @@ "defaultValue": null }, { + "name": "assigneeUsernames", + "description": "Usernames of users assigned to the issue", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + }, + "defaultValue": null + }, + { "name": "assigneeId", "description": "ID of a user assigned to the issues, \"none\" and \"any\" values supported", "type": { @@ -36964,7 +40932,7 @@ "args": [ { "name": "startDate", - "description": "List items within a time frame where items.start_date is between startDate and endDate parameters (endDate parameter must be present)", + "description": "List items overlapping a time frame defined by startDate..endDate (if one date is provided, both must be present). Deprecated in 13.5: Use timeframe.start", "type": { "kind": "SCALAR", "name": "Time", @@ -36974,7 +40942,7 @@ }, { "name": "endDate", - "description": "List items within a time frame where items.end_date is between startDate and endDate parameters (startDate parameter must be present)", + "description": "List items overlapping a time frame defined by startDate..endDate (if one date is provided, both must be present). Deprecated in 13.5: Use timeframe.end", "type": { "kind": "SCALAR", "name": "Time", @@ -36983,6 +40951,16 @@ "defaultValue": null }, { + "name": "timeframe", + "description": "List items overlapping the given timeframe", + "type": { + "kind": "INPUT_OBJECT", + "name": "Timeframe", + "ofType": null + }, + "defaultValue": null + }, + { "name": "state", "description": "Filter iterations by state", "type": { @@ -37536,7 +41514,7 @@ "args": [ { "name": "startDate", - "description": "List items within a time frame where items.start_date is between startDate and endDate parameters (endDate parameter must be present)", + "description": "List items overlapping a time frame defined by startDate..endDate (if one date is provided, both must be present). Deprecated in 13.5: Use timeframe.start", "type": { "kind": "SCALAR", "name": "Time", @@ -37546,7 +41524,7 @@ }, { "name": "endDate", - "description": "List items within a time frame where items.end_date is between startDate and endDate parameters (startDate parameter must be present)", + "description": "List items overlapping a time frame defined by startDate..endDate (if one date is provided, both must be present). Deprecated in 13.5: Use timeframe.end", "type": { "kind": "SCALAR", "name": "Time", @@ -37555,6 +41533,16 @@ "defaultValue": null }, { + "name": "timeframe", + "description": "List items overlapping the given timeframe", + "type": { + "kind": "INPUT_OBJECT", + "name": "Timeframe", + "ofType": null + }, + "defaultValue": null + }, + { "name": "ids", "description": "Array of global milestone IDs, e.g., \"gid://gitlab/Milestone/1\"", "type": { @@ -37583,6 +41571,36 @@ "defaultValue": null }, { + "name": "title", + "description": "The title of the milestone", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "searchTitle", + "description": "A search string for the title", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "containingDate", + "description": "A date that the milestone contains", + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "defaultValue": null + }, + { "name": "includeAncestors", "description": "Also return milestones in the project's parent group and its ancestors", "type": { @@ -38114,6 +42132,20 @@ "deprecationReason": null }, { + "name": "repositorySizeExcess", + "description": "Size of repository that exceeds the limit in bytes", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Float", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "requestAccessEnabled", "description": "Indicates if users can request member access to the project", "args": [ @@ -38129,7 +42161,7 @@ }, { "name": "requirement", - "description": "Find a single requirement. Available only when feature flag `requirements_management` is enabled.", + "description": "Find a single requirement", "args": [ { "name": "iid", @@ -38232,7 +42264,7 @@ }, { "name": "requirements", - "description": "Find requirements. Available only when feature flag `requirements_management` is enabled.", + "description": "Find requirements", "args": [ { "name": "iid", @@ -38727,6 +42759,59 @@ "deprecationReason": null }, { + "name": "terraformStates", + "description": "Terraform states associated with the project", + "args": [ + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "TerraformStateConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "userPermissions", "description": "Permissions for the current user on the resource", "args": [ @@ -39317,6 +43402,16 @@ "possibleTypes": null }, { + "kind": "SCALAR", + "name": "ProjectID", + "description": "Identifier of Project", + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { "kind": "OBJECT", "name": "ProjectMember", "description": "Represents a Project Membership", @@ -40982,6 +45077,26 @@ "defaultValue": null }, { + "name": "searchNamespaces", + "description": "Include namespace in project search", + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "sort", + "description": "Sort order of results", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { "name": "after", "description": "Returns the elements in the list that come after the specified cursor.", "type": { @@ -41031,6 +45146,59 @@ "deprecationReason": null }, { + "name": "runnerPlatforms", + "description": "Supported runner platforms", + "args": [ + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "RunnerPlatformConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "snippets", "description": "Find Snippets visible to the current user", "args": [ @@ -41617,6 +45785,33 @@ }, "isDeprecated": true, "deprecationReason": "Use `vulnerabilitiesCountByDay`. Deprecated in 13.3" + }, + { + "name": "vulnerability", + "description": "Find a vulnerability", + "args": [ + { + "name": "id", + "description": "The Global ID of the Vulnerability", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "VulnerabilityID", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "Vulnerability", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null } ], "inputFields": null, @@ -42880,7 +47075,7 @@ "name": null, "ofType": { "kind": "SCALAR", - "name": "ID", + "name": "AwardableID", "ofType": null } }, @@ -42996,7 +47191,7 @@ "name": null, "ofType": { "kind": "SCALAR", - "name": "ID", + "name": "ProjectID", "ofType": null } }, @@ -43218,6 +47413,34 @@ "deprecationReason": null }, { + "name": "description", + "description": "Description of the requirement", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "descriptionHtml", + "description": "The GitLab Flavored Markdown rendering of `description`", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "id", "description": "ID of the requirement", "args": [ @@ -43254,6 +47477,20 @@ "deprecationReason": null }, { + "name": "lastTestReportManuallyCreated", + "description": "Indicates if latest test report was created by user", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "lastTestReportState", "description": "Latest requirement test report state", "args": [ @@ -43381,6 +47618,20 @@ "deprecationReason": null }, { + "name": "titleHtml", + "description": "The GitLab Flavored Markdown rendering of `title`", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "updatedAt", "description": "Timestamp of when the requirement was last updated", "args": [ @@ -43790,6 +48041,108 @@ ] }, { + "kind": "INPUT_OBJECT", + "name": "RevertVulnerabilityToDetectedInput", + "description": "Autogenerated input type of RevertVulnerabilityToDetected", + "fields": null, + "inputFields": [ + { + "name": "id", + "description": "ID of the vulnerability to be reverted", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "VulnerabilityID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "RevertVulnerabilityToDetectedPayload", + "description": "Autogenerated return type of RevertVulnerabilityToDetected", + "fields": [ + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "errors", + "description": "Errors encountered during execution of the mutation.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "vulnerability", + "description": "The vulnerability after revert", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "Vulnerability", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { "kind": "OBJECT", "name": "RootStorageStatistics", "description": null, @@ -43849,6 +48202,24 @@ "deprecationReason": null }, { + "name": "pipelineArtifactsSize", + "description": "The CI pipeline artifacts size in bytes", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Float", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "repositorySize", "description": "The Git repository size in bytes", "args": [ @@ -44074,6 +48445,381 @@ }, { "kind": "OBJECT", + "name": "RunnerArchitecture", + "description": null, + "fields": [ + { + "name": "downloadLocation", + "description": "Download location for the runner for the platform architecture", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "name", + "description": "Name of the runner platform architecture", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "RunnerArchitectureConnection", + "description": "The connection type for RunnerArchitecture.", + "fields": [ + { + "name": "edges", + "description": "A list of edges.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "RunnerArchitectureEdge", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "nodes", + "description": "A list of nodes.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "RunnerArchitecture", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "pageInfo", + "description": "Information to aid in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "PageInfo", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "RunnerArchitectureEdge", + "description": "An edge in a connection.", + "fields": [ + { + "name": "cursor", + "description": "A cursor for use in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "node", + "description": "The item at the end of the edge.", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "RunnerArchitecture", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "RunnerPlatform", + "description": null, + "fields": [ + { + "name": "architectures", + "description": "Runner architectures supported for the platform", + "args": [ + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "RunnerArchitectureConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "humanReadableName", + "description": "Human readable name of the runner platform", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "name", + "description": "Name slug of the runner platform", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "RunnerPlatformConnection", + "description": "The connection type for RunnerPlatform.", + "fields": [ + { + "name": "edges", + "description": "A list of edges.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "RunnerPlatformEdge", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "nodes", + "description": "A list of nodes.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "RunnerPlatform", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "pageInfo", + "description": "Information to aid in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "PageInfo", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "RunnerPlatformEdge", + "description": "An edge in a connection.", + "fields": [ + { + "name": "cursor", + "description": "A cursor for use in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "node", + "description": "The item at the end of the edge.", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "RunnerPlatform", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", "name": "SastCiConfiguration", "description": "Represents a CI configuration of SAST", "fields": [ @@ -44479,6 +49225,63 @@ "possibleTypes": null }, { + "kind": "INPUT_OBJECT", + "name": "SastCiConfigurationAnalyzersEntityInput", + "description": "Represents the analyzers entity in SAST CI configuration", + "fields": null, + "inputFields": [ + { + "name": "name", + "description": "Name of analyzer", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "enabled", + "description": "State of the analyzer", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "variables", + "description": "List of variables for the analyzer", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "SastCiConfigurationEntityInput", + "ofType": null + } + } + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { "kind": "OBJECT", "name": "SastCiConfigurationEntity", "description": "Represents an entity in SAST CI configuration", @@ -44848,6 +49651,24 @@ } }, "defaultValue": null + }, + { + "name": "analyzers", + "description": "List of analyzers and related variables for the SAST configuration", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "SastCiConfigurationAnalyzersEntityInput", + "ofType": null + } + } + }, + "defaultValue": null } ], "interfaces": null, @@ -45195,6 +50016,20 @@ "description": "Represents summary of a security report", "fields": [ { + "name": "apiFuzzing", + "description": "Aggregated counts for the api_fuzzing scan", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "SecurityReportSummarySection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "containerScanning", "description": "Aggregated counts for the container_scanning scan", "args": [ @@ -45437,6 +50272,12 @@ "description": null, "isDeprecated": false, "deprecationReason": null + }, + { + "name": "API_FUZZING", + "description": null, + "isDeprecated": false, + "deprecationReason": null } ], "possibleTypes": null @@ -47364,24 +52205,69 @@ "name": "blobs", "description": "Snippet blobs", "args": [ - - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", + { + "name": "paths", + "description": "Paths of the blobs", + "type": { + "kind": "LIST", "name": null, "ofType": { - "kind": "OBJECT", - "name": "SnippetBlob", - "ofType": null + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } } - } + }, + "defaultValue": null + }, + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null } + ], + "type": { + "kind": "OBJECT", + "name": "SnippetBlobConnection", + "ofType": null }, "isDeprecated": false, "deprecationReason": null @@ -48037,6 +52923,118 @@ }, { "kind": "OBJECT", + "name": "SnippetBlobConnection", + "description": "The connection type for SnippetBlob.", + "fields": [ + { + "name": "edges", + "description": "A list of edges.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "SnippetBlobEdge", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "nodes", + "description": "A list of nodes.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "SnippetBlob", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "pageInfo", + "description": "Information to aid in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "PageInfo", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "SnippetBlobEdge", + "description": "An edge in a connection.", + "fields": [ + { + "name": "cursor", + "description": "A cursor for use in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "node", + "description": "The item at the end of the edge.", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "SnippetBlob", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", "name": "SnippetBlobViewer", "description": "Represents how the blob content should be displayed", "fields": [ @@ -48414,23 +53412,47 @@ { "name": "updated_desc", "description": "Updated at descending order", + "isDeprecated": true, + "deprecationReason": "Use UPDATED_DESC. Deprecated in 13.5" + }, + { + "name": "updated_asc", + "description": "Updated at ascending order", + "isDeprecated": true, + "deprecationReason": "Use UPDATED_ASC. Deprecated in 13.5" + }, + { + "name": "created_desc", + "description": "Created at descending order", + "isDeprecated": true, + "deprecationReason": "Use CREATED_DESC. Deprecated in 13.5" + }, + { + "name": "created_asc", + "description": "Created at ascending order", + "isDeprecated": true, + "deprecationReason": "Use CREATED_ASC. Deprecated in 13.5" + }, + { + "name": "UPDATED_DESC", + "description": "Updated at descending order", "isDeprecated": false, "deprecationReason": null }, { - "name": "updated_asc", + "name": "UPDATED_ASC", "description": "Updated at ascending order", "isDeprecated": false, "deprecationReason": null }, { - "name": "created_desc", + "name": "CREATED_DESC", "description": "Created at descending order", "isDeprecated": false, "deprecationReason": null }, { - "name": "created_asc", + "name": "CREATED_ASC", "description": "Created at ascending order", "isDeprecated": false, "deprecationReason": null @@ -48439,6 +53461,89 @@ "possibleTypes": null }, { + "kind": "OBJECT", + "name": "StatusAction", + "description": null, + "fields": [ + { + "name": "buttonTitle", + "description": "Title for the button, for example: Retry this job", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "icon", + "description": "Icon used in the action button", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "method", + "description": "Method for the action, for example: :post", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "path", + "description": "Path for the action", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "title", + "description": "Title for the action, for example: Retry", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { "kind": "SCALAR", "name": "String", "description": "Represents textual data as UTF-8 character sequences. This type is most often used by GraphQL to represent free-form human-readable text.", @@ -48764,12 +53869,237 @@ }, { "kind": "OBJECT", - "name": "TerraformStateRegistry", - "description": "Represents the sync and verification state of a terraform state", + "name": "TerraformState", + "description": null, + "fields": [ + { + "name": "createdAt", + "description": "Timestamp the Terraform state was created", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "id", + "description": "ID of the Terraform state", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "lockedAt", + "description": "Timestamp the Terraform state was locked", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "lockedByUser", + "description": "The user currently holding a lock on the Terraform state", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "User", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "name", + "description": "Name of the Terraform state", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "updatedAt", + "description": "Timestamp the Terraform state was updated", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "TerraformStateConnection", + "description": "The connection type for TerraformState.", + "fields": [ + { + "name": "edges", + "description": "A list of edges.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "TerraformStateEdge", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "nodes", + "description": "A list of nodes.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "TerraformState", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "pageInfo", + "description": "Information to aid in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "PageInfo", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "TerraformStateEdge", + "description": "An edge in a connection.", + "fields": [ + { + "name": "cursor", + "description": "A cursor for use in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "node", + "description": "The item at the end of the edge.", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "TerraformState", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "TerraformStateVersionRegistry", + "description": "Represents the Geo sync and verification state of a terraform state version", "fields": [ { "name": "createdAt", - "description": "Timestamp when the TerraformStateRegistry was created", + "description": "Timestamp when the TerraformStateVersionRegistry was created", "args": [ ], @@ -48783,7 +54113,7 @@ }, { "name": "id", - "description": "ID of the TerraformStateRegistry", + "description": "ID of the TerraformStateVersionRegistry", "args": [ ], @@ -48801,7 +54131,7 @@ }, { "name": "lastSyncFailure", - "description": "Error message during sync of the TerraformStateRegistry", + "description": "Error message during sync of the TerraformStateVersionRegistry", "args": [ ], @@ -48815,7 +54145,7 @@ }, { "name": "lastSyncedAt", - "description": "Timestamp of the most recent successful sync of the TerraformStateRegistry", + "description": "Timestamp of the most recent successful sync of the TerraformStateVersionRegistry", "args": [ ], @@ -48829,7 +54159,7 @@ }, { "name": "retryAt", - "description": "Timestamp after which the TerraformStateRegistry should be resynced", + "description": "Timestamp after which the TerraformStateVersionRegistry should be resynced", "args": [ ], @@ -48843,7 +54173,7 @@ }, { "name": "retryCount", - "description": "Number of consecutive failed sync attempts of the TerraformStateRegistry", + "description": "Number of consecutive failed sync attempts of the TerraformStateVersionRegistry", "args": [ ], @@ -48857,7 +54187,7 @@ }, { "name": "state", - "description": "Sync state of the TerraformStateRegistry", + "description": "Sync state of the TerraformStateVersionRegistry", "args": [ ], @@ -48870,8 +54200,8 @@ "deprecationReason": null }, { - "name": "terraformStateId", - "description": "ID of the TerraformState", + "name": "terraformStateVersionId", + "description": "ID of the terraform state version", "args": [ ], @@ -48897,8 +54227,8 @@ }, { "kind": "OBJECT", - "name": "TerraformStateRegistryConnection", - "description": "The connection type for TerraformStateRegistry.", + "name": "TerraformStateVersionRegistryConnection", + "description": "The connection type for TerraformStateVersionRegistry.", "fields": [ { "name": "edges", @@ -48911,7 +54241,7 @@ "name": null, "ofType": { "kind": "OBJECT", - "name": "TerraformStateRegistryEdge", + "name": "TerraformStateVersionRegistryEdge", "ofType": null } }, @@ -48929,7 +54259,7 @@ "name": null, "ofType": { "kind": "OBJECT", - "name": "TerraformStateRegistry", + "name": "TerraformStateVersionRegistry", "ofType": null } }, @@ -48964,7 +54294,7 @@ }, { "kind": "OBJECT", - "name": "TerraformStateRegistryEdge", + "name": "TerraformStateVersionRegistryEdge", "description": "An edge in a connection.", "fields": [ { @@ -48993,7 +54323,7 @@ ], "type": { "kind": "OBJECT", - "name": "TerraformStateRegistry", + "name": "TerraformStateVersionRegistry", "ofType": null }, "isDeprecated": false, @@ -49278,6 +54608,45 @@ ] }, { + "kind": "INPUT_OBJECT", + "name": "Timeframe", + "description": "A time-frame defined as a closed inclusive range of two dates", + "fields": null, + "inputFields": [ + { + "name": "start", + "description": "The start of the range", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Date", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "end", + "description": "The end of the range", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Date", + "ofType": null + } + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { "kind": "OBJECT", "name": "Timelog", "description": null, @@ -49831,6 +55200,16 @@ "possibleTypes": null }, { + "kind": "SCALAR", + "name": "TodoID", + "description": "Identifier of Todo", + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { "kind": "INPUT_OBJECT", "name": "TodoMarkDoneInput", "description": "Autogenerated input type of TodoMarkDone", @@ -49844,7 +55223,7 @@ "name": null, "ofType": { "kind": "SCALAR", - "name": "ID", + "name": "TodoID", "ofType": null } }, @@ -49950,7 +55329,7 @@ "name": null, "ofType": { "kind": "SCALAR", - "name": "ID", + "name": "TodoID", "ofType": null } }, @@ -49991,7 +55370,7 @@ "name": null, "ofType": { "kind": "SCALAR", - "name": "ID", + "name": "TodoID", "ofType": null } } @@ -50400,7 +55779,7 @@ "name": null, "ofType": { "kind": "SCALAR", - "name": "ID", + "name": "AwardableID", "ofType": null } }, @@ -51176,6 +56555,136 @@ }, { "kind": "INPUT_OBJECT", + "name": "UpdateBoardEpicUserPreferencesInput", + "description": "Autogenerated input type of UpdateBoardEpicUserPreferences", + "fields": null, + "inputFields": [ + { + "name": "boardId", + "description": "The board global ID", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "BoardID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "epicId", + "description": "ID of an epic to set preferences for", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "EpicID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "collapsed", + "description": "Whether the epic should be collapsed in the board", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "UpdateBoardEpicUserPreferencesPayload", + "description": "Autogenerated return type of UpdateBoardEpicUserPreferences", + "fields": [ + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "epicUserPreferences", + "description": "User preferences for the epic in the board after mutation", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "BoardEpicUserPreferences", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "errors", + "description": "Errors encountered during execution of the mutation.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "INPUT_OBJECT", "name": "UpdateBoardInput", "description": "Autogenerated input type of UpdateBoard", "fields": null, @@ -51188,7 +56697,7 @@ "name": null, "ofType": { "kind": "SCALAR", - "name": "ID", + "name": "BoardID", "ofType": null } }, @@ -51229,7 +56738,7 @@ "description": "The id of user to be assigned to the board.", "type": { "kind": "SCALAR", - "name": "ID", + "name": "UserID", "ofType": null }, "defaultValue": null @@ -51239,7 +56748,7 @@ "description": "The id of milestone to be assigned to the board.", "type": { "kind": "SCALAR", - "name": "ID", + "name": "MilestoneID", "ofType": null }, "defaultValue": null @@ -51255,6 +56764,42 @@ "defaultValue": null }, { + "name": "labels", + "description": "Labels of the issue", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + }, + "defaultValue": null + }, + { + "name": "labelIds", + "description": "The IDs of labels to be added to the board.", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "LabelID", + "ofType": null + } + } + }, + "defaultValue": null + }, + { "name": "clientMutationId", "description": "A unique identifier for the client performing the mutation.", "type": { @@ -51917,7 +57462,7 @@ "name": null, "ofType": { "kind": "SCALAR", - "name": "ID", + "name": "NoteID", "ofType": null } }, @@ -52060,16 +57605,6 @@ "defaultValue": null }, { - "name": "title", - "description": "Title of the issue", - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "defaultValue": null - }, - { "name": "description", "description": "Description of the issue", "type": { @@ -52084,7 +57619,7 @@ "description": "Due date of the issue", "type": { "kind": "SCALAR", - "name": "Time", + "name": "ISO8601Date", "ofType": null }, "defaultValue": null @@ -52110,8 +57645,28 @@ "defaultValue": null }, { + "name": "title", + "description": "Title of the issue", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "milestoneId", + "description": "The ID of the milestone to assign to the issue. On update milestone will be removed if set to null", + "type": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + }, + "defaultValue": null + }, + { "name": "addLabelIds", - "description": "The IDs of labels to be added to the issue.", + "description": "The IDs of labels to be added to the issue", "type": { "kind": "LIST", "name": null, @@ -52129,7 +57684,7 @@ }, { "name": "removeLabelIds", - "description": "The IDs of labels to be removed from the issue.", + "description": "The IDs of labels to be removed from the issue", "type": { "kind": "LIST", "name": null, @@ -52146,11 +57701,11 @@ "defaultValue": null }, { - "name": "milestoneId", - "description": "The ID of the milestone to be assigned, milestone will be removed if set to null.", + "name": "stateEvent", + "description": "Close or reopen an issue", "type": { - "kind": "SCALAR", - "name": "ID", + "kind": "ENUM", + "name": "IssueStateEvent", "ofType": null }, "defaultValue": null @@ -52166,6 +57721,16 @@ "defaultValue": null }, { + "name": "weight", + "description": "The weight of the issue", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { "name": "epicId", "description": "The ID of the parent epic. NULL when removing the association", "type": { @@ -52427,7 +57992,7 @@ "name": null, "ofType": { "kind": "SCALAR", - "name": "ID", + "name": "NoteID", "ofType": null } }, @@ -52552,38 +58117,48 @@ "defaultValue": null }, { - "name": "state", - "description": "State of the requirement", + "name": "description", + "description": "Description of the requirement", "type": { - "kind": "ENUM", - "name": "RequirementState", + "kind": "SCALAR", + "name": "String", "ofType": null }, "defaultValue": null }, { - "name": "iid", - "description": "The iid of the requirement to update", + "name": "projectPath", + "description": "Full project path the requirement is associated with", "type": { "kind": "NON_NULL", "name": null, "ofType": { "kind": "SCALAR", - "name": "String", + "name": "ID", "ofType": null } }, "defaultValue": null }, { - "name": "projectPath", - "description": "The project full path the requirement is associated with", + "name": "state", + "description": "State of the requirement", + "type": { + "kind": "ENUM", + "name": "RequirementState", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "iid", + "description": "The iid of the requirement to update", "type": { "kind": "NON_NULL", "name": null, "ofType": { "kind": "SCALAR", - "name": "ID", + "name": "String", "ofType": null } }, @@ -52661,7 +58236,7 @@ }, { "name": "requirement", - "description": "The requirement after mutation", + "description": "Requirement after mutation", "args": [ ], @@ -52822,6 +58397,20 @@ }, "isDeprecated": false, "deprecationReason": null + }, + { + "name": "spam", + "description": "Indicates whether the operation returns a record detected as spam", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null } ], "inputFields": null, @@ -52993,6 +58582,16 @@ "defaultValue": null }, { + "name": "authorUsername", + "description": "Username of the author", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { "name": "after", "description": "Returns the elements in the list that come after the specified cursor.", "type": { @@ -53188,6 +58787,16 @@ "defaultValue": null }, { + "name": "assigneeUsername", + "description": "Username of the assignee", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { "name": "after", "description": "Returns the elements in the list that come after the specified cursor.", "type": { @@ -54606,6 +60215,63 @@ "deprecationReason": null }, { + "name": "discussions", + "description": "All discussions on this noteable", + "args": [ + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "DiscussionConnection", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "id", "description": "GraphQL ID of the vulnerability", "args": [ @@ -54731,6 +60397,63 @@ "deprecationReason": null }, { + "name": "notes", + "description": "All notes on this noteable", + "args": [ + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "NoteConnection", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "primaryIdentifier", "description": "Primary identifier of the vulnerability.", "args": [ @@ -54760,7 +60483,7 @@ }, { "name": "reportType", - "description": "Type of the security report that found the vulnerability (SAST, DEPENDENCY_SCANNING, CONTAINER_SCANNING, DAST, SECRET_DETECTION, COVERAGE_FUZZING)", + "description": "Type of the security report that found the vulnerability (SAST, DEPENDENCY_SCANNING, CONTAINER_SCANNING, DAST, SECRET_DETECTION, COVERAGE_FUZZING, API_FUZZING)", "args": [ ], @@ -54820,7 +60543,7 @@ }, { "name": "state", - "description": "State of the vulnerability (DETECTED, DISMISSED, RESOLVED, CONFIRMED)", + "description": "State of the vulnerability (DETECTED, CONFIRMED, RESOLVED, DISMISSED)", "args": [ ], @@ -54899,6 +60622,112 @@ ], "inputFields": null, "interfaces": [ + { + "kind": "INTERFACE", + "name": "Noteable", + "ofType": null + } + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "INPUT_OBJECT", + "name": "VulnerabilityConfirmInput", + "description": "Autogenerated input type of VulnerabilityConfirm", + "fields": null, + "inputFields": [ + { + "name": "id", + "description": "ID of the vulnerability to be confirmed", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "VulnerabilityID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "VulnerabilityConfirmPayload", + "description": "Autogenerated return type of VulnerabilityConfirm", + "fields": [ + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "errors", + "description": "Errors encountered during execution of the mutation.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "vulnerability", + "description": "The vulnerability after state change", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "Vulnerability", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ ], "enumValues": null, @@ -54972,6 +60801,118 @@ "possibleTypes": null }, { + "kind": "INPUT_OBJECT", + "name": "VulnerabilityDismissInput", + "description": "Autogenerated input type of VulnerabilityDismiss", + "fields": null, + "inputFields": [ + { + "name": "id", + "description": "ID of the vulnerability to be dismissed", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "VulnerabilityID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "comment", + "description": "Reason why vulnerability should be dismissed", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "VulnerabilityDismissPayload", + "description": "Autogenerated return type of VulnerabilityDismiss", + "fields": [ + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "errors", + "description": "Errors encountered during execution of the mutation.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "vulnerability", + "description": "The vulnerability after dismissal", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "Vulnerability", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { "kind": "OBJECT", "name": "VulnerabilityEdge", "description": "An edge in a connection.", @@ -55993,6 +61934,12 @@ "description": null, "isDeprecated": false, "deprecationReason": null + }, + { + "name": "API_FUZZING", + "description": null, + "isDeprecated": false, + "deprecationReason": null } ], "possibleTypes": null @@ -56005,7 +61952,7 @@ "inputFields": [ { "name": "id", - "description": "ID of the vulnerability to be resolveed", + "description": "ID of the vulnerability to be resolved", "type": { "kind": "NON_NULL", "name": null, @@ -56100,6 +62047,108 @@ "possibleTypes": null }, { + "kind": "INPUT_OBJECT", + "name": "VulnerabilityRevertToDetectedInput", + "description": "Autogenerated input type of VulnerabilityRevertToDetected", + "fields": null, + "inputFields": [ + { + "name": "id", + "description": "ID of the vulnerability to be reverted", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "VulnerabilityID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "VulnerabilityRevertToDetectedPayload", + "description": "Autogenerated return type of VulnerabilityRevertToDetected", + "fields": [ + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "errors", + "description": "Errors encountered during execution of the mutation.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "vulnerability", + "description": "The vulnerability after revert", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "Vulnerability", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { "kind": "OBJECT", "name": "VulnerabilityScanner", "description": "Represents a vulnerability scanner", @@ -56443,6 +62492,54 @@ "description": "Severity in ascending order", "isDeprecated": false, "deprecationReason": null + }, + { + "name": "title_desc", + "description": "Title in descending order", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "title_asc", + "description": "Title in ascending order", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "detected_desc", + "description": "Detection timestamp in descending order", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "detected_asc", + "description": "Detection timestamp in ascending order", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "report_type_desc", + "description": "Report Type in descending order", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "report_type_asc", + "description": "Report Type in ascending order", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "state_desc", + "description": "State in descending order", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "state_asc", + "description": "State in ascending order", + "isDeprecated": false, + "deprecationReason": null } ], "possibleTypes": null @@ -56462,7 +62559,7 @@ "deprecationReason": null }, { - "name": "DISMISSED", + "name": "CONFIRMED", "description": null, "isDeprecated": false, "deprecationReason": null @@ -56474,7 +62571,7 @@ "deprecationReason": null }, { - "name": "CONFIRMED", + "name": "DISMISSED", "description": null, "isDeprecated": false, "deprecationReason": null diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md index fc27298aff2..dca00fc1286 100644 --- a/doc/api/graphql/reference/index.md +++ b/doc/api/graphql/reference/index.md @@ -77,6 +77,7 @@ Describes an alert from the project's Alert Management. | `details` | JSON | Alert details | | `detailsUrl` | String! | The URL of the alert detail page | | `endedAt` | Time | Timestamp the alert ended | +| `environment` | Environment | Environment for the alert | | `eventCount` | Int | Number of events of this alert | | `hosts` | String! => Array | List of hosts the alert came from | | `iid` | ID! | Internal ID of the alert | @@ -209,6 +210,57 @@ Represents a project or group board. | `name` | String | Name of the board | | `weight` | Int | Weight of the board. | +### BoardEpic + +Represents an epic on an issue board. + +| Field | Type | Description | +| ----- | ---- | ----------- | +| `author` | User! | Author of the epic | +| `closedAt` | Time | Timestamp of when the epic was closed | +| `confidential` | Boolean | Indicates if the epic is confidential | +| `createdAt` | Time | Timestamp of when the epic was created | +| `descendantCounts` | EpicDescendantCount | Number of open and closed descendant epics and issues | +| `descendantWeightSum` | EpicDescendantWeights | Total weight of open and closed issues in the epic and its descendants | +| `description` | String | Description of the epic | +| `downvotes` | Int! | Number of downvotes the epic has received | +| `dueDate` | Time | Due date of the epic | +| `dueDateFixed` | Time | Fixed due date of the epic | +| `dueDateFromMilestones` | Time | Inherited due date of the epic from milestones | +| `dueDateIsFixed` | Boolean | Indicates if the due date has been manually set | +| `group` | Group! | Group to which the epic belongs | +| `hasChildren` | Boolean! | Indicates if the epic has children | +| `hasIssues` | Boolean! | Indicates if the epic has direct issues | +| `hasParent` | Boolean! | Indicates if the epic has a parent epic | +| `healthStatus` | EpicHealthStatus | Current health status of the epic | +| `id` | ID! | ID of the epic | +| `iid` | ID! | Internal ID of the epic | +| `parent` | Epic | Parent epic of the epic | +| `reference` | String! | Internal reference of the epic. Returned in shortened format by default | +| `relationPath` | String | URI path of the epic-issue relationship | +| `relativePosition` | Int | The relative position of the epic in the epic tree | +| `startDate` | Time | Start date of the epic | +| `startDateFixed` | Time | Fixed start date of the epic | +| `startDateFromMilestones` | Time | Inherited start date of the epic from milestones | +| `startDateIsFixed` | Boolean | Indicates if the start date has been manually set | +| `state` | EpicState! | State of the epic | +| `subscribed` | Boolean! | Indicates the currently logged in user is subscribed to the epic | +| `title` | String | Title of the epic | +| `updatedAt` | Time | Timestamp of when the epic was updated | +| `upvotes` | Int! | Number of upvotes the epic has received | +| `userPermissions` | EpicPermissions! | Permissions for the current user on the resource | +| `userPreferences` | BoardEpicUserPreferences | User preferences for the epic on the issue board | +| `webPath` | String! | Web path of the epic | +| `webUrl` | String! | Web URL of the epic | + +### BoardEpicUserPreferences + +Represents user preferences for a board epic. + +| Field | Type | Description | +| ----- | ---- | ----------- | +| `collapsed` | Boolean! | Indicates epic should be displayed as collapsed | + ### BoardList Represents a list for an issue board. @@ -272,6 +324,7 @@ Represents the total number of issues and their weights for a particular day. | Field | Type | Description | | ----- | ---- | ----------- | +| `detailedStatus` | DetailedStatus | Detailed status of the group | | `name` | String | Name of the job group | | `size` | Int | Size of the group | @@ -279,12 +332,15 @@ Represents the total number of issues and their weights for a particular day. | Field | Type | Description | | ----- | ---- | ----------- | +| `detailedStatus` | DetailedStatus | Detailed status of the job | | `name` | String | Name of the job | +| `scheduledAt` | Time | Schedule for the build | ### CiStage | Field | Type | Description | | ----- | ---- | ----------- | +| `detailedStatus` | DetailedStatus | Detailed status of the stage | | `name` | String | Name of the stage | ### ClusterAgent @@ -421,6 +477,16 @@ Autogenerated return type of CreateAnnotation. | `clientMutationId` | String | A unique identifier for the client performing the mutation. | | `errors` | String! => Array | Errors encountered during execution of the mutation. | +### CreateBoardPayload + +Autogenerated return type of CreateBoard. + +| Field | Type | Description | +| ----- | ---- | ----------- | +| `board` | Board | The board after mutation. | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | + ### CreateBranchPayload Autogenerated return type of CreateBranch. @@ -471,6 +537,16 @@ Autogenerated return type of CreateImageDiffNote. | `errors` | String! => Array | Errors encountered during execution of the mutation. | | `note` | Note | The note after mutation | +### CreateIssuePayload + +Autogenerated return type of CreateIssue. + +| Field | Type | Description | +| ----- | ---- | ----------- | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | +| `issue` | Issue | The issue after mutation | + ### CreateIterationPayload Autogenerated return type of CreateIteration. @@ -499,7 +575,7 @@ Autogenerated return type of CreateRequirement. | ----- | ---- | ----------- | | `clientMutationId` | String | A unique identifier for the client performing the mutation. | | `errors` | String! => Array | Errors encountered during execution of the mutation. | -| `requirement` | Requirement | The requirement after mutation | +| `requirement` | Requirement | Requirement after mutation | ### CreateSnippetPayload @@ -510,6 +586,7 @@ Autogenerated return type of CreateSnippet. | `clientMutationId` | String | A unique identifier for the client performing the mutation. | | `errors` | String! => Array | Errors encountered during execution of the mutation. | | `snippet` | Snippet | The snippet after mutation | +| `spam` | Boolean | Indicates whether the operation returns a record detected as spam | ### CreateTestCasePayload @@ -541,8 +618,11 @@ Represents a DAST scanner profile. | `globalId` | DastScannerProfileID! | ID of the DAST scanner profile | | `id` **{warning-solid}** | ID! | **Deprecated:** Use `global_id`. Deprecated in 13.4 | | `profileName` | String | Name of the DAST scanner profile | +| `scanType` | DastScanTypeEnum | Indicates the type of DAST scan that will run. Either a Passive Scan or an Active Scan. | +| `showDebugMessages` | Boolean! | Indicates if debug messages should be included in DAST console output. True to include the debug messages. | | `spiderTimeout` | Int | The maximum number of minutes allowed for the spider to traverse the site | | `targetTimeout` | Int | The maximum number of seconds allowed for the site under test to respond to a request | +| `useAjaxSpider` | Boolean! | Indicates if the AJAX spider should be used to crawl the target site. True to run the AJAX spider in addition to the traditional spider, and false to run only the traditional spider. | ### DastScannerProfileCreatePayload @@ -624,6 +704,18 @@ Autogenerated return type of DastSiteProfileUpdate. | `errors` | String! => Array | Errors encountered during execution of the mutation. | | `id` | DastSiteProfileID | ID of the site profile. | +### DastSiteTokenCreatePayload + +Autogenerated return type of DastSiteTokenCreate. + +| Field | Type | Description | +| ----- | ---- | ----------- | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | +| `id` | DastSiteTokenID | ID of the site token. | +| `status` | DastSiteProfileValidationStatusEnum | The current validation status of the target. | +| `token` | String | Token string. | + ### DeleteAnnotationPayload Autogenerated return type of DeleteAnnotation. @@ -685,6 +777,7 @@ A collection of designs. | Field | Type | Description | | ----- | ---- | ----------- | +| `copyState` | DesignCollectionCopyState | Copy state of the design collection | | `design` | Design | Find a specific design | | `designAtVersion` | DesignAtVersion | Find a design as of a version | | `issue` | Issue! | Issue associated with the design collection | @@ -739,6 +832,16 @@ A specific version in which designs were added, modified or deleted. | `id` | ID! | ID of the design version | | `sha` | ID! | SHA of the design version | +### DestroyBoardListPayload + +Autogenerated return type of DestroyBoardList. + +| Field | Type | Description | +| ----- | ---- | ----------- | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | +| `list` | BoardList | The list after mutation. | + ### DestroyBoardPayload Autogenerated return type of DestroyBoard. @@ -773,14 +876,15 @@ Autogenerated return type of DestroySnippet. | Field | Type | Description | | ----- | ---- | ----------- | -| `detailsPath` | String! | Path of the details for the pipeline status | -| `favicon` | String! | Favicon of the pipeline status | -| `group` | String! | Group of the pipeline status | -| `hasDetails` | Boolean! | Indicates if the pipeline status has further details | -| `icon` | String! | Icon of the pipeline status | -| `label` | String! | Label of the pipeline status | -| `text` | String! | Text of the pipeline status | -| `tooltip` | String! | Tooltip associated with the pipeline status | +| `action` | StatusAction | Action information for the status. This includes method, button title, icon, path, and title | +| `detailsPath` | String | Path of the details for the status | +| `favicon` | String | Favicon of the status | +| `group` | String | Group of the status | +| `hasDetails` | Boolean | Indicates if the status has further details | +| `icon` | String | Icon of the status | +| `label` | String | Label of the status | +| `text` | String | Text of the status | +| `tooltip` | String | Tooltip associated with the status | ### DiffPosition @@ -866,9 +970,10 @@ Describes where code is deployed for a project. | Field | Type | Description | | ----- | ---- | ----------- | | `id` | ID! | ID of the environment | -| `latestOpenedMostSevereAlert` | AlertManagementAlert | The most severe open alert for the environment. If multiple alerts have equal severity, the most recent is returned. | +| `latestOpenedMostSevereAlert` | AlertManagementAlert | The most severe open alert for the environment. If multiple alerts have equal severity, the most recent is returned | | `metricsDashboard` | MetricsDashboard | Metrics dashboard schema for the environment | | `name` | String! | Human-readable name of the environment | +| `path` | String | The path to the environment. Will always return null if `expose_environment_path_in_alert_details` feature flag is disabled | | `state` | String! | State of the environment, for example: available/stopped | ### Epic @@ -878,9 +983,9 @@ Represents an epic. | Field | Type | Description | | ----- | ---- | ----------- | | `author` | User! | Author of the epic | -| `closedAt` | Time | Timestamp of the epic's closure | +| `closedAt` | Time | Timestamp of when the epic was closed | | `confidential` | Boolean | Indicates if the epic is confidential | -| `createdAt` | Time | Timestamp of the epic's creation | +| `createdAt` | Time | Timestamp of when the epic was created | | `descendantCounts` | EpicDescendantCount | Number of open and closed descendant epics and issues | | `descendantWeightSum` | EpicDescendantWeights | Total weight of open and closed issues in the epic and its descendants | | `description` | String | Description of the epic | @@ -907,7 +1012,7 @@ Represents an epic. | `state` | EpicState! | State of the epic | | `subscribed` | Boolean! | Indicates the currently logged in user is subscribed to the epic | | `title` | String | Title of the epic | -| `updatedAt` | Time | Timestamp of the epic's last activity | +| `updatedAt` | Time | Timestamp of when the epic was updated | | `upvotes` | Int! | Number of upvotes the epic has received | | `userPermissions` | EpicPermissions! | Permissions for the current user on the resource | | `webPath` | String! | Web path of the epic | @@ -984,6 +1089,7 @@ Relationship between an epic and an issue. | `relationPath` | String | URI path of the epic-issue relation | | `relativePosition` | Int | Relative position of the issue (used for positioning in epic tree and issue boards) | | `severity` | IssuableSeverity | Severity level of the incident | +| `slaDueAt` | Time | Timestamp of when the issue SLA expires. | | `state` | IssueState! | State of the issue | | `statusPagePublishedIncident` | Boolean | Indicates whether an issue is published to the status page | | `subscribed` | Boolean! | Indicates the currently logged in user is subscribed to the issue | @@ -1069,9 +1175,12 @@ Autogenerated return type of EpicTreeReorder. | Field | Type | Description | | ----- | ---- | ----------- | +| `actualRepositorySizeLimit` | Float | Size limit for repositories in the namespace in bytes | +| `additionalPurchasedStorageSize` | Float | Additional storage purchased for the root namespace in bytes | | `autoDevopsEnabled` | Boolean | Indicates whether Auto DevOps is enabled for all projects within this group | | `avatarUrl` | String | Avatar URL of the group | | `board` | Board | A single board of the group | +| `containsLockedProjects` | Boolean! | Includes at least one project where the repository size exceeds the limit | | `description` | String | Description of the namespace | | `descriptionHtml` | String | The GitLab Flavored Markdown rendering of `description` | | `emailsDisabled` | Boolean | Indicates if a group has email notifications disabled | @@ -1089,6 +1198,7 @@ Autogenerated return type of EpicTreeReorder. | `parent` | Group | Parent group | | `path` | String! | Path of the namespace | | `projectCreationLevel` | String | The permission level required to create projects in the group | +| `repositorySizeExcessProjectCount` | Int! | Number of projects in the root namespace where the repository size exceeds the limit | | `requestAccessEnabled` | Boolean | Indicates if users can request access to namespace | | `requireTwoFactorAuthentication` | Boolean | Indicates if all users in this group are required to set up two-factor authentication | | `rootStorageStatistics` | RootStorageStatistics | Aggregated storage statistics of the namespace. Only available for root namespaces | @@ -1096,6 +1206,8 @@ Autogenerated return type of EpicTreeReorder. | `storageSizeLimit` | Float | Total storage limit of the root namespace in bytes | | `subgroupCreationLevel` | String | The permission level required to create subgroups within the group | | `temporaryStorageIncreaseEndsOn` | Time | Date until the temporary storage increase is active | +| `totalRepositorySize` | Float | Total repository size of all projects in the root namespace in bytes | +| `totalRepositorySizeExcess` | Float | Total excess repository size of all projects in the root namespace in bytes | | `twoFactorGracePeriod` | Int | Time before two-factor authentication is enforced | | `userPermissions` | GroupPermissions! | Permissions for the current user on the resource | | `visibility` | String | Visibility of the namespace | @@ -1168,6 +1280,7 @@ Represents a recorded measurement (object count) for the Admins. | `reference` | String! | Internal reference of the issue. Returned in shortened format by default | | `relativePosition` | Int | Relative position of the issue (used for positioning in epic tree and issue boards) | | `severity` | IssuableSeverity | Severity level of the incident | +| `slaDueAt` | Time | Timestamp of when the issue SLA expires. | | `state` | IssueState! | State of the issue | | `statusPagePublishedIncident` | Boolean | Indicates whether an issue is published to the status page | | `subscribed` | Boolean! | Indicates the currently logged in user is subscribed to the issue | @@ -1195,6 +1308,16 @@ Autogenerated return type of IssueMoveList. | `errors` | String! => Array | Errors encountered during execution of the mutation. | | `issue` | Issue | The issue after mutation | +### IssueMovePayload + +Autogenerated return type of IssueMove. + +| Field | Type | Description | +| ----- | ---- | ----------- | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | +| `issue` | Issue | The issue after mutation | + ### IssuePermissions Check permissions for the current user on a issue. @@ -1486,6 +1609,21 @@ Autogenerated return type of MergeRequestCreate. | `errors` | String! => Array | Errors encountered during execution of the mutation. | | `mergeRequest` | MergeRequest | The merge request after mutation | +### MergeRequestDiffRegistry + +Represents the Geo sync and verification state of a Merge Request diff. + +| Field | Type | Description | +| ----- | ---- | ----------- | +| `createdAt` | Time | Timestamp when the MergeRequestDiffRegistry was created | +| `id` | ID! | ID of the MergeRequestDiffRegistry | +| `lastSyncFailure` | String | Error message during sync of the MergeRequestDiffRegistry | +| `lastSyncedAt` | Time | Timestamp of the most recent successful sync of the MergeRequestDiffRegistry | +| `mergeRequestDiffId` | ID! | ID of the Merge Request diff | +| `retryAt` | Time | Timestamp after which the MergeRequestDiffRegistry should be resynced | +| `retryCount` | Int | Number of consecutive failed sync attempts of the MergeRequestDiffRegistry | +| `state` | RegistryState | Sync state of the MergeRequestDiffRegistry | + ### MergeRequestPermissions Check permissions for the current user on a merge request. @@ -1630,6 +1768,9 @@ Contains statistics about a milestone. | Field | Type | Description | | ----- | ---- | ----------- | +| `actualRepositorySizeLimit` | Float | Size limit for repositories in the namespace in bytes | +| `additionalPurchasedStorageSize` | Float | Additional storage purchased for the root namespace in bytes | +| `containsLockedProjects` | Boolean! | Includes at least one project where the repository size exceeds the limit | | `description` | String | Description of the namespace | | `descriptionHtml` | String | The GitLab Flavored Markdown rendering of `description` | | `fullName` | String! | Full name of the namespace | @@ -1639,10 +1780,13 @@ Contains statistics about a milestone. | `lfsEnabled` | Boolean | Indicates if Large File Storage (LFS) is enabled for namespace | | `name` | String! | Name of the namespace | | `path` | String! | Path of the namespace | +| `repositorySizeExcessProjectCount` | Int! | Number of projects in the root namespace where the repository size exceeds the limit | | `requestAccessEnabled` | Boolean | Indicates if users can request access to namespace | | `rootStorageStatistics` | RootStorageStatistics | Aggregated storage statistics of the namespace. Only available for root namespaces | | `storageSizeLimit` | Float | Total storage limit of the root namespace in bytes | | `temporaryStorageIncreaseEndsOn` | Time | Date until the temporary storage increase is active | +| `totalRepositorySize` | Float | Total repository size of all projects in the root namespace in bytes | +| `totalRepositorySizeExcess` | Float | Total excess repository size of all projects in the root namespace in bytes | | `visibility` | String | Visibility of the namespace | ### NamespaceIncreaseStorageTemporarilyPayload @@ -1702,7 +1846,7 @@ Represents a package. ### PackageFileRegistry -Represents the sync and verification state of a package file. +Represents the Geo sync and verification state of a package file. | Field | Type | Description | | ----- | ---- | ----------- | @@ -1790,6 +1934,7 @@ Autogenerated return type of PipelineRetry. | Field | Type | Description | | ----- | ---- | ----------- | +| `actualRepositorySizeLimit` | Float | Size limit for the repository in bytes | | `alertManagementAlert` | AlertManagementAlert | A single Alert Management alert of the project | | `alertManagementAlertStatusCounts` | AlertManagementAlertStatusCountsType | Counts of alerts by status for the project | | `allowMergeOnSkippedPipeline` | Boolean | If `only_allow_merge_if_pipeline_succeeds` is true, indicates if merge requests of the project can also be merged with skipped jobs | @@ -1836,8 +1981,9 @@ Autogenerated return type of PipelineRetry. | `release` | Release | A single release of the project | | `removeSourceBranchAfterMerge` | Boolean | Indicates if `Delete source branch` option should be enabled by default for all new merge requests of the project | | `repository` | Repository | Git repository of the project | +| `repositorySizeExcess` | Float | Size of repository that exceeds the limit in bytes | | `requestAccessEnabled` | Boolean | Indicates if users can request member access to the project | -| `requirement` | Requirement | Find a single requirement. Available only when feature flag `requirements_management` is enabled. | +| `requirement` | Requirement | Find a single requirement | | `requirementStatesCount` | RequirementStatesCount | Number of requirements for the project by their state | | `sastCiConfiguration` | SastCiConfiguration | SAST CI configuration for the project | | `securityDashboardPath` | String | Path to project's security dashboard | @@ -2049,12 +2195,16 @@ Represents a requirement. | ----- | ---- | ----------- | | `author` | User! | Author of the requirement | | `createdAt` | Time! | Timestamp of when the requirement was created | +| `description` | String | Description of the requirement | +| `descriptionHtml` | String | The GitLab Flavored Markdown rendering of `description` | | `id` | ID! | ID of the requirement | | `iid` | ID! | Internal ID of the requirement | +| `lastTestReportManuallyCreated` | Boolean | Indicates if latest test report was created by user | | `lastTestReportState` | TestReportState | Latest requirement test report state | | `project` | Project! | Project to which the requirement belongs | | `state` | RequirementState! | State of the requirement | | `title` | String | Title of the requirement | +| `titleHtml` | String | The GitLab Flavored Markdown rendering of `title` | | `updatedAt` | Time! | Timestamp of when the requirement was last updated | | `userPermissions` | RequirementPermissions! | Permissions for the current user on the resource | @@ -2079,6 +2229,16 @@ Counts of requirements by their state. | `archived` | Int | Number of archived requirements | | `opened` | Int | Number of opened requirements | +### RevertVulnerabilityToDetectedPayload + +Autogenerated return type of RevertVulnerabilityToDetected. + +| Field | Type | Description | +| ----- | ---- | ----------- | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | +| `vulnerability` | Vulnerability | The vulnerability after revert | + ### RootStorageStatistics | Field | Type | Description | @@ -2086,6 +2246,7 @@ Counts of requirements by their state. | `buildArtifactsSize` | Float! | The CI artifacts size in bytes | | `lfsObjectsSize` | Float! | The LFS objects size in bytes | | `packagesSize` | Float! | The packages size in bytes | +| `pipelineArtifactsSize` | Float! | The CI pipeline artifacts size in bytes | | `repositorySize` | Float! | The Git repository size in bytes | | `snippetsSize` | Float! | The snippets size in bytes | | `storageSize` | Float! | The total storage in bytes | @@ -2101,6 +2262,20 @@ Autogenerated return type of RunDASTScan. | `errors` | String! => Array | Errors encountered during execution of the mutation. | | `pipelineUrl` | String | URL of the pipeline that was created. | +### RunnerArchitecture + +| Field | Type | Description | +| ----- | ---- | ----------- | +| `downloadLocation` | String! | Download location for the runner for the platform architecture | +| `name` | String! | Name of the runner platform architecture | + +### RunnerPlatform + +| Field | Type | Description | +| ----- | ---- | ----------- | +| `humanReadableName` | String! | Human readable name of the runner platform | +| `name` | String! | Name slug of the runner platform | + ### SastCiConfigurationAnalyzersEntity Represents an analyzer entity in SAST CI configuration. @@ -2150,6 +2325,7 @@ Represents summary of a security report. | Field | Type | Description | | ----- | ---- | ----------- | +| `apiFuzzing` | SecurityReportSummarySection | Aggregated counts for the api_fuzzing scan | | `containerScanning` | SecurityReportSummarySection | Aggregated counts for the container_scanning scan | | `coverageFuzzing` | SecurityReportSummarySection | Aggregated counts for the coverage_fuzzing scan | | `dast` | SecurityReportSummarySection | Aggregated counts for the dast scan | @@ -2302,7 +2478,6 @@ Represents a snippet entry. | ----- | ---- | ----------- | | `author` | User | The owner of the snippet | | `blob` **{warning-solid}** | SnippetBlob! | **Deprecated:** Use `blobs`. Deprecated in 13.3 | -| `blobs` | SnippetBlob! => Array | Snippet blobs | | `createdAt` | Time! | Timestamp this snippet was created | | `description` | String | Description of the snippet | | `descriptionHtml` | String | The GitLab Flavored Markdown rendering of `description` | @@ -2362,6 +2537,16 @@ Represents how the blob content should be displayed. | `reportSnippet` | Boolean! | Indicates the user can perform `report_snippet` on this resource | | `updateSnippet` | Boolean! | Indicates the user can perform `update_snippet` on this resource | +### StatusAction + +| Field | Type | Description | +| ----- | ---- | ----------- | +| `buttonTitle` | String | Title for the button, for example: Retry this job | +| `icon` | String | Icon used in the action button | +| `method` | String | Method for the action, for example: :post | +| `path` | String | Path for the action | +| `title` | String | Title for the action, for example: Retry | + ### Submodule | Field | Type | Description | @@ -2384,20 +2569,31 @@ Completion status of tasks. | `completedCount` | Int! | Number of completed tasks | | `count` | Int! | Number of total tasks | -### TerraformStateRegistry +### TerraformState + +| Field | Type | Description | +| ----- | ---- | ----------- | +| `createdAt` | Time! | Timestamp the Terraform state was created | +| `id` | ID! | ID of the Terraform state | +| `lockedAt` | Time | Timestamp the Terraform state was locked | +| `lockedByUser` | User | The user currently holding a lock on the Terraform state | +| `name` | String! | Name of the Terraform state | +| `updatedAt` | Time! | Timestamp the Terraform state was updated | + +### TerraformStateVersionRegistry -Represents the sync and verification state of a terraform state. +Represents the Geo sync and verification state of a terraform state version. | Field | Type | Description | | ----- | ---- | ----------- | -| `createdAt` | Time | Timestamp when the TerraformStateRegistry was created | -| `id` | ID! | ID of the TerraformStateRegistry | -| `lastSyncFailure` | String | Error message during sync of the TerraformStateRegistry | -| `lastSyncedAt` | Time | Timestamp of the most recent successful sync of the TerraformStateRegistry | -| `retryAt` | Time | Timestamp after which the TerraformStateRegistry should be resynced | -| `retryCount` | Int | Number of consecutive failed sync attempts of the TerraformStateRegistry | -| `state` | RegistryState | Sync state of the TerraformStateRegistry | -| `terraformStateId` | ID! | ID of the TerraformState | +| `createdAt` | Time | Timestamp when the TerraformStateVersionRegistry was created | +| `id` | ID! | ID of the TerraformStateVersionRegistry | +| `lastSyncFailure` | String | Error message during sync of the TerraformStateVersionRegistry | +| `lastSyncedAt` | Time | Timestamp of the most recent successful sync of the TerraformStateVersionRegistry | +| `retryAt` | Time | Timestamp after which the TerraformStateVersionRegistry should be resynced | +| `retryCount` | Int | Number of consecutive failed sync attempts of the TerraformStateVersionRegistry | +| `state` | RegistryState | Sync state of the TerraformStateVersionRegistry | +| `terraformStateVersionId` | ID! | ID of the terraform state version | ### TestReport @@ -2523,6 +2719,16 @@ Autogenerated return type of UpdateAlertStatus. | `issue` | Issue | The issue created after mutation | | `todo` | Todo | The todo after mutation | +### UpdateBoardEpicUserPreferencesPayload + +Autogenerated return type of UpdateBoardEpicUserPreferences. + +| Field | Type | Description | +| ----- | ---- | ----------- | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `epicUserPreferences` | BoardEpicUserPreferences | User preferences for the epic in the board after mutation | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | + ### UpdateBoardListPayload Autogenerated return type of UpdateBoardList. @@ -2611,7 +2817,7 @@ Autogenerated return type of UpdateRequirement. | ----- | ---- | ----------- | | `clientMutationId` | String | A unique identifier for the client performing the mutation. | | `errors` | String! => Array | Errors encountered during execution of the mutation. | -| `requirement` | Requirement | The requirement after mutation | +| `requirement` | Requirement | Requirement after mutation | ### UpdateSnippetPayload @@ -2622,6 +2828,7 @@ Autogenerated return type of UpdateSnippet. | `clientMutationId` | String | A unique identifier for the client performing the mutation. | | `errors` | String! => Array | Errors encountered during execution of the mutation. | | `snippet` | Snippet | The snippet after mutation | +| `spam` | Boolean | Indicates whether the operation returns a record detected as spam | ### User @@ -2690,16 +2897,36 @@ Represents a vulnerability. | `location` | VulnerabilityLocation | Location metadata for the vulnerability. Its fields depend on the type of security scan that found the vulnerability | | `primaryIdentifier` | VulnerabilityIdentifier | Primary identifier of the vulnerability. | | `project` | Project | The project on which the vulnerability was found | -| `reportType` | VulnerabilityReportType | Type of the security report that found the vulnerability (SAST, DEPENDENCY_SCANNING, CONTAINER_SCANNING, DAST, SECRET_DETECTION, COVERAGE_FUZZING) | +| `reportType` | VulnerabilityReportType | Type of the security report that found the vulnerability (SAST, DEPENDENCY_SCANNING, CONTAINER_SCANNING, DAST, SECRET_DETECTION, COVERAGE_FUZZING, API_FUZZING) | | `resolvedOnDefaultBranch` | Boolean! | Indicates whether the vulnerability is fixed on the default branch or not | | `scanner` | VulnerabilityScanner | Scanner metadata for the vulnerability. | | `severity` | VulnerabilitySeverity | Severity of the vulnerability (INFO, UNKNOWN, LOW, MEDIUM, HIGH, CRITICAL) | -| `state` | VulnerabilityState | State of the vulnerability (DETECTED, DISMISSED, RESOLVED, CONFIRMED) | +| `state` | VulnerabilityState | State of the vulnerability (DETECTED, CONFIRMED, RESOLVED, DISMISSED) | | `title` | String | Title of the vulnerability | | `userNotesCount` | Int! | Number of user notes attached to the vulnerability | | `userPermissions` | VulnerabilityPermissions! | Permissions for the current user on the resource | | `vulnerabilityPath` | String | URL to the vulnerability's details page | +### VulnerabilityConfirmPayload + +Autogenerated return type of VulnerabilityConfirm. + +| Field | Type | Description | +| ----- | ---- | ----------- | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | +| `vulnerability` | Vulnerability | The vulnerability after state change | + +### VulnerabilityDismissPayload + +Autogenerated return type of VulnerabilityDismiss. + +| Field | Type | Description | +| ----- | ---- | ----------- | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | +| `vulnerability` | Vulnerability | The vulnerability after dismissal | + ### VulnerabilityIdentifier Represents a vulnerability identifier. @@ -2812,6 +3039,16 @@ Autogenerated return type of VulnerabilityResolve. | `errors` | String! => Array | Errors encountered during execution of the mutation. | | `vulnerability` | Vulnerability | The vulnerability after state change | +### VulnerabilityRevertToDetectedPayload + +Autogenerated return type of VulnerabilityRevertToDetected. + +| Field | Type | Description | +| ----- | ---- | ----------- | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | +| `vulnerability` | Vulnerability | The vulnerability after revert | + ### VulnerabilityScanner Represents a vulnerability scanner. @@ -2890,6 +3127,8 @@ Values for sorting alerts. | Value | Description | | ----- | ----------- | +| `CREATED_ASC` | Created at ascending order | +| `CREATED_DESC` | Created at descending order | | `CREATED_TIME_ASC` | Created time by ascending order | | `CREATED_TIME_DESC` | Created time by descending order | | `ENDED_AT_ASC` | End time by ascending order | @@ -2902,12 +3141,14 @@ Values for sorting alerts. | `STARTED_AT_DESC` | Start time by descending order | | `STATUS_ASC` | Status by order: Ignored > Resolved > Acknowledged > Triggered | | `STATUS_DESC` | Status by order: Triggered > Acknowledged > Resolved > Ignored | +| `UPDATED_ASC` | Updated at ascending order | +| `UPDATED_DESC` | Updated at descending order | | `UPDATED_TIME_ASC` | Created time by ascending order | | `UPDATED_TIME_DESC` | Created time by descending order | -| `created_asc` | Created at ascending order | -| `created_desc` | Created at descending order | -| `updated_asc` | Updated at ascending order | -| `updated_desc` | Updated at descending order | +| `created_asc` **{warning-solid}** | **Deprecated:** Use CREATED_ASC. Deprecated in 13.5 | +| `created_desc` **{warning-solid}** | **Deprecated:** Use CREATED_DESC. Deprecated in 13.5 | +| `updated_asc` **{warning-solid}** | **Deprecated:** Use UPDATED_ASC. Deprecated in 13.5 | +| `updated_desc` **{warning-solid}** | **Deprecated:** Use UPDATED_DESC. Deprecated in 13.5 | ### AlertManagementSeverity @@ -2996,6 +3237,7 @@ Mode of a commit action. | Value | Description | | ----- | ----------- | +| `ACTIVE` | Active DAST scan. This scan will make active attacks against the target site. | | `PASSIVE` | Passive DAST scan. This scan will not make active attacks against the target site. | ### DastSiteProfileValidationStatusEnum @@ -3007,6 +3249,16 @@ Mode of a commit action. | `PASSED_VALIDATION` | Site validation process finished successfully | | `PENDING_VALIDATION` | Site validation process has not started | +### DesignCollectionCopyState + +Copy state of a DesignCollection. + +| Value | Description | +| ----- | ----------- | +| `ERROR` | The DesignCollection encountered an error during a copy | +| `IN_PROGRESS` | The DesignCollection is being copied | +| `READY` | The DesignCollection has no copy in progress | + ### DesignVersionEvent Mutation event of a design within a version. @@ -3115,6 +3367,8 @@ Values for sorting issues. | Value | Description | | ----- | ----------- | +| `CREATED_ASC` | Created at ascending order | +| `CREATED_DESC` | Created at descending order | | `DUE_DATE_ASC` | Due date by ascending order | | `DUE_DATE_DESC` | Due date by descending order | | `LABEL_PRIORITY_ASC` | Label priority by ascending order | @@ -3123,13 +3377,19 @@ Values for sorting issues. | `MILESTONE_DUE_DESC` | Milestone due date by descending order | | `PRIORITY_ASC` | Priority by ascending order | | `PRIORITY_DESC` | Priority by descending order | +| `PUBLISHED_ASC` | Published issues shown last | +| `PUBLISHED_DESC` | Published issues shown first | | `RELATIVE_POSITION_ASC` | Relative position by ascending order | +| `SEVERITY_ASC` | Severity from less critical to more critical | +| `SEVERITY_DESC` | Severity from more critical to less critical | +| `UPDATED_ASC` | Updated at ascending order | +| `UPDATED_DESC` | Updated at descending order | | `WEIGHT_ASC` | Weight by ascending order | | `WEIGHT_DESC` | Weight by descending order | -| `created_asc` | Created at ascending order | -| `created_desc` | Created at descending order | -| `updated_asc` | Updated at ascending order | -| `updated_desc` | Updated at descending order | +| `created_asc` **{warning-solid}** | **Deprecated:** Use CREATED_ASC. Deprecated in 13.5 | +| `created_desc` **{warning-solid}** | **Deprecated:** Use CREATED_DESC. Deprecated in 13.5 | +| `updated_asc` **{warning-solid}** | **Deprecated:** Use UPDATED_ASC. Deprecated in 13.5 | +| `updated_desc` **{warning-solid}** | **Deprecated:** Use UPDATED_DESC. Deprecated in 13.5 | ### IssueState @@ -3142,6 +3402,15 @@ State of a GitLab issue. | `locked` | | | `opened` | | +### IssueStateEvent + +Values for issue state events. + +| Value | Description | +| ----- | ----------- | +| `CLOSE` | Closes the issue | +| `REOPEN` | Reopens the issue | + ### IssueType Issue type. @@ -3184,6 +3453,10 @@ Possible identifier types for a measurement. | `ISSUES` | Issue count | | `MERGE_REQUESTS` | Merge request count | | `PIPELINES` | Pipeline count | +| `PIPELINES_CANCELED` | Pipeline count with canceled status | +| `PIPELINES_FAILED` | Pipeline count with failed status | +| `PIPELINES_SKIPPED` | Pipeline count with skipped status | +| `PIPELINES_SUCCEEDED` | Pipeline count with success status | | `PROJECTS` | Project count | | `USERS` | User count | @@ -3193,6 +3466,8 @@ Values for sorting merge requests. | Value | Description | | ----- | ----------- | +| `CREATED_ASC` | Created at ascending order | +| `CREATED_DESC` | Created at descending order | | `LABEL_PRIORITY_ASC` | Label priority by ascending order | | `LABEL_PRIORITY_DESC` | Label priority by descending order | | `MERGED_AT_ASC` | Merge time by ascending order | @@ -3201,10 +3476,12 @@ Values for sorting merge requests. | `MILESTONE_DUE_DESC` | Milestone due date by descending order | | `PRIORITY_ASC` | Priority by ascending order | | `PRIORITY_DESC` | Priority by descending order | -| `created_asc` | Created at ascending order | -| `created_desc` | Created at descending order | -| `updated_asc` | Updated at ascending order | -| `updated_desc` | Updated at descending order | +| `UPDATED_ASC` | Updated at ascending order | +| `UPDATED_DESC` | Updated at descending order | +| `created_asc` **{warning-solid}** | **Deprecated:** Use CREATED_ASC. Deprecated in 13.5 | +| `created_desc` **{warning-solid}** | **Deprecated:** Use CREATED_DESC. Deprecated in 13.5 | +| `updated_asc` **{warning-solid}** | **Deprecated:** Use UPDATED_ASC. Deprecated in 13.5 | +| `updated_desc` **{warning-solid}** | **Deprecated:** Use UPDATED_DESC. Deprecated in 13.5 | ### MergeRequestState @@ -3256,13 +3533,15 @@ Values for sorting projects. | Value | Description | | ----- | ----------- | -| `COMPOSER` | Packages from the composer package manager | -| `CONAN` | Packages from the conan package manager | -| `GENERIC` | Packages from the generic package manager | -| `MAVEN` | Packages from the maven package manager | -| `NPM` | Packages from the npm package manager | -| `NUGET` | Packages from the nuget package manager | -| `PYPI` | Packages from the pypi package manager | +| `COMPOSER` | Packages from the Composer package manager | +| `CONAN` | Packages from the Conan package manager | +| `DEBIAN` | Packages from the Debian package manager | +| `GENERIC` | Packages from the Generic package manager | +| `GOLANG` | Packages from the Golang package manager | +| `MAVEN` | Packages from the Maven package manager | +| `NPM` | Packages from the NPM package manager | +| `NUGET` | Packages from the Nuget package manager | +| `PYPI` | Packages from the PyPI package manager | ### PipelineConfigSourceEnum @@ -3352,6 +3631,7 @@ The type of the security scanner. | Value | Description | | ----- | ----------- | +| `API_FUZZING` | | | `CONTAINER_SCANNING` | | | `COVERAGE_FUZZING` | | | `DAST` | | @@ -3428,10 +3708,14 @@ Common sort values. | Value | Description | | ----- | ----------- | -| `created_asc` | Created at ascending order | -| `created_desc` | Created at descending order | -| `updated_asc` | Updated at ascending order | -| `updated_desc` | Updated at descending order | +| `CREATED_ASC` | Created at ascending order | +| `CREATED_DESC` | Created at descending order | +| `UPDATED_ASC` | Updated at ascending order | +| `UPDATED_DESC` | Updated at descending order | +| `created_asc` **{warning-solid}** | **Deprecated:** Use CREATED_ASC. Deprecated in 13.5 | +| `created_desc` **{warning-solid}** | **Deprecated:** Use CREATED_DESC. Deprecated in 13.5 | +| `updated_asc` **{warning-solid}** | **Deprecated:** Use UPDATED_ASC. Deprecated in 13.5 | +| `updated_desc` **{warning-solid}** | **Deprecated:** Use UPDATED_DESC. Deprecated in 13.5 | ### TestReportState @@ -3532,6 +3816,7 @@ The type of the security scan that found the vulnerability. | Value | Description | | ----- | ----------- | +| `API_FUZZING` | | | `CONTAINER_SCANNING` | | | `COVERAGE_FUZZING` | | | `DAST` | | @@ -3558,8 +3843,16 @@ Vulnerability sort values. | Value | Description | | ----- | ----------- | +| `detected_asc` | Detection timestamp in ascending order | +| `detected_desc` | Detection timestamp in descending order | +| `report_type_asc` | Report Type in ascending order | +| `report_type_desc` | Report Type in descending order | | `severity_asc` | Severity in ascending order | | `severity_desc` | Severity in descending order | +| `state_asc` | State in ascending order | +| `state_desc` | State in descending order | +| `title_asc` | Title in ascending order | +| `title_desc` | Title in descending order | ### VulnerabilityState diff --git a/doc/api/group_clusters.md b/doc/api/group_clusters.md index 17413ea2a3b..27b76d1f0c0 100644 --- a/doc/api/group_clusters.md +++ b/doc/api/group_clusters.md @@ -8,8 +8,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/30213) in GitLab 12.1. -NOTE: **Note:** -User will need at least maintainer access for the group to use these endpoints. +Users need at least [Maintainer](../user/permissions.md) access for the group to use these endpoints. ## List group clusters diff --git a/doc/api/group_iterations.md b/doc/api/group_iterations.md new file mode 100644 index 00000000000..62431244d78 --- /dev/null +++ b/doc/api/group_iterations.md @@ -0,0 +1,55 @@ +--- +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Group iterations API **(STARTER)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/118742) in [GitLab Starter](https://about.gitlab.com/pricing/) 13.5. + +This page describes the group iterations API. +There's a separate [project iterations API](./iterations.md) page. + +## List group iterations + +Returns a list of group iterations. + +```plaintext +GET /groups/:id/iterations +GET /groups/:id/iterations?state=opened +GET /groups/:id/iterations?state=closed +GET /groups/:id/iterations?title=1.0 +GET /groups/:id/iterations?search=version +``` + +| Attribute | Type | Required | Description | +| ------------------- | ------- | -------- | ----------- | +| `state` | string | no | Return only `opened`, `upcoming`, `started`, `closed`, or `all` iterations. Defaults to `all`. | +| `search` | string | no | Return only iterations with a title matching the provided string. | +| `include_ancestors` | boolean | no | Include iterations from parent group and its ancestors. Defaults to `true`. | + +Example request: + +```shell +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/5/iterations" +``` + +Example response: + +```json +[ + { + "id": 53, + "iid": 13, + "group_id": 5, + "title": "Iteration II", + "description": "Ipsum Lorem ipsum", + "state": 2, + "created_at": "2020-01-27T05:07:12.573Z", + "updated_at": "2020-01-27T05:07:12.573Z", + "due_date": "2020-02-01", + "start_date": "2020-02-14" + } +] +``` diff --git a/doc/api/group_milestones.md b/doc/api/group_milestones.md index 47350442b3e..3220707e9e3 100644 --- a/doc/api/group_milestones.md +++ b/doc/api/group_milestones.md @@ -9,7 +9,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/12819) in GitLab 9.5. This page describes the group milestones API. -There's a separate [project milestones API](./group_milestones.md) page. +There's a separate [project milestones API](./milestones.md) page. ## List group milestones diff --git a/doc/api/group_wikis.md b/doc/api/group_wikis.md index 414c795e092..c61a557fcc6 100644 --- a/doc/api/group_wikis.md +++ b/doc/api/group_wikis.md @@ -5,9 +5,9 @@ info: "To determine the technical writer assigned to the Stage/Group associated type: reference, api --- -# Wikis API +# Group wikis API **(PREMIUM)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/212199) in GitLab 13.2. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/212199) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.5. Available only in APIv4. diff --git a/doc/api/groups.md b/doc/api/groups.md index ae3300e24fb..53c92cf85ec 100644 --- a/doc/api/groups.md +++ b/doc/api/groups.md @@ -167,6 +167,89 @@ GET /groups/:id/subgroups ] ``` +## List a group's descendant groups + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/217115) in GitLab 13.5 + +Get a list of visible descendant groups of this group. +When accessed without authentication, only public groups are returned. + +By default, this request returns 20 results at a time because the API results [are paginated](README.md#pagination). + +Parameters: + +| Attribute | Type | Required | Description | +| ------------------------ | ----------------- | -------- | ----------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) of the immediate parent group | +| `skip_groups` | array of integers | no | Skip the group IDs passed | +| `all_available` | boolean | no | Show all the groups you have access to (defaults to `false` for authenticated users, `true` for admin). Attributes `owned` and `min_access_level` have precedence | +| `search` | string | no | Return the list of authorized groups matching the search criteria | +| `order_by` | string | no | Order groups by `name`, `path`, or `id`. Default is `name` | +| `sort` | string | no | Order groups in `asc` or `desc` order. Default is `asc` | +| `statistics` | boolean | no | Include group statistics (admins only) | +| `with_custom_attributes` | boolean | no | Include [custom attributes](custom_attributes.md) in response (admins only) | +| `owned` | boolean | no | Limit to groups explicitly owned by the current user | +| `min_access_level` | integer | no | Limit to groups where current user has at least this [access level](members.md#valid-access-levels) | + +```plaintext +GET /groups/:id/descendant_groups +``` + +```json +[ + { + "id": 2, + "name": "Bar Group", + "path": "foo/bar", + "description": "A subgroup of Foo Group", + "visibility": "public", + "share_with_group_lock": false, + "require_two_factor_authentication": false, + "two_factor_grace_period": 48, + "project_creation_level": "developer", + "auto_devops_enabled": null, + "subgroup_creation_level": "owner", + "emails_disabled": null, + "mentions_disabled": null, + "lfs_enabled": true, + "default_branch_protection": 2, + "avatar_url": "http://gitlab.example.com/uploads/group/avatar/1/bar.jpg", + "web_url": "http://gitlab.example.com/groups/foo/bar", + "request_access_enabled": false, + "full_name": "Bar Group", + "full_path": "foo/bar", + "file_template_project_id": 1, + "parent_id": 123, + "created_at": "2020-01-15T12:36:29.590Z" + }, + { + "id": 3, + "name": "Baz Group", + "path": "foo/bar/baz", + "description": "A subgroup of Bar Group", + "visibility": "public", + "share_with_group_lock": false, + "require_two_factor_authentication": false, + "two_factor_grace_period": 48, + "project_creation_level": "developer", + "auto_devops_enabled": null, + "subgroup_creation_level": "owner", + "emails_disabled": null, + "mentions_disabled": null, + "lfs_enabled": true, + "default_branch_protection": 2, + "avatar_url": "http://gitlab.example.com/uploads/group/avatar/1/baz.jpg", + "web_url": "http://gitlab.example.com/groups/foo/bar/baz", + "request_access_enabled": false, + "full_name": "Baz Group", + "full_path": "foo/bar/baz", + "file_template_project_id": 1, + "parent_id": 123, + "created_at": "2020-01-15T12:36:29.590Z" + } +] +``` + ## List a group's projects Get a list of projects in this group. When accessed without authentication, only public projects are returned. @@ -357,6 +440,7 @@ Example response: "import_status":"failed", "open_issues_count":10, "ci_default_git_depth":50, + "ci_forward_deployment_enabled":true, "public_jobs":true, "build_timeout":3600, "auto_cancel_pending_pipelines":"enabled", @@ -676,6 +760,7 @@ Parameters: | `default_branch_protection` | integer | no | See [Options for `default_branch_protection`](#options-for-default_branch_protection). Default to the global level default branch protection setting. | | `shared_runners_minutes_limit` | integer | no | **(STARTER ONLY)** Pipeline minutes quota for this group (included in plan). Can be `nil` (default; inherit system default), `0` (unlimited) or `> 0` | | `extra_shared_runners_minutes_limit` | integer | no | **(STARTER ONLY)** Extra pipeline minutes quota for this group (purchased in addition to the minutes included in the plan). | +| `shared_runners_setting` | string | no | See [Options for `shared_runners_setting`](#options-for-shared_runners_setting). Enable or disable shared runners for a group's subgroups and projects. | ### Options for `default_branch_protection` @@ -687,6 +772,16 @@ The `default_branch_protection` attribute determines whether developers and main | `1` | Partial protection. Developers and maintainers can: <br>- Push new commits | | `2` | Full protection. Only maintainers can: <br>- Push new commits | +### Options for `shared_runners_setting` + +The `shared_runners_setting` attribute determines whether shared runners are enabled for a group's subgroups and projects. + +| Value | Description | +|-------|-------------------------------------------------------------------------------------------------------------| +| `enabled` | Enables shared runners for all projects and subgroups in this group. | +| `disabled_with_override` | Disables shared runners for all projects and subgroups in this group, but allows subgroups to override this setting. | +| `disabled_and_unoverridable` | Disables shared runners for all projects and subgroups in this group, and prevents subgroups from overriding this setting. | + ## New Subgroup This is similar to creating a [New group](#new-group). You'll need the `parent_id` from the [List groups](#list-groups) call. You can then enter the desired: @@ -696,7 +791,7 @@ This is similar to creating a [New group](#new-group). You'll need the `parent_i ```shell curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --header "Content-Type: application/json" \ - --data '{"path": "<subgroup_path>", "name": "<subgroup_name>", "parent_id": <parent_group_id> } \ + --data '{"path": "<subgroup_path>", "name": "<subgroup_name>", "parent_id": <parent_group_id> }' \ "https://gitlab.example.com/api/v4/groups/" ``` diff --git a/doc/api/import.md b/doc/api/import.md index 54e7eb12ed1..e377853ade0 100644 --- a/doc/api/import.md +++ b/doc/api/import.md @@ -50,8 +50,8 @@ POST /import/bitbucket_server | `personal_access_token` | string | yes | Bitbucket Server personal access token/password | | `bitbucket_server_project` | string | yes | Bitbucket Project Key | | `bitbucket_server_repo` | string | yes | Bitbucket Repository Name | -| `new_name` | string | no | New repo name | -| `target_namespace` | string | no | Namespace to import repo into | +| `new_name` | string | no | New repository name | +| `target_namespace` | string | no | Namespace to import repository into | ```shell curl --request POST \ diff --git a/doc/api/instance_level_ci_variables.md b/doc/api/instance_level_ci_variables.md index d8f306a822c..e8550d41c44 100644 --- a/doc/api/instance_level_ci_variables.md +++ b/doc/api/instance_level_ci_variables.md @@ -70,7 +70,7 @@ curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/a Create a new instance-level variable. -[Since GitLab 13.1](https://gitlab.com/gitlab-org/gitlab/-/issues/216097), the maximum number of allowed instance-level variables can be changed. +[In GitLab 13.1 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/216097), the maximum number of allowed instance-level variables can be changed. ```plaintext POST /admin/ci/variables @@ -79,7 +79,7 @@ POST /admin/ci/variables | Attribute | Type | required | Description | |-----------------|---------|----------|-----------------------| | `key` | string | yes | The `key` of a variable. Max 255 characters, only `A-Z`, `a-z`, `0-9`, and `_` are allowed. | -| `value` | string | yes | The `value` of a variable. 10,000 characters allowed. [Since GitLab 13.3](https://gitlab.com/gitlab-org/gitlab/-/issues/220028) | +| `value` | string | yes | The `value` of a variable. 10,000 characters allowed ([GitLab 13.3 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/220028)). | | `variable_type` | string | no | The type of a variable. Available types are: `env_var` (default) and `file`. | | `protected` | boolean | no | Whether the variable is protected. | | `masked` | boolean | no | Whether the variable is masked. | @@ -109,7 +109,7 @@ PUT /admin/ci/variables/:key | Attribute | Type | required | Description | |-----------------|---------|----------|-------------------------| | `key` | string | yes | The `key` of a variable. | -| `value` | string | yes | The `value` of a variable. [Since GitLab 13.3](https://gitlab.com/gitlab-org/gitlab/-/issues/220028), around 10,000 characters allowed. Previously 700 characters. | +| `value` | string | yes | The `value` of a variable. 10,000 characters allowed ([GitLab 13.3 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/220028)). | | `variable_type` | string | no | The type of a variable. Available types are: `env_var` (default) and `file`. | | `protected` | boolean | no | Whether the variable is protected. | | `masked` | boolean | no | Whether the variable is masked. | diff --git a/doc/api/issue_links.md b/doc/api/issue_links.md index b6502bf099c..757910d0946 100644 --- a/doc/api/issue_links.md +++ b/doc/api/issue_links.md @@ -1,8 +1,11 @@ -# Issue links API **(STARTER)** +# Issue links API **(CORE)** + +> The simple "relates to" relationship [moved](https://gitlab.com/gitlab-org/gitlab/-/issues/212329) to [GitLab Core](https://about.gitlab.com/pricing/) in 13.4. ## List issue relations -Get a list of related issues of a given issue, sorted by the relationship creation datetime (ascending). +Get a list of a given issue's [related issues](../user/project/issues/related_issues.md), +sorted by the relationship creation datetime (ascending). Issues will be filtered according to the user authorizations. ```plaintext @@ -55,19 +58,20 @@ Parameters: ## Create an issue link -Creates a two-way relation between two issues. User must be allowed to update both issues in order to succeed. +Creates a two-way relation between two issues. The user must be allowed to +update both issues to succeed. ```plaintext POST /projects/:id/issues/:issue_iid/links ``` -| Attribute | Type | Required | Description | -|-------------|---------|----------|--------------------------------------| -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | -| `issue_iid` | integer | yes | The internal ID of a project's issue | +| Attribute | Type | Required | Description | +|---------------------|----------------|----------|--------------------------------------| +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | +| `issue_iid` | integer | yes | The internal ID of a project's issue | | `target_project_id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) of a target project | -| `target_issue_iid` | integer/string | yes | The internal ID of a target project's issue | -| `link_type` | string | no | The type of the relation ("relates_to", "blocks", "is_blocked_by"), defaults to "relates_to"). | +| `target_issue_iid` | integer/string | yes | The internal ID of a target project's issue | +| `link_type` | string | no | The type of the relation ("relates_to", "blocks", "is_blocked_by"), defaults to "relates_to"). | ```shell curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/4/issues/1/links?target_project_id=5&target_issue_iid=1" diff --git a/doc/api/issues.md b/doc/api/issues.md index d8249869cab..b50ea7b42be 100644 --- a/doc/api/issues.md +++ b/doc/api/issues.md @@ -16,7 +16,7 @@ are paginated. Read more on [pagination](README.md#pagination). -CAUTION: **Deprecation:** +DANGER: **Deprecated:** The `reference` attribute in responses is deprecated in favor of `references`. Introduced in [GitLab 12.6](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/20354). @@ -33,19 +33,19 @@ use parameter `scope=all`. ```plaintext GET /issues -GET /issues?state=opened -GET /issues?state=closed +GET /issues?assignee_id=5 +GET /issues?author_id=5 +GET /issues?confidential=true +GET /issues?iids[]=42&iids[]=43 GET /issues?labels=foo GET /issues?labels=foo,bar GET /issues?labels=foo,bar&state=opened GET /issues?milestone=1.0.0 GET /issues?milestone=1.0.0&state=opened -GET /issues?iids[]=42&iids[]=43 -GET /issues?author_id=5 -GET /issues?assignee_id=5 GET /issues?my_reaction_emoji=star GET /issues?search=foo&in=title -GET /issues?confidential=true +GET /issues?state=closed +GET /issues?state=opened ``` | Attribute | Type | Required | Description | @@ -194,7 +194,7 @@ the `health_status` parameter: ] ``` -NOTE: **Note:** +DANGER: **Deprecated:** The `assignee` column is deprecated. We now show it as a single-sized array `assignees` to conform to the GitLab EE API. @@ -212,19 +212,19 @@ The preferred way to do this, is by using [personal access tokens](../user/profi ```plaintext GET /groups/:id/issues -GET /groups/:id/issues?state=opened -GET /groups/:id/issues?state=closed +GET /groups/:id/issues?assignee_id=5 +GET /groups/:id/issues?author_id=5 +GET /groups/:id/issues?confidential=true +GET /groups/:id/issues?iids[]=42&iids[]=43 GET /groups/:id/issues?labels=foo GET /groups/:id/issues?labels=foo,bar GET /groups/:id/issues?labels=foo,bar&state=opened GET /groups/:id/issues?milestone=1.0.0 GET /groups/:id/issues?milestone=1.0.0&state=opened -GET /groups/:id/issues?iids[]=42&iids[]=43 -GET /groups/:id/issues?search=issue+title+or+description -GET /groups/:id/issues?author_id=5 -GET /groups/:id/issues?assignee_id=5 GET /groups/:id/issues?my_reaction_emoji=star -GET /groups/:id/issues?confidential=true +GET /groups/:id/issues?search=issue+title+or+description +GET /groups/:id/issues?state=closed +GET /groups/:id/issues?state=opened ``` | Attribute | Type | Required | Description | @@ -372,7 +372,7 @@ the `health_status` parameter: ] ``` -NOTE: **Note:** +DANGER: **Deprecated:** The `assignee` column is deprecated. We now show it as a single-sized array `assignees` to conform to the GitLab EE API. NOTE: **Note:** @@ -389,19 +389,19 @@ The preferred way to do this, is by using [personal access tokens](../user/profi ```plaintext GET /projects/:id/issues -GET /projects/:id/issues?state=opened -GET /projects/:id/issues?state=closed +GET /projects/:id/issues?assignee_id=5 +GET /projects/:id/issues?author_id=5 +GET /projects/:id/issues?confidential=true +GET /projects/:id/issues?iids[]=42&iids[]=43 GET /projects/:id/issues?labels=foo GET /projects/:id/issues?labels=foo,bar GET /projects/:id/issues?labels=foo,bar&state=opened GET /projects/:id/issues?milestone=1.0.0 GET /projects/:id/issues?milestone=1.0.0&state=opened -GET /projects/:id/issues?iids[]=42&iids[]=43 -GET /projects/:id/issues?search=issue+title+or+description -GET /projects/:id/issues?author_id=5 -GET /projects/:id/issues?assignee_id=5 GET /projects/:id/issues?my_reaction_emoji=star -GET /projects/:id/issues?confidential=true +GET /projects/:id/issues?search=issue+title+or+description +GET /projects/:id/issues?state=closed +GET /projects/:id/issues?state=opened ``` | Attribute | Type | Required | Description | @@ -555,7 +555,7 @@ the `health_status` parameter: ] ``` -NOTE: **Note:** +DANGER: **Deprecated:** The `assignee` column is deprecated. We now show it as a single-sized array `assignees` to conform to the GitLab EE API. NOTE: **Note:** @@ -574,7 +574,7 @@ GET /issues/:id | Attribute | Type | Required | Description | |-------------|---------|----------|--------------------------------------| -| `id` | integer | yes | The ID of the issue | +| `id` | integer | yes | The ID of the issue | ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/issues/41" @@ -663,10 +663,10 @@ Example response: "weight": null, "has_tasks": false, "_links": { - "self": "http://gitlab.dummy:3000/api/v4/projects/1/issues/1", - "notes": "http://gitlab.dummy:3000/api/v4/projects/1/issues/1/notes", - "award_emoji": "http://gitlab.dummy:3000/api/v4/projects/1/issues/1/award_emoji", - "project": "http://gitlab.dummy:3000/api/v4/projects/1" + "self": "http://gitlab.example:3000/api/v4/projects/1/issues/1", + "notes": "http://gitlab.example:3000/api/v4/projects/1/issues/1/notes", + "award_emoji": "http://gitlab.example:3000/api/v4/projects/1/issues/1/award_emoji", + "project": "http://gitlab.example:3000/api/v4/projects/1" }, "references": { "short": "#1", @@ -712,19 +712,19 @@ the `epic` property: } ``` -NOTE: **Note:** +DANGER: **Deprecated:** The `assignee` column is deprecated. We now show it as a single-sized array `assignees` to conform to the GitLab EE API. +DANGER: **Deprecated:** +The `epic_iid` attribute is deprecated, and [will be removed in version 5](https://gitlab.com/gitlab-org/gitlab/-/issues/35157). +Please use `iid` of the `epic` attribute instead. + NOTE: **Note:** The `closed_by` attribute was [introduced in GitLab 10.6](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/17042). This value is only present for issues closed after GitLab 10.6 and if the user account that closed the issue still exists. -NOTE: **Note:** -The `epic_iid` attribute is deprecated, and [will be removed in version 5](https://gitlab.com/gitlab-org/gitlab/-/issues/35157). -Please use `iid` of the `epic` attribute instead. - ## Single project issue Get a single project issue. @@ -874,17 +874,17 @@ property: ] ``` -NOTE: **Note:** +DANGER: **Deprecated:** The `assignee` column is deprecated. We now show it as a single-sized array `assignees` to conform to the GitLab EE API. +DANGER: **Deprecated:** +The `epic_iid` attribute is deprecated and [will be removed in version 5](https://gitlab.com/gitlab-org/gitlab/-/issues/35157). +Please use `iid` of the `epic` attribute instead. + NOTE: **Note:** The `closed_by` attribute was [introduced in GitLab 10.6](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/17042). This value is only present for issues closed after GitLab 10.6 and if the user account that closed the issue still exists. -NOTE: **Note:** -The `epic_iid` attribute is deprecated and [will be removed in version 5](https://gitlab.com/gitlab-org/gitlab/-/issues/35157). -Please use `iid` of the `epic` attribute instead. - ## New issue Creates a new project issue. @@ -895,21 +895,21 @@ POST /projects/:id/issues | Attribute | Type | Required | Description | |-------------------------------------------|----------------|----------|--------------| +| `assignee_ids` | integer array | no | The ID of the user(s) to assign the issue to. | +| `confidential` | boolean | no | Set an issue to be confidential. Default is `false`. | +| `created_at` | string | no | When the issue was created. Date time string, ISO 8601 formatted, for example `2016-03-11T03:45:40Z`. Requires administrator or project/group owner rights. | +| `description` | string | no | The description of an issue. Limited to 1,048,576 characters. | +| `discussion_to_resolve` | string | no | The ID of a discussion to resolve. This fills out the issue with a default description and mark the discussion as resolved. Use in combination with `merge_request_to_resolve_discussions_of`. | +| `due_date` | string | no | The due date. Date time string in the format YEAR-MONTH-DAY, for example `2016-03-11` | +| `epic_id` **(PREMIUM)** | integer | no | ID of the epic to add the issue to. Valid values are greater than or equal to 0. | +| `epic_iid` **(PREMIUM)** | integer | no | IID of the epic to add the issue to. Valid values are greater than or equal to 0. (deprecated, [will be removed in version 5](https://gitlab.com/gitlab-org/gitlab/-/issues/35157)) | | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | | `iid` | integer/string | no | The internal ID of the project's issue (requires administrator or project owner rights) | -| `title` | string | yes | The title of an issue | -| `description` | string | no | The description of an issue. Limited to 1,048,576 characters. | -| `confidential` | boolean | no | Set an issue to be confidential. Default is `false`. | -| `assignee_ids` | integer array | no | The ID of the user(s) to assign the issue to. | -| `milestone_id` | integer | no | The global ID of a milestone to assign issue | | `labels` | string | no | Comma-separated label names for an issue | -| `created_at` | string | no | Date time string, ISO 8601 formatted, for example `2016-03-11T03:45:40Z` (requires administrator or project/group owner rights) | -| `due_date` | string | no | Date time string in the format YEAR-MONTH-DAY, for example `2016-03-11` | | `merge_request_to_resolve_discussions_of` | integer | no | The IID of a merge request in which to resolve all issues. This fills out the issue with a default description and mark all discussions as resolved. When passing a description or title, these values take precedence over the default values.| -| `discussion_to_resolve` | string | no | The ID of a discussion to resolve. This fills out the issue with a default description and mark the discussion as resolved. Use in combination with `merge_request_to_resolve_discussions_of`. | +| `milestone_id` | integer | no | The global ID of a milestone to assign issue | +| `title` | string | yes | The title of an issue | | `weight` **(STARTER)** | integer | no | The weight of the issue. Valid values are greater than or equal to 0. | -| `epic_id` **(PREMIUM)** | integer | no | ID of the epic to add the issue to. Valid values are greater than or equal to 0. | -| `epic_iid` **(PREMIUM)** | integer | no | IID of the epic to add the issue to. Valid values are greater than or equal to 0. (deprecated, [will be removed in version 5](https://gitlab.com/gitlab-org/gitlab/-/issues/35157)) | ```shell curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/4/issues?title=Issues%20with%20auth&labels=bug" @@ -1002,7 +1002,7 @@ the `health_status` parameter: ] ``` -NOTE: **Note:** +DANGER: **Deprecated:** The `assignee` column is deprecated. We now show it as a single-sized array `assignees` to conform to the GitLab EE API. NOTE: **Note:** @@ -1019,29 +1019,43 @@ See [Issues rate limits](../user/admin_area/settings/rate_limit_on_issues_creati Updates an existing project issue. This call is also used to mark an issue as closed. +At least one of the following parameters is required for the request to be successful: + +- `:assignee_id` +- `:assignee_ids` +- `:confidential` +- `:created_at` +- `:description` +- `:discussion_locked` +- `:due_date` +- `:labels` +- `:milestone_id` +- `:state_event` +- `:title` + ```plaintext PUT /projects/:id/issues/:issue_iid ``` | Attribute | Type | Required | Description | |----------------|---------|----------|------------------------------------------------------------------------------------------------------------| +| `add_labels` | string | no | Comma-separated label names to add to an issue. | +| `assignee_ids` | integer array | no | The ID of the user(s) to assign the issue to. Set to `0` or provide an empty value to unassign all assignees. | +| `confidential` | boolean | no | Updates an issue to be confidential | +| `description` | string | no | The description of an issue. Limited to 1,048,576 characters. | +| `discussion_locked` | boolean | no | Flag indicating if the issue's discussion is locked. If the discussion is locked only project members can add or edit comments. | +| `due_date` | string | no | The due date. Date time string in the format YEAR-MONTH-DAY, for example `2016-03-11` | +| `epic_id` **(PREMIUM)** | integer | no | ID of the epic to add the issue to. Valid values are greater than or equal to 0. | +| `epic_iid` **(PREMIUM)** | integer | no | IID of the epic to add the issue to. Valid values are greater than or equal to 0. (deprecated, [will be removed in version 5](https://gitlab.com/gitlab-org/gitlab/-/issues/35157)) | | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | | `issue_iid` | integer | yes | The internal ID of a project's issue | -| `title` | string | no | The title of an issue | -| `description` | string | no | The description of an issue. Limited to 1,048,576 characters. | -| `confidential` | boolean | no | Updates an issue to be confidential | -| `assignee_ids` | integer array | no | The ID of the user(s) to assign the issue to. Set to `0` or provide an empty value to unassign all assignees. | -| `milestone_id` | integer | no | The global ID of a milestone to assign the issue to. Set to `0` or provide an empty value to unassign a milestone.| | `labels` | string | no | Comma-separated label names for an issue. Set to an empty string to unassign all labels. | -| `add_labels` | string | no | Comma-separated label names to add to an issue. | +| `milestone_id` | integer | no | The global ID of a milestone to assign the issue to. Set to `0` or provide an empty value to unassign a milestone.| | `remove_labels`| string | no | Comma-separated label names to remove from an issue. | | `state_event` | string | no | The state event of an issue. Set `close` to close the issue and `reopen` to reopen it | -| `updated_at` | string | no | Date time string, ISO 8601 formatted, for example `2016-03-11T03:45:40Z` (requires administrator or project owner rights). Empty string or null values are not accepted.| -| `due_date` | string | no | Date time string in the format YEAR-MONTH-DAY, for example `2016-03-11` | +| `title` | string | no | The title of an issue | +| `updated_at` | string | no | When the issue was updated. Date time string, ISO 8601 formatted, for example `2016-03-11T03:45:40Z` (requires administrator or project owner rights). Empty string or null values are not accepted.| | `weight` **(STARTER)** | integer | no | The weight of the issue. Valid values are greater than or equal to 0. 0 | -| `discussion_locked` | boolean | no | Flag indicating if the issue's discussion is locked. If the discussion is locked only project members can add or edit comments. | -| `epic_id` **(PREMIUM)** | integer | no | ID of the epic to add the issue to. Valid values are greater than or equal to 0. | -| `epic_iid` **(PREMIUM)** | integer | no | IID of the epic to add the issue to. Valid values are greater than or equal to 0. (deprecated, [will be removed in version 5](https://gitlab.com/gitlab-org/gitlab/-/issues/35157)) | ```shell curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/4/issues/85?state_event=close" @@ -1142,15 +1156,12 @@ the `health_status` parameter: ``` NOTE: **Note:** -At least one of following parameters is required to be passed for the request to be successful: `:assignee_id`, `:assignee_ids`, `:confidential`, `:created_at`, `:description`, `:discussion_locked`, `:due_date`, `:labels`, `:milestone_id`, `:state_event`, or `:title`. - -NOTE: **Note:** -`assignee` column is deprecated. We now show it as a single-sized array `assignees` to conform to the GitLab EE API. - -NOTE: **Note:** The `closed_by` attribute was [introduced in GitLab 10.6](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/17042). This value is only present for issues closed after GitLab 10.6 and if the user account that closed the issue still exists. +DANGER: **Deprecated:** +`assignee` column is deprecated. We now show it as a single-sized array `assignees` to conform to the GitLab EE API. + ## Delete an issue Only for admins and project owners. Deletes the issue in question. @@ -1309,7 +1320,7 @@ the `health_status` parameter: ] ``` -NOTE: **Note:** +DANGER: **Deprecated:** The `assignee` column is deprecated. We now show it as a single-sized array `assignees` to conform to the GitLab EE API. NOTE: **Note:** @@ -1418,7 +1429,7 @@ the `weight` parameter: } ``` -NOTE: **Note:** +DANGER: **Deprecated:** The `assignee` column is deprecated. We now show it as a single-sized array `assignees` to conform to the GitLab EE API. NOTE: **Note:** @@ -1496,10 +1507,10 @@ Example response: } ``` -## Create a to-do +## Create a to do -Manually creates a to-do for the current user on an issue. If -there already exists a to-do for the user on that issue, status code `304` is +Manually creates a to do for the current user on an issue. If +there already exists a to do for the user on that issue, status code `304` is returned. ```plaintext @@ -1608,7 +1619,7 @@ Example response: } ``` -NOTE: **Note:** +DANGER: **Deprecated:** The `assignee` column is deprecated. We now show it as a single-sized array `assignees` to conform to the GitLab EE API. NOTE: **Note:** @@ -1625,9 +1636,9 @@ POST /projects/:id/issues/:issue_iid/time_estimate | Attribute | Type | Required | Description | |-------------|---------|----------|------------------------------------------| +| `duration` | string | yes | The duration in human format. e.g: 3h30m | | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | | `issue_iid` | integer | yes | The internal ID of a project's issue | -| `duration` | string | yes | The duration in human format. e.g: 3h30m | ```shell curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/issues/93/time_estimate?duration=3h30m" @@ -1682,9 +1693,9 @@ POST /projects/:id/issues/:issue_iid/add_spent_time | Attribute | Type | Required | Description | |-------------|---------|----------|------------------------------------------| +| `duration` | string | yes | The duration in human format. e.g: 3h30m | | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | | `issue_iid` | integer | yes | The internal ID of a project's issue | -| `duration` | string | yes | The duration in human format. e.g: 3h30m | ```shell curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/issues/93/add_spent_time?duration=1h" diff --git a/doc/api/iterations.md b/doc/api/iterations.md new file mode 100644 index 00000000000..53a6bb00f23 --- /dev/null +++ b/doc/api/iterations.md @@ -0,0 +1,57 @@ +--- +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Project iterations API **(STARTER)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/118742) in [GitLab Starter](https://about.gitlab.com/pricing/) 13.5. + +This page describes the project iterations API. +There's a separate [group iterations API](./group_iterations.md) page. + +As of GitLab 13.5, we don't have project-level iterations, but you can use this endpoint to fetch the iterations of the project's ancestor groups. + +## List project iterations + +Returns a list of project iterations. + +```plaintext +GET /projects/:id/iterations +GET /projects/:id/iterations?state=opened +GET /projects/:id/iterations?state=closed +GET /projects/:id/iterations?title=1.0 +GET /projects/:id/iterations?search=version +``` + +| Attribute | Type | Required | Description | +| ------------------- | ------- | -------- | ----------- | +| `state` | string | no | Return only `opened`, `upcoming`, `started`, `closed`, or `all` iterations. Defaults to `all`. | +| `search` | string | no | Return only iterations with a title matching the provided string. | +| `include_ancestors` | boolean | no | Include iterations from parent group and its ancestors. Defaults to `true`. | + +Example request: + +```shell +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/iterations" +``` + +Example response: + +```json +[ + { + "id": 53, + "iid": 13, + "group_id": 5, + "title": "Iteration II", + "description": "Ipsum Lorem ipsum", + "state": 2, + "created_at": "2020-01-27T05:07:12.573Z", + "updated_at": "2020-01-27T05:07:12.573Z", + "due_date": "2020-02-01", + "start_date": "2020-02-14" + } +] +``` diff --git a/doc/api/job_artifacts.md b/doc/api/job_artifacts.md index 458877d6548..f5510f6ee91 100644 --- a/doc/api/job_artifacts.md +++ b/doc/api/job_artifacts.md @@ -63,6 +63,11 @@ the given reference name and job, provided the job finished successfully. This is the same as [getting the job's artifacts](#get-job-artifacts), but by defining the job's name instead of its ID. +NOTE: **Note:** +If a pipeline is [parent of other child pipelines](../ci/parent_child_pipelines.md), artifacts +are searched in hierarchical order from parent to child. For example, if both parent and +child pipelines have a job with the same name, the artifact from the parent pipeline will be returned. + ```plaintext GET /projects/:id/jobs/artifacts/:ref_name/download?job=name ``` @@ -157,6 +162,11 @@ Download a single artifact file for a specific job of the latest successful pipeline for the given reference name from within the job's artifacts archive. The file is extracted from the archive and streamed to the client. +In [GitLab 13.5](https://gitlab.com/gitlab-org/gitlab/-/issues/201784) and later, artifacts +for [parent and child pipelines](../ci/parent_child_pipelines.md) are searched in hierarchical +order from parent to child. For example, if both parent and child pipelines have a +job with the same name, the artifact from the parent pipeline is returned. + ```plaintext GET /projects/:id/jobs/artifacts/:ref_name/raw/*artifact_path?job=name ``` diff --git a/doc/api/license.md b/doc/api/license.md index 71e95fc3202..dcdf019059b 100644 --- a/doc/api/license.md +++ b/doc/api/license.md @@ -1,7 +1,7 @@ # License **(CORE ONLY)** -In order to interact with license endpoints, you need to authenticate yourself -as an admin. +To interact with license endpoints, you need to authenticate yourself as an +admin. ## Retrieve information about the current license diff --git a/doc/api/lint.md b/doc/api/lint.md index f4d8a0bc011..c82e0845f99 100644 --- a/doc/api/lint.md +++ b/doc/api/lint.md @@ -4,11 +4,14 @@ group: Continuous Integration info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# Validate the `.gitlab-ci.yml` (API) +# CI Lint API + +## Validate the CI YAML configuration > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/5953) in GitLab 8.12. -Checks if your `.gitlab-ci.yml` file is valid. +Checks if CI/CD YAML configuration is valid. This endpoint validates basic CI/CD +configuration syntax. It doesn't have any namespace specific context. ```plaintext POST /ci/lint @@ -16,13 +19,15 @@ POST /ci/lint | Attribute | Type | Required | Description | | ---------- | ------- | -------- | -------- | -| `content` | string | yes | the `.gitlab-ci.yaml` content| +| `content` | string | yes | The CI/CD configuration content. | +| `include_merged_yaml` | boolean | no | If the [expanded CI/CD configuration](#yaml-expansion) should be included in the response. | ```shell curl --header "Content-Type: application/json" "https://gitlab.example.com/api/v4/ci/lint" --data '{"content": "{ \"image\": \"ruby:2.6\", \"services\": [\"postgres\"], \"before_script\": [\"bundle install\", \"bundle exec rake db:create\"], \"variables\": {\"DB_NAME\": \"postgres\"}, \"types\": [\"test\", \"deploy\", \"notify\"], \"rspec\": { \"script\": \"rake spec\", \"tags\": [\"ruby\", \"postgres\"], \"only\": [\"branches\"]}}"}' ``` -Be sure to copy paste the exact contents of `.gitlab-ci.yml` as YAML is very picky about indentation and spaces. +Be sure to paste the exact contents of your GitLab CI/CD YAML configuration because YAML +is very sensitive about indentation and spacing. Example responses: @@ -53,3 +58,180 @@ Example responses: "error": "content is missing" } ``` + +### YAML expansion + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/29568) in GitLab 13.5. + +The CI lint returns an expanded version of the configuration. The expansion does not +work for CI configuration added with [`include: local`](../ci/yaml/README.md#includelocal), +or with [`extends:`](../ci/yaml/README.md#extends). + +Example contents of a `.gitlab-ci.yml` passed to the CI Lint API with +`include_merged_yaml` set as true: + +```yaml +include: + remote: 'https://example.com/remote.yaml' + +test: + stage: test + script: + - echo 1 +``` + +Example contents of `https://example.com/remote.yaml`: + +```yaml +another_test: + stage: test + script: + - echo 2 +``` + +Example response: + +```json +{ + "status": "valid", + "errors": [], + "merged_config": "---\n:another_test:\n :stage: test\n :script: echo 2\n:test:\n :stage: test\n :script: echo 1\n" +} +``` + +## Validate a project's CI configuration + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/231352) in GitLab 13.5. + +Checks if a project's latest (`HEAD` of the project's default branch) +`.gitlab-ci.yml` configuration is valid. This endpoint uses all namespace +specific data available, including variables, local includes, and so on. + +```plaintext +GET /projects/:id/ci/lint +``` + +| Attribute | Type | Required | Description | +| ---------- | ------- | -------- | -------- | +| `dry_run` | boolean | no | Run pipeline creation simulation, or only do static check. | + +Example request: + +```shell +curl "https://gitlab.example.com/api/v4/projects/:id/ci/lint" +``` + +Example responses: + +- Valid configuration: + +```json +{ + "valid": true, + "merged_yaml": "---\n:test_job:\n :script: echo 1\n", + "errors": [], + "warnings": [] +} +``` + +- Invalid configuration: + +```json +{ + "valid": false, + "merged_yaml": "---\n:test_job:\n :script: echo 1\n", + "errors": [ + "jobs config should contain at least one visible job" + ], + "warnings": [] +} +``` + +## Use jq to create and process YAML & JSON payloads + +To `POST` a YAML configuration to the CI Lint endpoint, it must be properly escaped and JSON encoded. +You can use `jq` and `curl` to escape and upload YAML to the GitLab API. + +### Escape YAML for JSON encoding + +To escape quotes and encode your YAML in a format suitable for embedding within +a JSON payload, you can use `jq`. For example, create a file named `example-gitlab-ci.yml`: + +```yaml +.api_test: + rules: + - if: '$CI_PIPELINE_SOURCE=="merge_request_event"' + changes: + - src/api/* +deploy: + extends: + - .api_test + rules: + - when: manual + allow_failure: true + script: + - echo "hello world" +``` + +Next, use `jq` to escape and encode the YAML file into JSON: + +```shell +jq --raw-input --slurp < example-gitlab-ci.yml +``` + +To escape and encode an input YAML file (`example-gitlab-ci.yml`), and `POST` it to the +GitLab API using `curl` and `jq` in a one-line command: + +```shell +jq --null-input --arg yaml "$(<example-gitlab-ci.yml)" '.content=$yaml' \ +| curl 'https://gitlab.com/api/v4/ci/lint?include_merged_yaml=true' \ +--header 'Content-Type: application/json' \ +--data @- +``` + +### Parse a CI Lint response + +To reformat the CI Lint response, you can use `jq`. You can pipe the CI Lint response to `jq`, +or store the API response as a text file and provide it as an argument: + +```shell +jq --raw-output '.merged_yaml | fromjson' <your_input_here> +``` + +Example input: + +```json +{"status":"valid","errors":[],"merged_yaml":"---\n:.api_test:\n :rules:\n - :if: $CI_PIPELINE_SOURCE==\"merge_request_event\"\n :changes:\n - src/api/*\n:deploy:\n :rules:\n - :when: manual\n :allow_failure: true\n :extends:\n - \".api_test\"\n :script:\n - echo \"hello world\"\n"} +``` + +Becomes: + +```yaml +:.api_test: + :rules: + - :if: $CI_PIPELINE_SOURCE=="merge_request_event" + :changes: + - src/api/* +:deploy: + :rules: + - :when: manual + :allow_failure: true + :extends: + - ".api_test" + :script: + - echo "hello world" +``` + +With a one-line command, you can: + +1. Escape the YAML +1. Encode it in JSON +1. POST it to the API with curl +1. Format the response + +```shell +jq --null-input --arg yaml "$(<example-gitlab-ci.yml)" '.content=$yaml' \ +| curl 'https://gitlab.com/api/v4/ci/lint?include_merged_yaml=true' \ +--header 'Content-Type: application/json' --data @- \ +| jq --raw-output '.merged_yaml | fromjson' +``` diff --git a/doc/api/members.md b/doc/api/members.md index 76d63b277c4..4440b70c512 100644 --- a/doc/api/members.md +++ b/doc/api/members.md @@ -180,6 +180,7 @@ Example response: "web_url": "http://192.168.1.8:3000/root", "access_level": 30, "email": "john@example.com", + "created_at": "2012-10-22T14:13:35Z", "expires_at": null, "group_saml_identity": null } @@ -223,6 +224,61 @@ Example response: } ``` +## List all billable members of a group + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/217384) in GitLab 13.5. + +Gets a list of group members that count as billable. The list includes members in the subgroup or subproject. + +NOTE: +Unlike other API endpoints, billable members is updated once per day at 12:00 UTC. + +This function takes [pagination](README.md#pagination) parameters `page` and `per_page` to restrict the list of users. + +```plaintext +GET /groups/:id/billable_members +``` + +| Attribute | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) owned by the authenticated user | + +```shell +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/:id/billable_members" +``` + +Example response: + +```json +[ + { + "id": 1, + "username": "raymond_smith", + "name": "Raymond Smith", + "state": "active", + "avatar_url": "https://www.gravatar.com/avatar/c2525a7f58ae3776070e44c106c48e15?s=80&d=identicon", + "web_url": "http://192.168.1.8:3000/root", + }, + { + "id": 2, + "username": "john_doe", + "name": "John Doe", + "state": "active", + "avatar_url": "https://www.gravatar.com/avatar/c2525a7f58ae3776070e44c106c48e15?s=80&d=identicon", + "web_url": "http://192.168.1.8:3000/root", + "email": "john@example.com" + }, + { + "id": 3, + "username": "foo_bar", + "name": "Foo bar", + "state": "active", + "avatar_url": "https://www.gravatar.com/avatar/c2525a7f58ae3776070e44c106c48e15?s=80&d=identicon", + "web_url": "http://192.168.1.8:3000/root" + } +] +``` + ## Add a member to a group or project Adds a member to a group or project. @@ -235,7 +291,7 @@ POST /projects/:id/members | Attribute | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `id` | integer/string | yes | The ID or [URL-encoded path of the project or group](README.md#namespaced-path-encoding) owned by the authenticated user | -| `user_id` | integer | yes | The user ID of the new member | +| `user_id` | integer/string | yes | The user ID of the new member or multiple IDs separated by commas | | `access_level` | integer | yes | A valid access level | | `expires_at` | string | no | A date string in the format YEAR-MONTH-DAY | diff --git a/doc/api/merge_request_approvals.md b/doc/api/merge_request_approvals.md index 643d03b6fb8..89e4224c735 100644 --- a/doc/api/merge_request_approvals.md +++ b/doc/api/merge_request_approvals.md @@ -60,7 +60,7 @@ POST /projects/:id/approvals | `disable_overriding_approvers_per_merge_request` | boolean | no | Allow/Disallow overriding approvers per MR | | `merge_requests_author_approval` | boolean | no | Allow/Disallow authors from self approving merge requests; `true` means authors can self approve | | `merge_requests_disable_committers_approval` | boolean | no | Allow/Disallow committers from self approving merge requests | -| `require_password_to_approve` | boolean | no | Require approver to enter a password in order to authenticate before adding the approval | +| `require_password_to_approve` | boolean | no | Require approver to enter a password to authenticate before adding the approval | ```json { diff --git a/doc/api/merge_requests.md b/doc/api/merge_requests.md index faefc445210..194f48c6e84 100644 --- a/doc/api/merge_requests.md +++ b/doc/api/merge_requests.md @@ -72,6 +72,9 @@ Parameters: | `in` | string | no | Modify the scope of the `search` attribute. `title`, `description`, or a string joining them with comma. Default is `title,description` | | `wip` | string | no | Filter merge requests against their `wip` status. `yes` to return *only* WIP merge requests, `no` to return *non* WIP merge requests | | `not` | Hash | no | Return merge requests that do not match the parameters supplied. Accepts: `labels`, `milestone`, `author_id`, `author_username`, `assignee_id`, `assignee_username`, `my_reaction_emoji` | +| `environment` | string | no | Returns merge requests deployed to the given environment +| `deployed_before` | datetime | no | Return merge requests deployed before the given date/time +| `deployed_after` | datetime | no | Return merge requests deployed after the given date/time NOTE: **Note:** [Starting in GitLab 13.0](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/31890), @@ -1004,6 +1007,7 @@ order for it to take effect: value of zero disables approvals for that project. 1. The provided value of `approvals_before_merge` must be greater than the target project's `approvals_before_merge`. +1. This API returns 201 (created) for a successful response. ```json { @@ -1310,7 +1314,7 @@ curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://git Merge changes submitted with MR using this API. -If merge request is unable to be accepted (ie: Draft, Closed, Pipeline Pending Completion, or Failed while requiring Success) - you'll get a `405` and the error message 'Method Not Allowed' +If merge request is unable to be accepted (such as Draft, Closed, Pipeline Pending Completion, or Failed while requiring Success) - you'll get a `405` and the error message 'Method Not Allowed' If it has some conflicts and can not be merged - you'll get a `406` and the error message 'Branch cannot be merged' @@ -2085,10 +2089,10 @@ the `approvals_before_merge` parameter: } ``` -## Create a to-do +## Create a to do -Manually creates a to-do for the current user on a merge request. -If there already exists a to-do for the user on that merge request, +Manually creates a to do for the current user on a merge request. +If there already exists a to do for the user on that merge request, status code `304` is returned. ```plaintext diff --git a/doc/api/metrics_dashboard_annotations.md b/doc/api/metrics_dashboard_annotations.md index 10dfd3d1c3b..c23ed657583 100644 --- a/doc/api/metrics_dashboard_annotations.md +++ b/doc/api/metrics_dashboard_annotations.md @@ -1,6 +1,6 @@ --- stage: Monitor -group: APM +group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: concepts, howto --- @@ -18,14 +18,11 @@ POST /environments/:id/metrics_dashboard/annotations/ POST /clusters/:id/metrics_dashboard/annotations/ ``` -NOTE: **Note:** -The value of `dashboard_path` will be treated as a CGI-escaped path, and automatically un-escaped. - Parameters: | Attribute | Type | Required | Description | |:---------------|:---------------|:---------|:-----------------------------------------------------------------------------| -| `dashboard_path` | string | yes | ID of the dashboard which needs to be annotated. | +| `dashboard_path` | string | yes | ID of the dashboard which needs to be annotated. Treated as a CGI-escaped path, and automatically un-escaped. | | `starting_at` | string | yes | Date time string, ISO 8601 formatted, such as `2016-03-11T03:45:40Z`. Timestamp marking start point of annotation. | | `ending_at` | string | no | Date time string, ISO 8601 formatted, such as `2016-03-11T03:45:40Z`. Timestamp marking end point of annotation. When not supplied annotation will be displayed as single event at start point. | | `description` | string | yes | Description of the annotation. | diff --git a/doc/api/metrics_user_starred_dashboards.md b/doc/api/metrics_user_starred_dashboards.md index df9cdd3b0e4..8c2894293ba 100644 --- a/doc/api/metrics_user_starred_dashboards.md +++ b/doc/api/metrics_user_starred_dashboards.md @@ -1,6 +1,6 @@ --- stage: Monitor -group: APM +group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: concepts, howto --- diff --git a/doc/api/namespaces.md b/doc/api/namespaces.md index ba59d467bc8..0792c6d4a3b 100644 --- a/doc/api/namespaces.md +++ b/doc/api/namespaces.md @@ -87,6 +87,26 @@ the `plan` parameter associated with a namespace: ] ``` +Users on GitLab.com will also see `max_seats_used` and `seats_in_use` parameters. +`max_seats_used` is the highest number of users the group had. `seats_in_use` is +the number of license seats currently being used. Both values are updated +once a day. + +`max_seats_used` and `seats_in_use` will be non-zero only for namespaces on paid plans. + +```json +[ + { + "id": 1, + "name": "user1", + "billable_members_count": 2, + "max_seats_used": 3, + "seats_in_use": 2, + ... + } +] +``` + NOTE: **Note:** Only group maintainers/owners are presented with `members_count_with_descendants`, as well as `plan` **(BRONZE ONLY)**. @@ -123,6 +143,8 @@ Example response: "web_url": "https://gitlab.example.com/groups/twitter", "members_count_with_descendants": 2, "billable_members_count": 2, + "max_seats_used": 0, + "seats_in_use": 0, "plan": "default", "trial_ends_on": null, "trial": false @@ -162,6 +184,8 @@ Example response: "web_url": "https://gitlab.example.com/groups/group1", "members_count_with_descendants": 2, "billable_members_count": 2, + "max_seats_used": 0, + "seats_in_use": 0, "plan": "default", "trial_ends_on": null, "trial": false @@ -188,6 +212,8 @@ Example response: "web_url": "https://gitlab.example.com/groups/group1", "members_count_with_descendants": 2, "billable_members_count": 2, + "max_seats_used": 0, + "seats_in_use": 0, "plan": "default", "trial_ends_on": null, "trial": false diff --git a/doc/api/openapi/openapi.yaml b/doc/api/openapi/openapi.yaml index 8aa4de62501..8c46804d86f 100644 --- a/doc/api/openapi/openapi.yaml +++ b/doc/api/openapi/openapi.yaml @@ -9,9 +9,9 @@ info: When viewing this on gitlab.com, you can test API calls directly from the browser against the `gitlab.com` instance, if you are logged in. The feature uses the current [GitLab session cookie](https://docs.gitlab.com/ee/api/README.html#session-cookie), - so each request is made using your account. + so each request is made using your account. - Read more at <https://docs.gitlab.com/ee/development/documentation/styleguide.html#restful-api>. + Read more at <https://docs.gitlab.com/ee/development/documentation/restful_api_styleguide.html>. version: "v4" title: "GitLab API" termsOfService: "https://about.gitlab.com/terms/" diff --git a/doc/api/packages.md b/doc/api/packages.md index cf65b518844..d4e69b9bc66 100644 --- a/doc/api/packages.md +++ b/doc/api/packages.md @@ -26,7 +26,7 @@ GET /projects/:id/packages | `id` | integer/string | yes | ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) | | `order_by`| string | no | The field to use as order. One of `created_at` (default), `name`, `version`, or `type`. | | `sort` | string | no | The direction of the order, either `asc` (default) for ascending order or `desc` for descending order. | -| `package_type` | string | no | Filter the returned packages by type. One of `conan`, `maven`, `npm`, `pypi`, `composer`, or `nuget`. (_Introduced in GitLab 12.9_) +| `package_type` | string | no | Filter the returned packages by type. One of `conan`, `maven`, `npm`, `pypi`, `composer`, `nuget`, or `golang`. (_Introduced in GitLab 12.9_) | `package_name` | string | no | Filter the project packages with a fuzzy search by name. (_Introduced in GitLab 12.9_) ```shell @@ -50,6 +50,19 @@ Example response: "version": "1.0.3", "package_type": "npm", "created_at": "2019-11-27T03:37:38.711Z" + }, + { + "id": 3, + "name": "Hello/0.1@mycompany/stable", + "conan_package_name": "Hello", + "version": "0.1", + "package_type": "conan", + "_links": { + "web_path": "/foo/bar/-/packages/3", + "delete_api_path": "https://gitlab.example.com/api/v4/projects/1/packages/3" + }, + "created_at": "2029-12-16T20:33:34.316Z", + "tags": [] } ] ``` @@ -73,7 +86,7 @@ GET /groups/:id/packages | `exclude_subgroups` | boolean | false | If the parameter is included as true, packages from projects from subgroups are not listed. Default is `false`. | | `order_by`| string | no | The field to use as order. One of `created_at` (default), `name`, `version`, `type`, or `project_path`. | | `sort` | string | no | The direction of the order, either `asc` (default) for ascending order or `desc` for descending order. | -| `package_type` | string | no | Filter the returned packages by type. One of `conan`, `maven`, `npm`, `pypi`, `composer`, or `nuget`. (_Introduced in GitLab 12.9_) | +| `package_type` | string | no | Filter the returned packages by type. One of `conan`, `maven`, `npm`, `pypi`, `composer`, `nuget`, or `golang`. (_Introduced in GitLab 12.9_) | | `package_name` | string | no | Filter the project packages with a fuzzy search by name. (_[Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/30980) in GitLab 13.0_) ```shell diff --git a/doc/api/personal_access_tokens.md b/doc/api/personal_access_tokens.md index 517e26f3d85..43310570fe8 100644 --- a/doc/api/personal_access_tokens.md +++ b/doc/api/personal_access_tokens.md @@ -24,7 +24,7 @@ curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/a ``` ```json -[ +[ { "id": 4, "name": "Test Token", @@ -45,7 +45,7 @@ curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/a ``` ```json -[ +[ { "id": 4, "name": "Test Token", diff --git a/doc/api/project_badges.md b/doc/api/project_badges.md index 936bd40d1ee..b8ea55ab72f 100644 --- a/doc/api/project_badges.md +++ b/doc/api/project_badges.md @@ -16,7 +16,7 @@ Badges support placeholders that will be replaced in real time in both the link - **%{project_path}**: will be replaced by the project path. - **%{project_id}**: will be replaced by the project ID. - **%{default_branch}**: will be replaced by the project default branch. -- **%{commit_sha}**: will be replaced by the last project's commit sha. +- **%{commit_sha}**: will be replaced by the last project's commit SHA. ## List all badges of a project diff --git a/doc/api/project_clusters.md b/doc/api/project_clusters.md index 04694157561..ce175184179 100644 --- a/doc/api/project_clusters.md +++ b/doc/api/project_clusters.md @@ -8,8 +8,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/23922) in GitLab 11.7. -NOTE: **Note:** -User will need at least maintainer access to use these endpoints. +Users need at least [Maintainer](../user/permissions.md) access to use these endpoints. ## List project clusters diff --git a/doc/api/project_repository_storage_moves.md b/doc/api/project_repository_storage_moves.md index 2010fccc624..b490b6235b1 100644 --- a/doc/api/project_repository_storage_moves.md +++ b/doc/api/project_repository_storage_moves.md @@ -194,7 +194,7 @@ Parameters: | Attribute | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `project_id` | integer | yes | ID of the project | -| `destination_storage_name` | string | yes | Name of the destination storage shard | +| `destination_storage_name` | string | no | Name of the destination storage shard. If not provided the storage will be selected automatically. | Example request: diff --git a/doc/api/project_snippets.md b/doc/api/project_snippets.md index eccc8b4212d..cc8bb20b003 100644 --- a/doc/api/project_snippets.md +++ b/doc/api/project_snippets.md @@ -83,12 +83,17 @@ POST /projects/:id/snippets Parameters: -- `id` (required) - The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user -- `title` (required) - The title of a snippet -- `file_name` (required) - The name of a snippet file -- `description` (optional) - The description of a snippet -- `content` (required) - The content of a snippet -- `visibility` (required) - The snippet's visibility +| Attribute | Type | Required | Description | +|:------------------|:----------------|:---------|:----------------------------------------------------------------------------------------------------------------| +| `id` | integer | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | +| `title` | string | yes | Title of a snippet | +| `file_name` | string | no | Deprecated: Use `files` instead. Name of a snippet file | +| `content` | string | no | Deprecated: Use `files` instead. Content of a snippet | +| `description` | string | no | Description of a snippet | +| `visibility` | string | no | Snippet's [visibility](#snippet-visibility-level) | +| `files` | array of hashes | no | An array of snippet files | +| `files:file_path` | string | yes | File path of the snippet file | +| `files:content` | string | yes | Content of the snippet file | Example request: @@ -105,9 +110,13 @@ curl --request POST "https://gitlab.com/api/v4/projects/:id/snippets" \ { "title" : "Example Snippet Title", "description" : "More verbose snippet description", - "file_name" : "example.txt", - "content" : "source code \n with multiple lines\n", - "visibility" : "private" + "visibility" : "private", + "files": [ + { + "file_path": "example.txt", + "content" : "source code \n with multiple lines\n", + } + ] } ``` @@ -121,13 +130,22 @@ PUT /projects/:id/snippets/:snippet_id Parameters: -- `id` (required) - The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user -- `snippet_id` (required) - The ID of a project's snippet -- `title` (optional) - The title of a snippet -- `file_name` (optional) - The name of a snippet file -- `description` (optional) - The description of a snippet -- `content` (optional) - The content of a snippet -- `visibility` (optional) - The snippet's visibility +| Attribute | Type | Required | Description | +|:----------------------|:----------------|:---------|:----------------------------------------------------------------------------------------------------------------| +| `id` | integer | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | +| `snippet_id` | integer | yes | The ID of a project's snippet | +| `title` | string | no | Title of a snippet | +| `file_name` | string | no | Deprecated: Use `files` instead. Name of a snippet file | +| `content` | string | no | Deprecated: Use `files` instead. Content of a snippet | +| `description` | string | no | Description of a snippet | +| `visibility` | string | no | Snippet's [visibility](#snippet-visibility-level) | +| `files` | array of hashes | no | An array of snippet files | +| `files:action` | string | yes | Type of action to perform on the file, one of: 'create', 'update', 'delete', 'move' | +| `files:file_path` | string | no | File path of the snippet file | +| `files:previous_path` | string | no | Previous path of the snippet file | +| `files:content` | string | no | Content of the snippet file | + +Updates to snippets with multiple files *must* use the `files` attribute. Example request: @@ -144,9 +162,14 @@ curl --request PUT "https://gitlab.com/api/v4/projects/:id/snippets/:snippet_id" { "title" : "Updated Snippet Title", "description" : "More verbose snippet description", - "file_name" : "new_filename.txt", - "content" : "updated source code \n with multiple lines\n", - "visibility" : "private" + "visibility" : "private", + "files": [ + { + "action": "update", + "file_path": "example.txt", + "content" : "updated source code \n with multiple lines\n" + } + ] } ``` diff --git a/doc/api/projects.md b/doc/api/projects.md index ad26457ad99..f6ed905cda1 100644 --- a/doc/api/projects.md +++ b/doc/api/projects.md @@ -156,6 +156,7 @@ When the user is authenticated and `simple` is not set this returns something li "star_count": 0, "runners_token": "b8547b1dc37721d05889db52fa2f02", "ci_default_git_depth": 50, + "ci_forward_deployment_enabled": true, "public_jobs": true, "shared_with_groups": [], "only_allow_merge_if_pipeline_succeeds": false, @@ -248,6 +249,7 @@ When the user is authenticated and `simple` is not set this returns something li "star_count": 0, "runners_token": "b8547b1dc37721d05889db52fa2f02", "ci_default_git_depth": 0, + "ci_forward_deployment_enabled": true, "public_jobs": true, "shared_with_groups": [], "only_allow_merge_if_pipeline_succeeds": false, @@ -410,6 +412,7 @@ This endpoint supports [keyset pagination](README.md#keyset-based-pagination) fo "star_count": 0, "runners_token": "b8547b1dc37721d05889db52fa2f02", "ci_default_git_depth": 50, + "ci_forward_deployment_enabled": true, "public_jobs": true, "shared_with_groups": [], "only_allow_merge_if_pipeline_succeeds": false, @@ -502,6 +505,7 @@ This endpoint supports [keyset pagination](README.md#keyset-based-pagination) fo "star_count": 0, "runners_token": "b8547b1dc37721d05889db52fa2f02", "ci_default_git_depth": 0, + "ci_forward_deployment_enabled": true, "public_jobs": true, "shared_with_groups": [], "only_allow_merge_if_pipeline_succeeds": false, @@ -856,6 +860,7 @@ GET /projects/:id "star_count": 0, "runners_token": "b8bc4a7a29eb76ea83cf79e4908c2b", "ci_default_git_depth": 50, + "ci_forward_deployment_enabled": true, "public_jobs": true, "shared_with_groups": [ { @@ -1218,6 +1223,7 @@ PUT /projects/:id | `build_coverage_regex` | string | no | Test coverage parsing | | `ci_config_path` | string | no | The path to CI configuration file | | `ci_default_git_depth` | integer | no | Default number of revisions for [shallow cloning](../ci/pipelines/settings.md#git-shallow-clone) | +| `ci_forward_deployment_enabled` | boolean | no | When a new deployment job starts, [skip older deployment jobs](../ci/pipelines/settings.md#skip-outdated-deployment-jobs) that are still pending | | `auto_devops_enabled` | boolean | no | Enable Auto DevOps for this project | | `auto_devops_deploy_strategy` | string | no | Auto Deploy strategy (`continuous`, `manual` or `timed_incremental`) | | `repository_storage` | string | no | Which storage shard the repository is on. Available only to admins | @@ -1701,6 +1707,7 @@ Example response: "star_count": 0, "runners_token": "b8bc4a7a29eb76ea83cf79e4908c2b", "ci_default_git_depth": 50, + "ci_forward_deployment_enabled": true, "public_jobs": true, "shared_with_groups": [], "only_allow_merge_if_pipeline_succeeds": false, @@ -1811,6 +1818,7 @@ Example response: "star_count": 0, "runners_token": "b8bc4a7a29eb76ea83cf79e4908c2b", "ci_default_git_depth": 50, + "ci_forward_deployment_enabled": true, "public_jobs": true, "shared_with_groups": [], "only_allow_merge_if_pipeline_succeeds": false, @@ -2241,6 +2249,113 @@ PUT /projects/:id/transfer | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) | | `namespace` | integer/string | yes | The ID or path of the namespace to transfer to project to | +Example request: + +```shell +curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/transfer?namespace=14" +``` + +Example response: + +```json + { + "id": 7, + "description": "", + "name": "hello-world", + "name_with_namespace": "cute-cats / hello-world", + "path": "hello-world", + "path_with_namespace": "cute-cats/hello-world", + "created_at": "2020-10-15T16:25:22.415Z", + "default_branch": "master", + "tag_list": [], + "ssh_url_to_repo": "git@gitlab.example.com:cute-cats/hello-world.git", + "http_url_to_repo": "https://gitlab.example.com/cute-cats/hello-world.git", + "web_url": "https://gitlab.example.com/cute-cats/hello-world", + "readme_url": "https://gitlab.example.com/cute-cats/hello-world/-/blob/master/README.md", + "avatar_url": null, + "forks_count": 0, + "star_count": 0, + "last_activity_at": "2020-10-15T16:25:22.415Z", + "namespace": { + "id": 18, + "name": "cute-cats", + "path": "cute-cats", + "kind": "group", + "full_path": "cute-cats", + "parent_id": null, + "avatar_url": null, + "web_url": "https://gitlab.example.com/groups/cute-cats" + }, + "_links": { + "self": "https://gitlab.example.com/api/v4/projects/7", + "issues": "https://gitlab.example.com/api/v4/projects/7/issues", + "merge_requests": "https://gitlab.example.com/api/v4/projects/7/merge_requests", + "repo_branches": "https://gitlab.example.com/api/v4/projects/7/repository/branches", + "labels": "https://gitlab.example.com/api/v4/projects/7/labels", + "events": "https://gitlab.example.com/api/v4/projects/7/events", + "members": "https://gitlab.example.com/api/v4/projects/7/members" + }, + "packages_enabled": true, + "empty_repo": false, + "archived": false, + "visibility": "private", + "resolve_outdated_diff_discussions": false, + "container_registry_enabled": true, + "container_expiration_policy": { + "cadence": "7d", + "enabled": false, + "keep_n": null, + "older_than": null, + "name_regex": null, + "name_regex_keep": null, + "next_run_at": "2020-10-22T16:25:22.746Z" + }, + "issues_enabled": true, + "merge_requests_enabled": true, + "wiki_enabled": true, + "jobs_enabled": true, + "snippets_enabled": true, + "service_desk_enabled": false, + "service_desk_address": null, + "can_create_merge_request_in": true, + "issues_access_level": "enabled", + "repository_access_level": "enabled", + "merge_requests_access_level": "enabled", + "forking_access_level": "enabled", + "wiki_access_level": "enabled", + "builds_access_level": "enabled", + "snippets_access_level": "enabled", + "pages_access_level": "enabled", + "emails_disabled": null, + "shared_runners_enabled": true, + "lfs_enabled": true, + "creator_id": 2, + "import_status": "none", + "open_issues_count": 0, + "ci_default_git_depth": 50, + "public_jobs": true, + "build_timeout": 3600, + "auto_cancel_pending_pipelines": "enabled", + "build_coverage_regex": null, + "ci_config_path": null, + "shared_with_groups": [], + "only_allow_merge_if_pipeline_succeeds": false, + "allow_merge_on_skipped_pipeline": null, + "request_access_enabled": true, + "only_allow_merge_if_all_discussions_are_resolved": false, + "remove_source_branch_after_merge": true, + "printing_merge_request_link_enabled": true, + "merge_method": "merge", + "suggestion_commit_message": null, + "auto_devops_enabled": true, + "auto_devops_deploy_strategy": "continuous", + "autoclose_referenced_issues": true, + "approvals_before_merge": 0, + "mirror": false, + "compliance_frameworks": [] +} +``` + ## Branches Read more in the [Branches](branches.md) documentation. diff --git a/doc/api/protected_branches.md b/doc/api/protected_branches.md index 05d586738d0..79b848bdf15 100644 --- a/doc/api/protected_branches.md +++ b/doc/api/protected_branches.md @@ -295,6 +295,67 @@ Example response: } ``` +### Example with allow to push and allow to merge access **(STARTER)** + +Example request: + +```shell +curl --request POST \ + --header "PRIVATE-TOKEN: <your_access_token>" \ + --header "Content-Type: application/json" \ + --data '{ + "id": 5, + "name": "master", + "allowed_to_push": [{"access_level": 30}], + "allowed_to_merge": [{ + "access_level": 30 + },{ + "access_level": 40 + } + ]}' + "https://gitlab.example.com/api/v4/projects/5/protected_branches" +``` + +Example response: + +```json +{ + "id": 5, + "name": "master", + "push_access_levels": [ + { + "access_level": 30, + "access_level_description": "Developers + Maintainers", + "user_id": null, + "group_id": null + } + ], + "merge_access_levels": [ + { + "access_level": 30, + "access_level_description": "Developers + Maintainers", + "user_id": null, + "group_id": null + }, + { + "access_level": 40, + "access_level_description": "Maintainers", + "user_id": null, + "group_id": null + } + ], + "unprotect_access_levels": [ + { + "access_level": 40, + "access_level_description": "Maintainers", + "user_id": null, + "group_id": null + } + ], + "code_owner_approval_required": false +} +``` + ## Unprotect repository branches Unprotects the given protected branch or wildcard protected branch. diff --git a/doc/api/releases/index.md b/doc/api/releases/index.md index 357f7e7a125..4dac9f61469 100644 --- a/doc/api/releases/index.md +++ b/doc/api/releases/index.md @@ -22,6 +22,8 @@ GET /projects/:id/releases | Attribute | Type | Required | Description | | ------------- | -------------- | -------- | ----------------------------------------------------------------------------------- | | `id` | integer/string | yes | The ID or [URL-encoded path of the project](../README.md#namespaced-path-encoding). | +| `order_by` | string | no | The field to use as order. Either `released_at` (default) or `created_at`. | +| `sort` | string | no | The direction of the order. Either `desc` (default) for descending order or `asc` for ascending order. | Example request: @@ -361,11 +363,11 @@ POST /projects/:id/releases | `tag_name` | string | yes | The tag where the release will be created from. | | `description` | string | no | The description of the release. You can use [Markdown](../../user/markdown.md). | | `ref` | string | yes, if `tag_name` doesn't exist | If a tag specified in `tag_name` doesn't exist, the release will be created from `ref` and tagged with `tag_name`. It can be a commit SHA, another tag name, or a branch name. | -| `milestones` | array of string | no | The title of each milestone the release is associated with. | +| `milestones` | array of string | no | The title of each milestone the release is associated with. [GitLab Premium](https://about.gitlab.com/pricing/) customers can specify group milestones. | | `assets:links` | array of hash | no | An array of assets links. | | `assets:links:name`| string | required by: `assets:links` | The name of the link. | | `assets:links:url` | string | required by: `assets:links` | The URL of the link. | -| `assets:links:filepath` | string | no | Optional path for a [Direct Asset link](../../user/project/releases.md). +| `assets:links:filepath` | string | no | Optional path for a [Direct Asset link](../../user/project/releases/index.md). | `assets:links:link_type` | string | no | The type of the link: `other`, `runbook`, `image`, `package`. Defaults to `other`. | `released_at` | datetime | no | The date when the release will be/was ready. Defaults to the current time. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). | @@ -484,6 +486,15 @@ Example response: } ``` +### Group milestones **(PREMIUM ONLY)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/235391) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.5. + +Group milestones associated with the project may be specified in the `milestones` +array for [Create a release](#create-a-release) and [Update a release](#update-a-release) +API calls. Only milestones associated with the project's group may be specified, and +adding milestones for ancestor groups will raise an error. + ## Collect release evidence **(PREMIUM ONLY)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/199065) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.10. @@ -525,7 +536,7 @@ PUT /projects/:id/releases/:tag_name | `tag_name` | string | yes | The tag where the release will be created from. | | `name` | string | no | The release name. | | `description` | string | no | The description of the release. You can use [Markdown](../../user/markdown.md). | -| `milestones` | array of string | no | The title of each milestone to associate with the release (`[]` to remove all milestones from the release). | +| `milestones` | array of string | no | The title of each milestone to associate with the release. [GitLab Premium](https://about.gitlab.com/pricing/) customers can specify group milestones. To remove all milestones from the release, specify `[]`. | | `released_at` | datetime | no | The date when the release will be/was ready. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). | Example request: diff --git a/doc/api/repositories.md b/doc/api/repositories.md index 305216f853a..7e94ff7b7f2 100644 --- a/doc/api/repositories.md +++ b/doc/api/repositories.md @@ -24,7 +24,8 @@ Parameters: - `path` (optional) - The path inside repository. Used to get content of subdirectories - `ref` (optional) - The name of a repository branch or tag or if not given the default branch - `recursive` (optional) - Boolean value used to get a recursive tree (false by default) -- `per_page` (optional) - Number of results to show per page. If not specified, defaults to `20` +- `per_page` (optional) - Number of results to show per page. If not specified, defaults to `20`. + Read more on [pagination](README.md#pagination). ```json [ diff --git a/doc/api/resource_iteration_events.md b/doc/api/resource_iteration_events.md index f774cdfe9c7..47f4d70fdf1 100644 --- a/doc/api/resource_iteration_events.md +++ b/doc/api/resource_iteration_events.md @@ -6,14 +6,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Resource iteration events API **(STARTER)** -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/40850) in [GitLab Starter](https://about.gitlab.com/pricing/) 13.4 -> - It's [deployed behind a feature flag](../user/feature_flags.md), enabled by default. -> - It's enabled on GitLab.com. -> - It's recommended for production use. -> - For GitLab self-managed instances, GitLab administrators can opt to [disable it](#enable-or-disable-iterations-events-tracking). - -NOTE: **Note:** -This feature might not be available to you. Check the **version history** note above for details. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/229463) in [GitLab Starter](https://about.gitlab.com/pricing/) 13.4. +> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/229463) in [GitLab Starter](https://about.gitlab.com/pricing/) 13.5. Resource iteration events keep track of what happens to GitLab [issues](../user/project/issues/). @@ -154,22 +148,3 @@ Example response: "action": "remove" } ``` - -### Enable or disable iterations events tracking **(STARTER)** - -Iterations events tracking is under development but ready for production use. -It is deployed behind a feature flag that is **enabled by default**. -[GitLab administrators with access to the GitLab Rails console](../administration/feature_flags.md) -can opt to disable it. - -To enable it: - -```ruby -Feature.enable(:track_iteration_change_events) -``` - -To disable it: - -```ruby -Feature.disable(:track_iteration_change_events) -``` diff --git a/doc/api/runners.md b/doc/api/runners.md index 436abe0a706..16ecdebcd4f 100644 --- a/doc/api/runners.md +++ b/doc/api/runners.md @@ -271,20 +271,20 @@ Example response: } ``` -## Remove a runner +### Pause a runner -Remove a runner. +Pause a specific runner. ```plaintext -DELETE /runners/:id +PUT --form "active=false" /runners/:runner_id ``` -| Attribute | Type | Required | Description | -|-----------|---------|----------|---------------------| -| `id` | integer | yes | The ID of a runner | +| Attribute | Type | Required | Description | +|-------------|---------|----------|---------------------| +| `runner_id` | integer | yes | The ID of a runner | ```shell -curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/runners/6" +curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" --form "active=false" "https://gitlab.example.com/api/v4/runners/6" ``` ## List runner's jobs @@ -466,7 +466,7 @@ Example response: Disable a specific runner from the project. It works only if the project isn't the only project associated with the specified runner. If so, an error is -returned. Use the [Remove a runner](#remove-a-runner) call instead. +returned. Use the call to [delete a runner](#delete-a-runner) instead. ```plaintext DELETE /projects/:id/runners/:runner_id @@ -553,7 +553,7 @@ POST /runners |--------------|---------|----------|---------------------| | `token` | string | yes | [Registration token](#registration-and-authentication-tokens). | | `description`| string | no | Runner's description| -| `info` | hash | no | Runner's metadata | +| `info` | hash | no | Runner's metadata. You can include `name`, `version`, `revision`, `platform`, and `architecture`, but only `version` is displayed in the Admin area of the UI. | | `active` | boolean | no | Whether the runner is active | | `locked` | boolean | no | Whether the runner should be locked for current project | | `run_untagged` | boolean | no | Whether the runner should handle untagged jobs | @@ -580,9 +580,32 @@ Example response: } ``` -## Delete a registered runner +## Delete a runner + +There are two ways to delete a runner: + +- By specifying the runner ID. +- By specifying the runner's authentication token. -Deletes a registered runner. +### Delete a runner by ID + +To delete the runner by ID, use your access token with the runner's ID: + +```plaintext +DELETE /runners/:id +``` + +| Attribute | Type | Required | Description | +|-------------|---------|----------|---------------------| +| `id` | integer | yes | The ID of a runner. The ID is visible in the UI under **Settings > CI/CD**. Expand **Runners**, and below the **Remove Runner** button is an ID preceded by the pound sign, for example, `#6`. | + +```shell +curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/runners/6" +``` + +### Delete a runner by authentication token + +To delete the runner by using its authentication token: ```plaintext DELETE /runners @@ -590,7 +613,7 @@ DELETE /runners | Attribute | Type | Required | Description | |-------------|---------|----------|---------------------| -| `token` | string | yes | Runner's [authentication token](#registration-and-authentication-tokens). | +| `token` | string | yes | The runner's [authentication token](#registration-and-authentication-tokens). | ```shell curl --request DELETE "https://gitlab.example.com/api/v4/runners" --form "token=<authentication_token>" diff --git a/doc/api/search.md b/doc/api/search.md index cb90b9a064c..bdf5bdd4924 100644 --- a/doc/api/search.md +++ b/doc/api/search.md @@ -24,10 +24,11 @@ GET /search | `scope` | string | yes | The scope to search in | | `search` | string | yes | The search query | | `state` | string | no | Filter by state. Issues and merge requests are supported; it is ignored for other scopes. | +| `confidential` | boolean | no | Filter by confidentiality. Issues scope is supported; it is ignored for other scopes. This parameter is behind a [feature flag (`search_filter_by_confidential`)](../administration/feature_flags.md). | Search the expression within the specified scope. Currently these scopes are supported: projects, issues, merge_requests, milestones, snippet_titles, users. -If Elasticsearch is enabled additional scopes available are blobs, wiki_blobs and commits. Find more about [the feature](../integration/elasticsearch.md). **(STARTER)** +If Elasticsearch is enabled additional scopes available are blobs, wiki_blobs, notes, and commits. Find more about [the feature](../integration/elasticsearch.md). **(STARTER)** The response depends on the requested scope. @@ -362,6 +363,40 @@ Example response: NOTE: **Note:** `filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` will be only the file name and not the full path. For details, see [issue 34521](https://gitlab.com/gitlab-org/gitlab/-/issues/34521). +### Scope: notes **(STARTER)** + +This scope is available only if [Elasticsearch](../integration/elasticsearch.md) is enabled. + +```shell +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/6/search?scope=notes&search=maxime" +``` + +Example response: + +```json +[ + { + "id": 191, + "body": "Harum maxime consequuntur et et deleniti assumenda facilis.", + "attachment": null, + "author": { + "id": 23, + "name": "User 1", + "username": "user1", + "state": "active", + "avatar_url": "https://www.gravatar.com/avatar/111d68d06e2d317b5a59c2c6c5bad808?s=80&d=identicon", + "web_url": "http://localhost:3000/user1" + }, + "created_at": "2017-09-05T08:01:32.068Z", + "updated_at": "2017-09-05T08:01:32.068Z", + "system": false, + "noteable_id": 22, + "noteable_type": "Issue", + "noteable_iid": 2 + } +] +``` + ### Scope: users ```shell @@ -399,10 +434,11 @@ GET /groups/:id/search | `scope` | string | yes | The scope to search in | | `search` | string | yes | The search query | | `state` | string | no | Filter by state. Issues and merge requests are supported; it is ignored for other scopes. | +| `confidential` | boolean | no | Filter by confidentiality. Issues scope is supported; it is ignored for other scopes. This parameter is behind a [feature flag (`search_filter_by_confidential`)](../administration/feature_flags.md). | Search the expression within the specified scope. Currently these scopes are supported: projects, issues, merge_requests, milestones, users. -If Elasticsearch is enabled additional scopes available are blobs, wiki_blobs and commits. Find more about [the feature](../integration/elasticsearch.md). **(STARTER)** +If Elasticsearch is enabled additional scopes available are blobs, wiki_blobs, notes, and commits. Find more about [the feature](../integration/elasticsearch.md). **(STARTER)** The response depends on the requested scope. @@ -706,6 +742,40 @@ Example response: NOTE **Note:** `filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` will be only the file name and not the full path. For details, see [issue 34521](https://gitlab.com/gitlab-org/gitlab/-/issues/34521). +### Scope: notes **(STARTER)** + +This scope is available only if [Elasticsearch](../integration/elasticsearch.md) is enabled. + +```shell +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/6/search?scope=notes&search=maxime" +``` + +Example response: + +```json +[ + { + "id": 191, + "body": "Harum maxime consequuntur et et deleniti assumenda facilis.", + "attachment": null, + "author": { + "id": 23, + "name": "User 1", + "username": "user1", + "state": "active", + "avatar_url": "https://www.gravatar.com/avatar/111d68d06e2d317b5a59c2c6c5bad808?s=80&d=identicon", + "web_url": "http://localhost:3000/user1" + }, + "created_at": "2017-09-05T08:01:32.068Z", + "updated_at": "2017-09-05T08:01:32.068Z", + "system": false, + "noteable_id": 22, + "noteable_type": "Issue", + "noteable_iid": 2 + } +] +``` + ### Scope: users ```shell @@ -744,6 +814,7 @@ GET /projects/:id/search | `search` | string | yes | The search query | | `ref` | string | no | The name of a repository branch or tag to search on. The project's default branch is used by default. This is only applicable for scopes: commits, blobs, and wiki_blobs. | | `state` | string | no | Filter by state. Issues and merge requests are supported; it is ignored for other scopes. | +| `confidential` | boolean | no | Filter by confidentiality. Issues scope is supported; it is ignored for other scopes. This parameter is behind a [feature flag (`search_filter_by_confidential`)](../administration/feature_flags.md). | Search the expression within the specified scope. Currently these scopes are supported: issues, merge_requests, milestones, notes, wiki_blobs, commits, blobs, users. diff --git a/doc/api/services.md b/doc/api/services.md index 405047a433d..7c01e43a4d8 100644 --- a/doc/api/services.md +++ b/doc/api/services.md @@ -659,7 +659,7 @@ Parameters: | `webhook` | string | true | The Hangouts Chat webhook. For example, `https://chat.googleapis.com/v1/spaces...`. | | `notify_only_broken_pipelines` | boolean | false | Send notifications for broken pipelines | | `notify_only_default_branch` | boolean | false | DEPRECATED: This parameter has been replaced with `branches_to_be_notified` | -| `branches_to_be_notified` | string | all | Branches to send notifications for. Valid options are "all", "default", "protected", and "default_and_protected" | +| `branches_to_be_notified` | string | false | Branches to send notifications for. Valid options are "all", "default", "protected", and "default_and_protected" | | `push_events` | boolean | false | Enable notifications for push events | | `issues_events` | boolean | false | Enable notifications for issue events | | `confidential_issues_events` | boolean | false | Enable notifications for confidential issue events | @@ -714,7 +714,7 @@ Parameters: | `merge_requests_events` | boolean | false | Enable notifications for merge request events | | `tag_push_events` | boolean | false | Enable notifications for tag push events | | `note_events` | boolean | false | Enable notifications for note events | -| `confidental_note_events` | boolean | false | Enable notifications for confidential note events | +| `confidential_note_events` | boolean | false | Enable notifications for confidential note events | | `pipeline_events` | boolean | false | Enable notifications for pipeline events | ### Delete HipChat service diff --git a/doc/api/settings.md b/doc/api/settings.md index c8a466d1fcd..236cd10a30e 100644 --- a/doc/api/settings.md +++ b/doc/api/settings.md @@ -2,8 +2,8 @@ These API calls allow you to read and modify GitLab instance [application settings](#list-of-settings-that-can-be-accessed-via-api-calls) -as appear in `/admin/application_settings/general`. You have to be an -administrator in order to perform this action. +as they appear in `/admin/application_settings/general`. You must be an +administrator to perform this action. ## Get current application settings @@ -185,13 +185,14 @@ Example responses: **(PREMIUM ONLY)** ## List of settings that can be accessed via API calls -In general, all settings are optional. Certain settings though, if enabled, will -require other settings to be set in order to function properly. These requirements -are listed in the descriptions of the relevant settings. +In general, all settings are optional. Certain settings though, if enabled, +require other settings to be set to function properly. These requirements are +listed in the descriptions of the relevant settings. -| Attribute | Type | Required | Description | -| --------- | ---- | :------: | ----------- | -| `admin_notification_email` | string | no | Abuse reports will be sent to this address if it is set. Abuse reports are always available in the Admin Area. | +| Attribute | Type | Required | Description | +|------------------------------------------|------------------|:------------------------------------:|-------------| +| `admin_notification_email` | string | no | Deprecated: Use `abuse_notification_email` instead. If set, [abuse reports](../user/admin_area/abuse_reports.md) are sent to this address. Abuse reports are always available in the Admin Area. | +| `abuse_notification_email` | string | no | If set, [abuse reports](../user/admin_area/abuse_reports.md) are sent to this address. Abuse reports are always available in the Admin Area. | | `after_sign_out_path` | string | no | Where to redirect users after logout. | | `after_sign_up_text` | string | no | Text shown to the user after signing up | | `akismet_api_key` | string | required by: `akismet_enabled` | API key for Akismet spam protection. | @@ -208,6 +209,7 @@ are listed in the descriptions of the relevant settings. | `authorized_keys_enabled` | boolean | no | By default, we write to the `authorized_keys` file to support Git over SSH without additional configuration. GitLab can be optimized to authenticate SSH keys via the database file. Only disable this if you have configured your OpenSSH server to use the AuthorizedKeysCommand. | | `auto_devops_domain` | string | no | Specify a domain to use by default for every project's Auto Review Apps and Auto Deploy stages. | | `auto_devops_enabled` | boolean | no | Enable Auto DevOps for projects by default. It will automatically build, test, and deploy applications based on a predefined CI/CD configuration. | +| `automatic_purchased_storage_allocation` | boolean | no | Enabling this permits automatic allocation of purchased storage within a namespace. | | `check_namespace_plan` | boolean | no | **(PREMIUM)** Enabling this will make only licensed EE features available to projects if the project namespace's plan includes the feature or if the project is public. | | `commit_email_hostname` | string | no | Custom hostname (for private commit emails). | | `container_registry_token_expire_delay` | integer | no | Container Registry token duration in minutes. | @@ -255,10 +257,10 @@ are listed in the descriptions of the relevant settings. | `external_auth_client_cert` | string | no | (**If enabled, requires:** `external_auth_client_key`) The certificate to use to authenticate with the external authorization service | | `external_auth_client_key_pass` | string | no | Passphrase to use for the private key when authenticating with the external service this is encrypted when stored | | `external_auth_client_key` | string | required by: `external_auth_client_cert` | Private key for the certificate when authentication is required for the external authorization service, this is encrypted when stored | -| `external_authorization_service_default_label` | string | required by: `external_authorization_service_enabled` | The default classification label to use when requesting authorization and no classification label has been specified on the project | +| `external_authorization_service_default_label` | string | required by:<br>`external_authorization_service_enabled` | The default classification label to use when requesting authorization and no classification label has been specified on the project. | | `external_authorization_service_enabled` | boolean | no | (**If enabled, requires:** `external_authorization_service_default_label`, `external_authorization_service_timeout` and `external_authorization_service_url`) Enable using an external authorization service for accessing projects | -| `external_authorization_service_timeout` | float | required by: `external_authorization_service_enabled` | The timeout after which an authorization request is aborted, in seconds. When a request times out, access is denied to the user. (min: 0.001, max: 10, step: 0.001) | -| `external_authorization_service_url` | string | required by: `external_authorization_service_enabled` | URL to which authorization requests will be directed | +| `external_authorization_service_timeout` | float | required by:<br>`external_authorization_service_enabled` | The timeout after which an authorization request is aborted, in seconds. When a request times out, access is denied to the user. (min: 0.001, max: 10, step: 0.001). | +| `external_authorization_service_url` | string | required by:<br>`external_authorization_service_enabled` | URL to which authorization requests are directed. | | `file_template_project_id` | integer | no | **(PREMIUM)** The ID of a project to load custom file templates from | | `first_day_of_week` | integer | no | Start day of the week for calendar views and date pickers. Valid values are `0` (default) for Sunday, `1` for Monday, and `6` for Saturday. | | `geo_node_allowed_ips` | string | yes | **(PREMIUM)** Comma-separated list of IPs and CIDRs of allowed secondary nodes. For example, `1.1.1.1, 2.2.2.0/24`. | @@ -298,7 +300,7 @@ are listed in the descriptions of the relevant settings. | `mirror_max_capacity` | integer | no | **(PREMIUM)** Maximum number of mirrors that can be synchronizing at the same time. | | `mirror_max_delay` | integer | no | **(PREMIUM)** Maximum time (in minutes) between updates that a mirror can have when scheduled to synchronize. | | `npm_package_requests_forwarding` | boolean | no | **(PREMIUM)** Use npmjs.org as a default remote repository when the package is not found in the GitLab NPM Registry | -| `outbound_local_requests_whitelist` | array of strings | no | Define a list of trusted domains or ip addresses to which local requests are allowed when local requests for hooks and services are disabled. +| `outbound_local_requests_whitelist` | array of strings | no | Define a list of trusted domains or IP addresses to which local requests are allowed when local requests for hooks and services are disabled. | `pages_domain_verification_enabled` | boolean | no | Require users to prove ownership of custom domains. Domain verification is an essential security measure for public GitLab sites. Users are required to demonstrate they control a domain before it is enabled. | | `password_authentication_enabled_for_git` | boolean | no | Enable authentication for Git over HTTP(S) via a GitLab account password. Default is `true`. | | `password_authentication_enabled_for_web` | boolean | no | Enable authentication for the web interface via a GitLab account password. Default is `true`. | @@ -321,7 +323,7 @@ are listed in the descriptions of the relevant settings. | `receive_max_input_size` | integer | no | Maximum push size (MB). | | `repository_checks_enabled` | boolean | no | GitLab will periodically run `git fsck` in all project and wiki repositories to look for silent disk corruption issues. | | `repository_size_limit` | integer | no | **(PREMIUM)** Size limit per repository (MB) | -| `repository_storages_weighted` | hash of strings to integers | no | (GitLab 13.1 and later) Hash of names of taken from `gitlab.yml` to weights. New projects are created in one of these stores, chosen by a weighted random selection. | +| `repository_storages_weighted` | hash of strings to integers | no | (GitLab 13.1 and later) Hash of names of taken from `gitlab.yml` to [weights](../administration/repository_storage_paths.md#choose-where-new-repositories-will-be-stored). New projects are created in one of these stores, chosen by a weighted random selection. | | `repository_storages` | array of strings | no | (GitLab 13.0 and earlier) List of names of enabled storage paths, taken from `gitlab.yml`. New projects are created in one of these stores, chosen at random. | | `require_two_factor_authentication` | boolean | no | (**If enabled, requires:** `two_factor_grace_period`) Require all users to set up Two-factor authentication. | | `restricted_visibility_levels` | array of strings | no | Selected levels cannot be used by non-admin users for groups, projects or snippets. Can take `private`, `internal` and `public` as a parameter. Default is `null` which means there is no restriction. | @@ -351,17 +353,17 @@ are listed in the descriptions of the relevant settings. | `terminal_max_session_time` | integer | no | Maximum time for web terminal websocket connection (in seconds). Set to `0` for unlimited time. | | `terms` | text | required by: `enforce_terms` | (**Required by:** `enforce_terms`) Markdown content for the ToS. | | `throttle_authenticated_api_enabled` | boolean | no | (**If enabled, requires:** `throttle_authenticated_api_period_in_seconds` and `throttle_authenticated_api_requests_per_period`) Enable authenticated API request rate limit. Helps reduce request volume (for example, from crawlers or abusive bots). | -| `throttle_authenticated_api_period_in_seconds` | integer | required by: `throttle_authenticated_api_enabled` | Rate limit period in seconds. | -| `throttle_authenticated_api_requests_per_period` | integer | required by: `throttle_authenticated_api_enabled` | Max requests per period per user. | +| `throttle_authenticated_api_period_in_seconds` | integer | required by:<br>`throttle_authenticated_api_enabled` | Rate limit period in seconds. | +| `throttle_authenticated_api_requests_per_period` | integer | required by:<br>`throttle_authenticated_api_enabled` | Max requests per period per user. | | `throttle_authenticated_web_enabled` | boolean | no | (**If enabled, requires:** `throttle_authenticated_web_period_in_seconds` and `throttle_authenticated_web_requests_per_period`) Enable authenticated web request rate limit. Helps reduce request volume (for example, from crawlers or abusive bots). | -| `throttle_authenticated_web_period_in_seconds` | integer | required by: `throttle_authenticated_web_enabled` | Rate limit period in seconds. | -| `throttle_authenticated_web_requests_per_period` | integer | required by: `throttle_authenticated_web_enabled` | Max requests per period per user. | +| `throttle_authenticated_web_period_in_seconds` | integer | required by:<br>`throttle_authenticated_web_enabled` | Rate limit period in seconds. | +| `throttle_authenticated_web_requests_per_period` | integer | required by:<br>`throttle_authenticated_web_enabled` | Max requests per period per user. | | `throttle_unauthenticated_enabled` | boolean | no | (**If enabled, requires:** `throttle_unauthenticated_period_in_seconds` and `throttle_unauthenticated_requests_per_period`) Enable unauthenticated request rate limit. Helps reduce request volume (for example, from crawlers or abusive bots). | -| `throttle_unauthenticated_period_in_seconds` | integer | required by: `throttle_unauthenticated_enabled` | Rate limit period in seconds. | -| `throttle_unauthenticated_requests_per_period` | integer | required by: `throttle_unauthenticated_enabled` | Max requests per period per IP. | +| `throttle_unauthenticated_period_in_seconds` | integer | required by:<br>`throttle_unauthenticated_enabled` | Rate limit period in seconds. | +| `throttle_unauthenticated_requests_per_period` | integer | required by:<br>`throttle_unauthenticated_enabled` | Max requests per period per IP. | | `time_tracking_limit_to_hours` | boolean | no | Limit display of time tracking units to hours. Default is `false`. | | `two_factor_grace_period` | integer | required by: `require_two_factor_authentication` | Amount of time (in hours) that users are allowed to skip forced configuration of two-factor authentication. | -| `unique_ips_limit_enabled` | boolean | no | (**If enabled, requires:** `unique_ips_limit_per_user` and `unique_ips_limit_time_window`) Limit sign in from multiple ips. | +| `unique_ips_limit_enabled` | boolean | no | (**If enabled, requires:** `unique_ips_limit_per_user` and `unique_ips_limit_time_window`) Limit sign in from multiple IPs. | | `unique_ips_limit_per_user` | integer | required by: `unique_ips_limit_enabled` | Maximum number of IPs per user. | | `unique_ips_limit_time_window` | integer | required by: `unique_ips_limit_enabled` | How many seconds an IP will be counted towards the limit. | | `usage_ping_enabled` | boolean | no | Every week GitLab will report license usage back to GitLab, Inc. | diff --git a/doc/api/snippets.md b/doc/api/snippets.md index 6863763ff24..431d745ac84 100644 --- a/doc/api/snippets.md +++ b/doc/api/snippets.md @@ -198,22 +198,40 @@ POST /snippets Parameters: -| Attribute | Type | Required | Description | -|:--------------|:-------|:---------|:---------------------------------------------------| -| `title` | string | yes | Title of a snippet. | -| `file_name` | string | yes | Name of a snippet file. | -| `content` | string | yes | Content of a snippet. | -| `description` | string | no | Description of a snippet. | -| `visibility` | string | no | Snippet's [visibility](#snippet-visibility-level). | +| Attribute | Type | Required | Description | +|:------------------|:----------------|:---------|:--------------------------------------------------------| +| `title` | string | yes | Title of a snippet | +| `file_name` | string | no | Deprecated: Use `files` instead. Name of a snippet file | +| `content` | string | no | Deprecated: Use `files` instead. Content of a snippet | +| `description` | string | no | Description of a snippet | +| `visibility` | string | no | Snippet's [visibility](#snippet-visibility-level) | +| `files` | array of hashes | no | An array of snippet files | +| `files:file_path` | string | yes | File path of the snippet file | +| `files:content` | string | yes | Content of the snippet file | Example request: ```shell -curl --request POST \ - --data '{"title": "This is a snippet", "content": "Hello world", "description": "Hello World snippet", "file_name": "test.txt", "visibility": "internal" }' \ +curl --request POST "https://gitlab.example.com/api/v4/snippets" \ --header 'Content-Type: application/json' \ --header "PRIVATE-TOKEN: <your_access_token>" \ - "https://gitlab.example.com/api/v4/snippets" + -d @snippet.json +``` + +`snippet.json` used in the above example request: + +```json +{ + "title": "This is a snippet", + "description": "Hello World snippet", + "visibility": "internal", + "files": [ + { + "content": "Hello world", + "file_path": "test.txt" + } + ] +} ``` Example response: @@ -222,7 +240,6 @@ Example response: { "id": 1, "title": "This is a snippet", - "file_name": "test.txt", "description": "Hello World snippet", "visibility": "internal", "author": { @@ -238,7 +255,16 @@ Example response: "created_at": "2012-06-28T10:52:04Z", "project_id": null, "web_url": "http://example.com/snippets/1", - "raw_url": "http://example.com/snippets/1/raw" + "raw_url": "http://example.com/snippets/1/raw", + "ssh_url_to_repo": "ssh://git@gitlab.example.com:snippets/1.git", + "http_url_to_repo": "https://gitlab.example.com/snippets/1.git", + "file_name": "test.txt", + "files": [ + { + "path": "text.txt", + "raw_url": "https://gitlab.example.com/-/snippets/1/raw/master/renamed.md" + } + ] } ``` @@ -255,23 +281,44 @@ PUT /snippets/:id Parameters: -| Attribute | Type | Required | Description | -|:--------------|:--------|:---------|:---------------------------------------------------| -| `id` | integer | yes | ID of snippet to update. | -| `title` | string | no | Title of a snippet. | -| `file_name` | string | no | Name of a snippet file. | -| `description` | string | no | Description of a snippet. | -| `content` | string | no | Content of a snippet. | -| `visibility` | string | no | Snippet's [visibility](#snippet-visibility-level). | +| Attribute | Type | Required | Description | +|:----------------------|:----------------|:---------|:------------------------------------------------------------------------------------| +| `id` | integer | yes | ID of snippet to update | +| `title` | string | no | Title of a snippet | +| `file_name` | string | no | Deprecated: Use `files` instead. Name of a snippet file | +| `content` | string | no | Deprecated: Use `files` instead. Content of a snippet | +| `description` | string | no | Description of a snippet | +| `visibility` | string | no | Snippet's [visibility](#snippet-visibility-level) | +| `files` | array of hashes | no | An array of snippet files | +| `files:action` | string | yes | Type of action to perform on the file, one of: 'create', 'update', 'delete', 'move' | +| `files:file_path` | string | no | File path of the snippet file | +| `files:previous_path` | string | no | Previous path of the snippet file | +| `files:content` | string | no | Content of the snippet file | + +Updates to snippets with multiple files *must* use the `files` attribute. Example request: ```shell -curl --request PUT \ - --data '{"title": "foo", "content": "bar"}' \ +curl --request PUT "https://gitlab.example.com/api/v4/snippets/1" \ --header 'Content-Type: application/json' \ --header "PRIVATE-TOKEN: <your_access_token>" \ - "https://gitlab.example.com/api/v4/snippets/1" + -d @snippet.json +``` + +`snippet.json` used in the above example request: + +```json +{ + "title": "foo", + "files": [ + { + "action": "move", + "previous_path": "test.txt", + "file_path": "renamed.md" + } + ] +} ``` Example response: @@ -280,7 +327,6 @@ Example response: { "id": 1, "title": "test", - "file_name": "add.rb", "description": "description of snippet", "visibility": "internal", "author": { @@ -296,7 +342,16 @@ Example response: "created_at": "2012-06-28T10:52:04Z", "project_id": null, "web_url": "http://example.com/snippets/1", - "raw_url": "http://example.com/snippets/1/raw" + "raw_url": "http://example.com/snippets/1/raw", + "ssh_url_to_repo": "ssh://git@gitlab.example.com:snippets/1.git", + "http_url_to_repo": "https://gitlab.example.com/snippets/1.git", + "file_name": "renamed.md", + "files": [ + { + "path": "renamed.md", + "raw_url": "https://gitlab.example.com/-/snippets/1/raw/master/renamed.md" + } + ] } ``` diff --git a/doc/api/templates/gitlab_ci_ymls.md b/doc/api/templates/gitlab_ci_ymls.md index dfe22fc453e..45bc0f55095 100644 --- a/doc/api/templates/gitlab_ci_ymls.md +++ b/doc/api/templates/gitlab_ci_ymls.md @@ -135,7 +135,7 @@ Example response: ```json { "name": "Ruby", - "content": "# This file is a template, and might need editing before it works on your project.\n# Official language image. Look for the different tagged releases at:\n# https://hub.docker.com/r/library/ruby/tags/\nimage: \"ruby:2.5\"\n\n# Pick zero or more services to be used on all builds.\n# Only needed when using a docker container to run your tests in.\n# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service\nservices:\n - mysql:latest\n - redis:latest\n - postgres:latest\n\nvariables:\n POSTGRES_DB: database_name\n\n# Cache gems in between builds\ncache:\n paths:\n - vendor/ruby\n\n# This is a basic example for a gem or script which doesn't use\n# services such as redis or postgres\nbefore_script:\n - ruby -v # Print out ruby version for debugging\n # Uncomment next line if your rails app needs a JS runtime:\n # - apt-get update -q && apt-get install nodejs -yqq\n - bundle install -j $(nproc) --path vendor # Install dependencies into ./vendor/ruby\n\n# Optional - Delete if not using `rubocop`\nrubocop:\n script:\n - rubocop\n\nrspec:\n script:\n - rspec spec\n\nrails:\n variables:\n DATABASE_URL: \"postgresql://postgres:postgres@postgres:5432/$POSTGRES_DB\"\n script:\n - rails db:migrate\n - rails db:seed\n - rails test\n\n# This deploy job uses a simple deploy flow to Heroku, other providers, e.g. AWS Elastic Beanstalk\n# are supported too: https://github.com/travis-ci/dpl\ndeploy:\n type: deploy\n environment: production\n script:\n - gem install dpl\n - dpl --provider=heroku --app=$HEROKU_APP_NAME --api-key=$HEROKU_PRODUCTION_KEY\n" + "content": "# This file is a template, and might need editing before it works on your project.\n# Official language image. Look for the different tagged releases at:\n# https://hub.docker.com/r/library/ruby/tags/\nimage: \"ruby:2.5\"\n\n# Pick zero or more services to be used on all builds.\n# Only needed when using a docker container to run your tests in.\n# Check out: http://docs.gitlab.com/ee/ci/docker/using_docker_images.html#what-is-a-service\nservices:\n - mysql:latest\n - redis:latest\n - postgres:latest\n\nvariables:\n POSTGRES_DB: database_name\n\n# Cache gems in between builds\ncache:\n paths:\n - vendor/ruby\n\n# This is a basic example for a gem or script which doesn't use\n# services such as redis or postgres\nbefore_script:\n - ruby -v # Print out ruby version for debugging\n # Uncomment next line if your rails app needs a JS runtime:\n # - apt-get update -q && apt-get install nodejs -yqq\n - bundle install -j $(nproc) --path vendor # Install dependencies into ./vendor/ruby\n\n# Optional - Delete if not using `rubocop`\nrubocop:\n script:\n - rubocop\n\nrspec:\n script:\n - rspec spec\n\nrails:\n variables:\n DATABASE_URL: \"postgresql://postgres:postgres@postgres:5432/$POSTGRES_DB\"\n script:\n - rails db:migrate\n - rails db:seed\n - rails test\n\n# This deploy job uses a simple deploy flow to Heroku, other providers, e.g. AWS Elastic Beanstalk\n# are supported too: https://github.com/travis-ci/dpl\ndeploy:\n type: deploy\n environment: production\n script:\n - gem install dpl\n - dpl --provider=heroku --app=$HEROKU_APP_NAME --api-key=$HEROKU_PRODUCTION_KEY\n" } ``` diff --git a/doc/api/todos.md b/doc/api/todos.md index ebe10ecbd49..ab36021d694 100644 --- a/doc/api/todos.md +++ b/doc/api/todos.md @@ -4,13 +4,13 @@ group: Project Management info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# To-dos API +# To dos API > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/3188) in GitLab 8.10. -## Get a list of to-dos +## Get a list of to dos -Returns a list of to-dos. When no filter is applied, it returns all pending to-dos +Returns a list of to dos. When no filter is applied, it returns all pending to dos for the current user. Different filters allow the user to precise the request. ```plaintext @@ -25,8 +25,8 @@ Parameters: | `author_id` | integer | no | The ID of an author | | `project_id` | integer | no | The ID of a project | | `group_id` | integer | no | The ID of a group | -| `state` | string | no | The state of the to-do. Can be either `pending` or `done` | -| `type` | string | no | The type of a to-do. Can be either `Issue`, `MergeRequest`, `DesignManagement::Design` or `AlertManagement::Alert` | +| `state` | string | no | The state of the to do. Can be either `pending` or `done` | +| `type` | string | no | The type of a to do. Can be either `Issue`, `MergeRequest`, `DesignManagement::Design` or `AlertManagement::Alert` | ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/todos" @@ -187,10 +187,10 @@ Example Response: ] ``` -## Mark a to-do as done +## Mark a to do as done -Marks a single pending to-do given by its ID for the current user as done. The -to-do marked as done is returned in the response. +Marks a single pending to do given by its ID for the current user as done. The +to do marked as done is returned in the response. ```plaintext POST /todos/:id/mark_as_done @@ -200,7 +200,7 @@ Parameters: | Attribute | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `id` | integer | yes | The ID of a to-do | +| `id` | integer | yes | The ID of a to do | ```shell curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/todos/130/mark_as_done" @@ -285,9 +285,9 @@ Example Response: } ``` -## Mark all to-dos as done +## Mark all to dos as done -Marks all pending to-dos for the current user as done. It returns the HTTP status code `204` with an empty response. +Marks all pending to dos for the current user as done. It returns the HTTP status code `204` with an empty response. ```plaintext POST /todos/mark_as_done diff --git a/doc/api/users.md b/doc/api/users.md index 634e0bd0842..beaea689fb7 100644 --- a/doc/api/users.md +++ b/doc/api/users.md @@ -950,7 +950,7 @@ Returns `204 No Content` on success, or `404 Not found` if the key cannot be fou ## List all GPG keys for given user -Get a list of a specified user's GPG keys. Available only for admins. +Get a list of a specified user's GPG keys. This endpoint can be accessed without authentication. ```plaintext GET /users/:id/gpg_keys @@ -980,7 +980,8 @@ Example response: ## Get a specific GPG key for a given user -Get a specific GPG key for a given user. Available only for admins. +Get a specific GPG key for a given user. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/43693) +in GitLab 13.5, this endpoint can be accessed without admin authentication. ```plaintext GET /users/:id/gpg_keys/:key_id @@ -1209,7 +1210,9 @@ Returns: - `201 OK` on success. - `404 User Not Found` if user cannot be found. -- `403 Forbidden` when trying to block an already blocked user by LDAP synchronization. +- `403 Forbidden` when trying to block: + - A user that is blocked through LDAP. + - An internal user. ## Unblock user @@ -1246,7 +1249,8 @@ Returns: - `404 User Not Found` if user cannot be found. - `403 Forbidden` when trying to deactivate a user: - Blocked by admin or by LDAP synchronization. - - That has any activity in past 180 days. These users cannot be deactivated. + - That has any activity in past 90 days. These users cannot be deactivated. + - That is internal. ## Activate user diff --git a/doc/api/v3_to_v4.md b/doc/api/v3_to_v4.md index 4139438bea0..c351c14e24c 100644 --- a/doc/api/v3_to_v4.md +++ b/doc/api/v3_to_v4.md @@ -1,6 +1,6 @@ # API V3 to API V4 -Since GitLab 9.0, API V4 is the preferred version to be used. +In GitLab 9.0 and later, API V4 is the preferred version to be used. API V3 was unsupported from GitLab 9.5, released on August 22, 2017. API v3 was removed in [GitLab 11.0](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/36819). diff --git a/doc/api/vulnerabilities.md b/doc/api/vulnerabilities.md index 73c765e2ccc..f89f2bda2eb 100644 --- a/doc/api/vulnerabilities.md +++ b/doc/api/vulnerabilities.md @@ -220,3 +220,53 @@ Example response: "closed_at": null } ``` + +## Revert vulnerability to detected state + +Reverts a given vulnerability to detected state. Returns status code `304` if the vulnerability is already in detected state. + +If an authenticated user does not have permission to +[revert vulnerability to detected state](../user/permissions.md#project-members-permissions), +this request will result in a `403` status code. + +```plaintext +POST /vulnerabilities/:id/revert +``` + +| Attribute | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `id` | integer or string | yes | The ID of a vulnerability to revert to detected state | + +```shell +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/vulnerabilities/5/dismiss" +``` + +Example response: + +```json +{ + "id": 2, + "title": "Predictable pseudorandom number generator", + "description": null, + "state": "detected", + "severity": "medium", + "confidence": "medium", + "report_type": "sast", + "project": { + "id": 32, + "name": "security-reports", + "full_path": "/gitlab-examples/security/security-reports", + "full_name": "gitlab-examples / security / security-reports" + }, + "author_id": 1, + "updated_by_id": null, + "last_edited_by_id": null, + "closed_by_id": null, + "start_date": null, + "due_date": null, + "created_at": "2019-10-13T15:08:40.219Z", + "updated_at": "2019-10-13T15:09:40.382Z", + "last_edited_at": null, + "closed_at": null +} +``` diff --git a/doc/api/vulnerability_exports.md b/doc/api/vulnerability_exports.md index d19d41b647e..0ee8a18a46a 100644 --- a/doc/api/vulnerability_exports.md +++ b/doc/api/vulnerability_exports.md @@ -198,18 +198,18 @@ The response will be `404 Not Found` if the vulnerability export is not finished Example response: ```csv -Group Name,Project Name,Scanner Type,Scanner Name,Status,Vulnerability,Details,Additional Info,Severity,CVE -Gitlab.org,Defend,container_scanning,Clair,confirmed,CVE-2017-16997 in glibc,,CVE-2017-16997 in glibc,critical,CVE-2017-16997 +Group Name,Project Name,Scanner Type,Scanner Name,Status,Vulnerability,Details,Additional Info,Severity,CVE,CWE,Other Identifiers +Gitlab.org,Defend,container_scanning,Clair,detected,CVE-2017-16997 in glibc,,CVE-2017-16997 in glibc,critical,CVE-2017-16997 Gitlab.org,Defend,container_scanning,Clair,detected,CVE-2017-18269 in glibc,,CVE-2017-18269 in glibc,critical,CVE-2017-18269 Gitlab.org,Defend,container_scanning,Clair,detected,CVE-2018-1000001 in glibc,,CVE-2018-1000001 in glibc,high,CVE-2018-1000001 Gitlab.org,Defend,container_scanning,Clair,detected,CVE-2016-10228 in glibc,,CVE-2016-10228 in glibc,medium,CVE-2016-10228 -Gitlab.org,Defend,container_scanning,Clair,confirmed,CVE-2010-4052 in glibc,,CVE-2010-4052 in glibc,low,CVE-2010-4052 +Gitlab.org,Defend,container_scanning,Clair,detected,CVE-2010-4052 in glibc,,CVE-2010-4052 in glibc,low,CVE-2010-4052 Gitlab.org,Defend,container_scanning,Clair,detected,CVE-2018-18520 in elfutils,,CVE-2018-18520 in elfutils,low,CVE-2018-18520 -Gitlab.org,Defend,container_scanning,Clair,detected,CVE-2018-16869 in nettle,,CVE-2018-16869 in nettle,unknown,CVE-2018-16869 -Gitlab.org,Defend,dependency_scanning,Gemnasium,detected,Regular Expression Denial of Service in debug,,Regular Expression Denial of Service in debug,unknown,yarn.lock:debug:gemnasium:37283ed4-0380-40d7-ada7-2d994afcc62a -Gitlab.org,Defend,dependency_scanning,Gemnasium,detected,Authentication bypass via incorrect DOM traversal and canonicalization in saml2-js,,Authentication bypass via incorrect DOM traversal and canonicalization in saml2-js,unknown,yarn.lock:saml2-js:gemnasium:9952e574-7b5b-46fa-a270-aeb694198a98 -Gitlab.org,Defend,sast,Find Security Bugs,detected,Predictable pseudorandom number generator,,Predictable pseudorandom number generator,medium,818bf5dacb291e15d9e6dc3c5ac32178:PREDICTABLE_RANDOM:src/main/java/com/gitlab/security_products/tests/App.java:47 -Gitlab.org,Defend,sast,Find Security Bugs,detected,Cipher with no integrity,,Cipher with no integrity,medium,e6449b89335daf53c0db4c0219bc1634:CIPHER_INTEGRITY:src/main/java/com/gitlab/security_products/tests/App.java:29 -Gitlab.org,Defend,sast,Find Security Bugs,detected,Predictable pseudorandom number generator,,Predictable pseudorandom number generator,medium,e8ff1d01f74cd372f78da8f5247d3e73:PREDICTABLE_RANDOM:src/main/java/com/gitlab/security_products/tests/App.java:41 -Gitlab.org,Defend,sast,Find Security Bugs,confirmed,ECB mode is insecure 2,,ECB mode is insecure,medium,ea0f905fc76f2739d5f10a1fd1e37a10:ECB_MODE:src/main/java/com/gitlab/security_products/tests/App.java:29 -Gitlab.org,Defend,``` +Gitlab.org,Defend,container_scanning,Clair,detected,CVE-2018-16869 in nettle,,CVE-2018-16869 in nettle,unknown,CVE-2018-16869,CWE-1 +Gitlab.org,Defend,dependency_scanning,Gemnasium,detected,Regular Expression Denial of Service in debug,,Regular Expression Denial of Service in debug,unknown,CVE-2021-1234,CWE-2,"""yarn.lock:debug:gemnasium:37283ed4-0380-40d7-ada7-2d994afcc62a""" +Gitlab.org,Defend,dependency_scanning,Gemnasium,detected,Authentication bypass via incorrect DOM traversal and canonicalization in saml2-js,,Authentication bypass via incorrect DOM traversal and canonicalization in saml2-js,unknown,,,"""yarn.lock:saml2-js:gemnasium:9952e574-7b5b-46fa-a270-aeb694198a98""" +Gitlab.org,Defend,sast,Find Security Bugs,detected,Predictable pseudorandom number generator,,Predictable pseudorandom number generator,medium,,,"""818bf5dacb291e15d9e6dc3c5ac32178:PREDICTABLE_RANDOM:src/main/java/com/gitlab/security_products/tests/App.java:47""" +Gitlab.org,Defend,sast,Find Security Bugs,detected,Cipher with no integrity,,Cipher with no integrity,medium,,,"""e6449b89335daf53c0db4c0219bc1634:CIPHER_INTEGRITY:src/main/java/com/gitlab/security_products/tests/App.java:29""" +Gitlab.org,Defend,sast,Find Security Bugs,detected,Predictable pseudorandom number generator,,Predictable pseudorandom number generator,medium,,,"""e8ff1d01f74cd372f78da8f5247d3e73:PREDICTABLE_RANDOM:src/main/java/com/gitlab/security_products/tests/App.java:41""" +Gitlab.org,Defend,sast,Find Security Bugs,detected,ECB mode is insecure,,ECB mode is insecure,medium,,,"""ea0f905fc76f2739d5f10a1fd1e37a10:ECB_MODE:src/main/java/com/gitlab/security_products/tests/App.java:29""" +``` diff --git a/doc/api/wikis.md b/doc/api/wikis.md index 7d16a5a38ee..a8c002d4fac 100644 --- a/doc/api/wikis.md +++ b/doc/api/wikis.md @@ -5,7 +5,7 @@ info: "To determine the technical writer assigned to the Stage/Group associated type: reference, api --- -# Wikis API +# Project wikis API > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/13372) in GitLab 10.0. diff --git a/doc/architecture/blueprints/cloud_native_build_logs/index.md b/doc/architecture/blueprints/cloud_native_build_logs/index.md new file mode 100644 index 00000000000..25abfe36e88 --- /dev/null +++ b/doc/architecture/blueprints/cloud_native_build_logs/index.md @@ -0,0 +1,141 @@ +--- +comments: false +description: 'Next iteration of build logs architecture at GitLab' +--- + +# Cloud Native Build Logs + +Cloud native and the adoption of Kubernetes has been recognised by GitLab to be +one of the top two biggest tailwinds that are helping us grow faster as a +company behind the project. + +This effort is described in a more details [in the infrastructure team +handbook](https://about.gitlab.com/handbook/engineering/infrastructure/production/kubernetes/gitlab-com/). + +## Traditional build logs + +Traditional job logs depend a lot on availability of a local shared storage. + +Every time a GitLab Runner sends a new partial build output, we write this +output to a file on a disk. This is simple, but this mechanism depends on +shared local storage - the same file needs to be available on every GitLab web +node machine, because GitLab Runner might connect to a different one every time +it performs an API request. Sidekiq also needs access to the file because when +a job is complete, a trace file contents will be sent to the object store. + +## New architecture + +New architecture writes data to Redis instead of writing build logs into a +file. + +In order to make this performant and resilient enough, we implemented a chunked +I/O mechanism - we store data in Redis in chunks, and migrate them to an object +store once we reach a desired chunk size. + +Simplified sequence diagram is available below. + +```mermaid +sequenceDiagram + autonumber + participant U as User + participant R as Runner + participant G as GitLab (rails) + participant I as Redis + participant D as Database + participant O as Object store + + loop incremental trace update sent by a runner + Note right of R: Runner appends a build trace + R->>+G: PATCH trace [build.id, offset, data] + G->>+D: find or create chunk [chunk.index] + D-->>-G: chunk [id, index] + G->>I: append chunk data [chunk.index, data] + G-->>-R: 200 OK + end + + Note right of R: User retrieves a trace + U->>+G: GET build trace + loop every trace chunk + G->>+D: find chunk [index] + D-->>-G: chunk [id] + G->>+I: read chunk data [chunk.index] + I-->>-G: chunk data [data, size] + end + G-->>-U: build trace + + Note right of R: Trace chunk is full + R->>+G: PATCH trace [build.id, offset, data] + G->>+D: find or create chunk [chunk.index] + D-->>-G: chunk [id, index] + G->>I: append chunk data [chunk.index, data] + G->>G: chunk full [index] + G-->>-R: 200 OK + G->>+I: read chunk data [chunk.index] + I-->>-G: chunk data [data, size] + G->>O: send chunk data [data, size] + G->>+D: update data store type [chunk.id] + G->>+I: delete chunk data [chunk.index] +``` + +## NFS coupling + +In 2017, we experienced serious problems of scaling our NFS infrastructure. We +even tried to replace NFS with +[CephFS](https://docs.ceph.com/docs/master/cephfs/) - unsuccessfully. + +Since that time it has become apparent that the cost of operations and +maintenance of a NFS cluster is significant and that if we ever decide to +migrate to Kubernetes [we need to decouple GitLab from a shared local storage +and +NFS](https://gitlab.com/gitlab-org/gitlab-pages/-/issues/426#note_375646396). + +1. NFS might be a single point of failure +1. NFS can only be reliably scaled vertically +1. Moving to Kubernetes means increasing the number of mount points by an order + of magnitude +1. NFS depends on extremely reliable network which can be difficult to provide + in Kubernetes environment +1. Storing customer data on NFS involves additional security risks + +Moving GitLab to Kubernetes without NFS decoupling would result in an explosion +of complexity, maintenance cost and enormous, negative impact on availability. + +## Iterations + +1. ✓ Implement the new architecture in way that it does not depend on shared local storage +1. ✓ Evaluate performance and edge-cases, iterate to improve the new architecture +1. ✓ Design cloud native build logs correctness verification mechanisms +1. ✓ Build observability mechanisms around performance and correctness +1. Rollout the feature into production environment incrementally + +The work needed to make the new architecture production ready and enabled on +GitLab.com is being tracked in [Cloud Native Build Logs on +GitLab.com](https://gitlab.com/groups/gitlab-org/-/epics/4275) epic. + +Enabling this feature on GitLab.com is a subtask of [making the new +architecture generally +available](https://gitlab.com/groups/gitlab-org/-/epics/3791) for everyone. + +## Who + +Proposal: + +<!-- vale gitlab.Spelling = NO --> + +| Role | Who +|------------------------------|-------------------------| +| Author | Grzegorz Bizon | +| Architecture Evolution Coach | Gerardo Lopez-Fernandez | +| Engineering Leader | Darby Frey | +| Domain Expert | Kamil Trzciński | +| Domain Expert | Sean McGivern | + +DRIs: + +| Role | Who +|------------------------------|------------------------| +| Product | Jason Yavorska | +| Leadership | Darby Frey | +| Engineering | Grzegorz Bizon | + +<!-- vale gitlab.Spelling = YES --> diff --git a/doc/architecture/blueprints/cloud_native_gitlab_pages/index.md b/doc/architecture/blueprints/cloud_native_gitlab_pages/index.md new file mode 100644 index 00000000000..37e69d46ae1 --- /dev/null +++ b/doc/architecture/blueprints/cloud_native_gitlab_pages/index.md @@ -0,0 +1,135 @@ +--- +comments: false +description: 'Making GitLab Pages a Cloud Native application - architecture blueprint.' +--- + +# GitLab Pages New Architecture + +GitLab Pages is an important component of the GitLab product. It is mostly +being used to serve static content, and has a limited set of well defined +responsibilities. That being said, unfortunately it has become a blocker for +GitLab.com Kubernetes migration. + +Cloud Native and the adoption of Kubernetes has been recognised by GitLab to be +one of the top two biggest tailwinds that are helping us grow faster as a +company behind the project. + +This effort is described in more detail [in the infrastructure team handbook +page](https://about.gitlab.com/handbook/engineering/infrastructure/production/kubernetes/gitlab-com/). + +GitLab Pages is tightly coupled with NFS and in order to unblock Kubernetes +migration a significant change to GitLab Pages' architecture is required. This +is an ongoing work that we have started more than a year ago. This blueprint +might be useful to understand why it is important, and what is the roadmap. + +## How GitLab Pages Works + +GitLab Pages is a daemon designed to serve static content, written in +[Go](https://golang.org/). + +Initially, GitLab Pages has been designed to store static content on a local +shared block storage (NFS) in a hierarchical group > project directory +structure. Each directory, representing a project, was supposed to contain a +configuration file and static content that GitLab Pages daemon was supposed to +read and serve. + +```mermaid +graph LR + A(GitLab Rails) -- Writes new pages deployment --> B[(NFS)] + C(GitLab Pages) -. Reads static content .-> B +``` + +This initial design has become outdated because of a few reasons - NFS coupling +being one of them - and we decided to replace it with more "decoupled +service"-like architecture. The new architecture, that we are working on, is +described in this blueprint. + +## NFS coupling + +In 2017, we experienced serious problems of scaling our NFS infrastructure. We +even tried to replace NFS with +[CephFS](https://docs.ceph.com/docs/master/cephfs/) - unsuccessfully. + +Since that time it has become apparent that the cost of operations and +maintenance of a NFS cluster is significant and that if we ever decide to +migrate to Kubernetes [we need to decouple GitLab from a shared local storage +and +NFS](https://gitlab.com/gitlab-org/gitlab-pages/-/issues/426#note_375646396). + +1. NFS might be a single point of failure +1. NFS can only be reliably scaled vertically +1. Moving to Kubernetes means increasing the number of mount points by an order + of magnitude +1. NFS depends on extremely reliable network which can be difficult to provide + in Kubernetes environment +1. Storing customer data on NFS involves additional security risks + +Moving GitLab to Kubernetes without NFS decoupling would result in an explosion +of complexity, maintenance cost and enormous, negative impact on availability. + +## New GitLab Pages Architecture + +- GitLab Pages is going to source domains' configuration from GitLab's internal + API, instead of reading `config.json` files from a local shared storage. +- GitLab Pages is going to serve static content from Object Storage. + +```mermaid +graph TD + A(User) -- Pushes pages deployment --> B{GitLab} + C((GitLab Pages)) -. Reads configuration from API .-> B + C -. Reads static content .-> D[(Object Storage)] + C -- Serves static content --> E(Visitors) +``` + +This new architecture has been briefly described in [the blog +post](https://about.gitlab.com/blog/2020/08/03/how-gitlab-pages-uses-the-gitlab-api-to-serve-content/) +too. + +## Iterations + +1. ✓ Redesign GitLab Pages configuration source to use GitLab's API +1. ✓ Evaluate performance and build reliable caching mechanisms +1. ✓ Incrementally rollout the new source on GitLab.com +1. ✓ Make GitLab Pages API domains config source enabled by default +1. Enable experimentation with different servings through feature flags +1. Triangulate object store serving design through meaningful experiments +1. Design pages migration mechanisms that can work incrementally +1. Gradually migrate towards object storage serving on GitLab.com + +[GitLab Pages Architecture](https://gitlab.com/groups/gitlab-org/-/epics/1316) +epic with detailed roadmap is also available. + +## Who + +Proposal: + +<!-- vale gitlab.Spelling = NO --> + +| Role | Who +|------------------------------|-------------------------| +| Author | Grzegorz Bizon | +| Architecture Evolution Coach | Kamil Trzciński | +| Engineering Leader | Daniel Croft | +| Domain Expert | Grzegorz Bizon | +| Domain Expert | Vladimir Shushlin | +| Domain Expert | Jaime Martinez | + +DRIs: + +| Role | Who +|------------------------------|------------------------| +| Product | Jackie Porter | +| Leadership | Daniel Croft | +| Engineering | Kamil Trzciński | + +Domain Experts: + +| Role | Who +|------------------------------|------------------------| +| Domain Expert | Kamil Trzciński | +| Domain Expert | Grzegorz Bizon | +| Domain Expert | Vladimir Shushlin | +| Domain Expert | Jaime Martinez | +| Domain Expert | Krasimir Angelov | + +<!-- vale gitlab.Spelling = YES --> diff --git a/doc/architecture/blueprints/feature_flags_development/index.md b/doc/architecture/blueprints/feature_flags_development/index.md new file mode 100644 index 00000000000..0aeb2b51b39 --- /dev/null +++ b/doc/architecture/blueprints/feature_flags_development/index.md @@ -0,0 +1,140 @@ +--- +comments: false +description: 'Internal usage of Feature Flags for GitLab development' +--- + +# Usage of Feature Flags for GitLab development + +Usage of feature flags become crucial for the development of GitLab. The +feature flags are a convenient way to ship changes early, and safely rollout +them to wide audience ensuring that feature is stable and performant. + +Since the presence of feature is controlled with a dedicated condition, a +developer can decide for a best time for testing the feature, ensuring that +feature is not enable prematurely. + +## Challenges + +The extensive usage of feature flags poses a few challenges + +- Each feature flag that we add to codebase is a ~"technical debt" as it adds a + matrix of configurations. +- Testing each combination of feature flags is close to impossible, so we + instead try to optimise our testing of feature flags to the most common + scenarios. +- There's a growing challenge of maintaining a growing number of feature flags. + We sometimes forget how our feature flags are configured or why we haven't + yet removed the feature flag. +- The usage of feature flags can also be confusing to people outside of + development that might not fully understand dependence of ~feature or ~bug + fix on feature flag and how this feature flag is configured. Or if the feature + should be announced as part of release post. +- Maintaining feature flags poses additional challenge of having to manage + different configurations across different environments/target. We have + different configuration of feature flags for testing, for development, for + staging, for production and what is being shipped to our customers as part of + on-premise offering. + +## Goals + +The biggest challenge today with our feature flags usage is their implicit +nature. Feature flags are part of the codebase, making them hard to understand +outside of development function. + +We should aim to make our feature flag based development to be accessible to +any interested party. + +- developer / engineer + - can easily add a new feature flag, and configure it's state + - can quickly find who to reach if touches another feature flag + - can quickly find stale feature flags +- engineering manager + - can understand what feature flags her/his group manages +- engineering manager and director + - can understand how much ~"technical debt" is inflicted due to amount of feature flags that we have to manage + - can understand how many feature flags are added and removed in each release +- product manager and documentation writer + - can understand what features are gated by what feature flags + - can understand if feature and thus feature flag is generally available on GitLab.com + - can understand if feature and thus feature flag is enabled by default for on-premise installations +- delivery engineer + - can understand what feature flags are introduced and changed between subsequent deployments +- support and reliability engineer + - can understand how feature flags changed between releases: what feature flags become enabled, what removed + - can quickly find relevant information about feature flag to know individuals which might help with an ongoing support request or incident + +## Proposal + +To help with above goals we should aim to make our feature flags usage explicit +and understood by all involved parties. + +Introduce a YAML-described `feature-flags/<name-of-feature.yml>` that would +allow us to have: + +1. A central place where all feature flags are documented, +1. A description of why the given feature flag was introduced, +1. A what relevant issue and merge request it was introduced by, +1. Build automated documentation with all feature flags in the codebase, +1. Track how many feature flags are per given group +1. Track how many feature flags are added and removed between releases +1. Make this information easily accessible for all +1. Allow our customers to easily discover how to enable features and quickly + find out information what did change between different releases + +### The `YAML` + +```yaml +--- +name: ci_disallow_to_create_merge_request_pipelines_in_target_project +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/40724 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/235119 +group: group::progressive delivery +type: development +default_enabled: false +``` + +## Reasons + +These are reason why these changes are needed: + +- we have around 500 different feature flags today +- we have hard time tracking their usage +- we have ambiguous usage of feature flag with different `default_enabled:` and + different `actors` used +- we lack a clear indication who owns what feature flag and where to find + relevant informations +- we do not emphasise the desire to create feature flag rollout issue to + indicate that feature flag is in fact a ~"technical debt" +- we don't know exactly what feature flags we have in our codebase +- we don't know exactly how our feature flags are configured for different + environments: what is being used for `test`, what we ship for `on-premise`, + what is our settings for `staging`, `qa` and `production` + +## Iterations + +This work is being done as part of dedicated epic: [Improve internal usage of +Feature Flags](https://gitlab.com/groups/gitlab-org/-/epics/3551). This epic +describes a meta reasons for making these changes. + +## Who + +Proposal: + +<!-- vale gitlab.Spelling = NO --> + +| Role | Who +|------------------------------|-------------------------| +| Author | Kamil Trzciński | +| Architecture Evolution Coach | Gerardo Lopez-Fernandez | +| Engineering Leader | Kamil Trzciński | +| Domain Expert | Shinya Maeda | + +DRIs: + +| Role | Who +|------------------------------|------------------------| +| Product | Kenny Johnston | +| Leadership | Craig Gomes | +| Engineering | Kamil Trzciński | + +<!-- vale gitlab.Spelling = YES --> diff --git a/doc/architecture/index.md b/doc/architecture/index.md new file mode 100644 index 00000000000..0a2ade6b7b0 --- /dev/null +++ b/doc/architecture/index.md @@ -0,0 +1,9 @@ +--- +comments: false +description: 'Architecture Practice at GitLab' +--- + +# Architecture at GitLab + +- [Architecture at GitLab](https://about.gitlab.com/handbook/engineering/architecture/) +- [Architecture Workflow](https://about.gitlab.com/handbook/engineering/architecture/workflow/) diff --git a/doc/articles/artifactory_and_gitlab/index.md b/doc/articles/artifactory_and_gitlab/index.md deleted file mode 100644 index ed9fd135e7c..00000000000 --- a/doc/articles/artifactory_and_gitlab/index.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../../ci/examples/artifactory_and_gitlab/index.md' ---- - -This document was moved to [another location](../../ci/examples/artifactory_and_gitlab/index.md) diff --git a/doc/articles/how_to_configure_ldap_gitlab_ce/index.md b/doc/articles/how_to_configure_ldap_gitlab_ce/index.md deleted file mode 100644 index 2fbeb6f2506..00000000000 --- a/doc/articles/how_to_configure_ldap_gitlab_ce/index.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../../administration/auth/ldap/index.md' ---- - -This document was moved to [another location](../../administration/auth/ldap/index.md). diff --git a/doc/articles/how_to_configure_ldap_gitlab_ee/index.md b/doc/articles/how_to_configure_ldap_gitlab_ee/index.md deleted file mode 100644 index 2fbeb6f2506..00000000000 --- a/doc/articles/how_to_configure_ldap_gitlab_ee/index.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../../administration/auth/ldap/index.md' ---- - -This document was moved to [another location](../../administration/auth/ldap/index.md). diff --git a/doc/articles/how_to_install_git/index.md b/doc/articles/how_to_install_git/index.md deleted file mode 100644 index 62598101895..00000000000 --- a/doc/articles/how_to_install_git/index.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../../topics/git/how_to_install_git/index.md' ---- - -This document was moved to [another location](../../topics/git/how_to_install_git/index.md). diff --git a/doc/articles/index.md b/doc/articles/index.md deleted file mode 100644 index 4b965b0256f..00000000000 --- a/doc/articles/index.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../topics/index.md' ---- - -This document was moved to [another location](../topics/index.md) diff --git a/doc/articles/laravel_with_gitlab_and_envoy/index.md b/doc/articles/laravel_with_gitlab_and_envoy/index.md deleted file mode 100644 index fa4f6243410..00000000000 --- a/doc/articles/laravel_with_gitlab_and_envoy/index.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../../ci/examples/laravel_with_gitlab_and_envoy/index.md' ---- - -This document was moved to [another location](../../ci/examples/laravel_with_gitlab_and_envoy/index.md). diff --git a/doc/articles/numerous_undo_possibilities_in_git/index.md b/doc/articles/numerous_undo_possibilities_in_git/index.md deleted file mode 100644 index 83aac82db4e..00000000000 --- a/doc/articles/numerous_undo_possibilities_in_git/index.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../../topics/git/numerous_undo_possibilities_in_git/index.md' ---- - -This document was moved to [another location](../../topics/git/numerous_undo_possibilities_in_git/index.md). diff --git a/doc/articles/openshift_and_gitlab/index.md b/doc/articles/openshift_and_gitlab/index.md deleted file mode 100644 index 822d012aa3d..00000000000 --- a/doc/articles/openshift_and_gitlab/index.md +++ /dev/null @@ -1,3 +0,0 @@ ---- -redirect_to: '../../install/openshift_and_gitlab/index.html' ---- diff --git a/doc/articles/runner_autoscale_aws/index.md b/doc/articles/runner_autoscale_aws/index.md deleted file mode 100644 index fb769731256..00000000000 --- a/doc/articles/runner_autoscale_aws/index.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: 'https://docs.gitlab.com/runner/configuration/runner_autoscale_aws/index.html' ---- - -This document was moved to [another location](https://docs.gitlab.com/runner/configuration/runner_autoscale_aws/index.html). diff --git a/doc/ci/README.md b/doc/ci/README.md index 9d3f7f2a8f2..dca6d8baa79 100644 --- a/doc/ci/README.md +++ b/doc/ci/README.md @@ -16,7 +16,7 @@ through the [continuous methodologies](introduction/index.md#introduction-to-cic - Continuous Delivery (CD) - Continuous Deployment (CD) -NOTE: **Note:** +TIP: **Tip:** Out-of-the-box management systems can decrease hours spent on maintaining toolchains by 10% or more. Watch our ["Mastering continuous software development"](https://about.gitlab.com/webcast/mastering-ci-cd/) webcast to learn about continuous methods and how GitLab’s built-in CI can help you simplify and scale software development. @@ -24,7 +24,7 @@ webcast to learn about continuous methods and how GitLab’s built-in CI can hel ## Overview Continuous Integration works by pushing small code chunks to your -application's code base hosted in a Git repository, and, to every +application's codebase hosted in a Git repository, and to every push, run a pipeline of scripts to build, test, and validate the code changes before merging them into the main branch. @@ -73,13 +73,14 @@ to your needs:  +While building your `.gitlab-ci.yml`, you can use the [CI/CD configuration visualization](yaml/visualization.md) to facilate your writing experience. + For a broader overview, see the [CI/CD getting started](quick_start/README.md) guide. Once you're familiar with how GitLab CI/CD works, see the [`.gitlab-ci.yml` full reference](yaml/README.md) for all the attributes you can set and use. -NOTE: **Note:** GitLab CI/CD and [shared runners](runners/README.md#shared-runners) are enabled on GitLab.com and available for all users, limited only by the [pipeline quota](../user/gitlab_com/index.md#shared-runners). ## Concepts @@ -94,7 +95,7 @@ GitLab CI/CD uses a number of concepts to describe and run your build and deploy | [Job artifacts](pipelines/job_artifacts.md) | Output, use, and reuse job artifacts. | | [Cache dependencies](caching/index.md) | Cache your dependencies for a faster execution. | | [GitLab Runner](https://docs.gitlab.com/runner/) | Configure your own runners to execute your scripts. | -| [Pipeline efficiency](pipelines/pipeline_efficiency.md) | Configure your pipelines to run quickly and effienctly. | +| [Pipeline efficiency](pipelines/pipeline_efficiency.md) | Configure your pipelines to run quickly and efficiently. | ## Configuration diff --git a/doc/ci/caching/index.md b/doc/ci/caching/index.md index 50f7d5252d8..df41f7da2d9 100644 --- a/doc/ci/caching/index.md +++ b/doc/ci/caching/index.md @@ -24,7 +24,6 @@ how it is defined in `.gitlab-ci.yml`. ## Cache vs artifacts -NOTE: **Note:** Be careful if you use cache and artifacts to store the same path in your jobs as **caches are restored before artifacts** and the content could be overwritten. @@ -73,7 +72,6 @@ Artifacts: - Are always uploaded to GitLab (known as coordinator). - Can have an expiration value for controlling disk usage (30 days by default). -NOTE: **Note:** Both artifacts and caches define their paths relative to the project directory, and can't link to files outside it. @@ -204,9 +202,7 @@ runs of jobs for things like dependencies and commonly used libraries (Node.js packages, PHP packages, rubygems, Python libraries, etc.), so they don't have to be re-fetched from the public internet. -NOTE: **Note:** -For more examples, check out our [GitLab CI/CD -templates](https://gitlab.com/gitlab-org/gitlab/tree/master/lib/gitlab/ci/templates). +For more examples, check out our [GitLab CI/CD templates](https://gitlab.com/gitlab-org/gitlab/tree/master/lib/gitlab/ci/templates). ### Caching Node.js dependencies diff --git a/doc/ci/ci_cd_for_external_repos/bitbucket_integration.md b/doc/ci/ci_cd_for_external_repos/bitbucket_integration.md index 74c48f087b2..f8e2a2b7eb0 100644 --- a/doc/ci/ci_cd_for_external_repos/bitbucket_integration.md +++ b/doc/ci/ci_cd_for_external_repos/bitbucket_integration.md @@ -9,7 +9,7 @@ type: howto GitLab CI/CD can be used with Bitbucket Cloud by: -1. Creating a [CI/CD project](../../user/project/ci_cd_for_external_repo.md). +1. Creating a [CI/CD project](index.md). 1. Connecting your Git repository via URL. To use GitLab CI/CD with a Bitbucket Cloud repository: diff --git a/doc/ci/ci_cd_for_external_repos/github_integration.md b/doc/ci/ci_cd_for_external_repos/github_integration.md index 661d935fc1d..7ee9f98bd39 100644 --- a/doc/ci/ci_cd_for_external_repos/github_integration.md +++ b/doc/ci/ci_cd_for_external_repos/github_integration.md @@ -21,7 +21,6 @@ cannot be used to authenticate with GitHub as an external CI/CD repository. ## Connect with Personal Access Token -NOTE: **Note:** Personal access tokens can only be used to connect GitHub.com repositories to GitLab, and the GitHub user must have the [owner role](https://docs.github.com/en/github/getting-started-with-github/access-permissions-on-github). @@ -53,7 +52,6 @@ GitLab will: ## Connect manually -NOTE: **Note:** To use **GitHub Enterprise** with **GitLab.com**, use this method. To manually enable GitLab CI/CD for your repository: diff --git a/doc/ci/ci_cd_for_external_repos/index.md b/doc/ci/ci_cd_for_external_repos/index.md index 78988e8a057..c6590599849 100644 --- a/doc/ci/ci_cd_for_external_repos/index.md +++ b/doc/ci/ci_cd_for_external_repos/index.md @@ -74,7 +74,6 @@ If changes are pushed to the branch referenced by the Pull Request and the Pull Request is still open, a pipeline for the external pull request is created. -NOTE: **Note:** GitLab CI/CD will create 2 pipelines in this case. One for the branch push and one for the external pull request. diff --git a/doc/ci/cloud_deployment/index.md b/doc/ci/cloud_deployment/index.md index 6fa0e6d9475..af7df0e1153 100644 --- a/doc/ci/cloud_deployment/index.md +++ b/doc/ci/cloud_deployment/index.md @@ -8,7 +8,10 @@ type: howto # Cloud deployment Interacting with a major cloud provider may have become a much needed task that's -part of your delivery process. GitLab is making this process less painful by providing Docker images +part of your delivery process. With GitLab you can +[deploy your application anywhere](https://about.gitlab.com/stages-devops-lifecycle/deploy-targets/). + +For some specific deployment targets, GitLab makes this process less painful by providing Docker images that come with the needed libraries and tools pre-installed. By referencing them in your CI/CD pipeline, you'll be able to interact with your chosen cloud provider more easily. @@ -200,3 +203,81 @@ deploy: script: - aws ecs register-task-definition ... ``` + +### Provision and deploy to your AWS Elastic Compute Cloud (EC2) + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/201742) in GitLab 13.5. + +You can use the `AWS/CF-Provision-and-Deploy-EC2` CI template to perform the +following actions within the same pipeline: + +1. **Create stack**: Provision your own infrastructure by leveraging the [AWS CloudFormation](https://aws.amazon.com/cloudformation/) API. +1. **Push to S3**: Push your previously-built artifact to an [AWS S3](https://aws.amazon.com/s3/) bucket. +1. **Deploy to EC2**: Deploy this pushed content onto an [AWS EC2](https://aws.amazon.com/ec2/) instance. + + + +#### Run the `AWS/CF-Provision-and-Deploy-EC2.gitlab-ci.yml` template + +To run the `AWS/CF-Provision-and-Deploy-EC2.gitlab-ci.yml` template, you must +pass three JSON input objects, based on existing templates: + +1. The AWS documentation provides templates for the _Create stack_ and _Deploy to EC2_ steps (links + below). We provide the template for the remaining step, _Push to S3_: + + - [Template for the _Create stack_ step on AWS](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/template-anatomy.html). + - Template for the _Push to S3_ step. Note that `source` is where a preceding `build` job built + your application, exporting the build through [`artifacts:paths`](../yaml/README.md#artifactspaths): + + ```json + { + "applicationName": "string", + "source": "string", + "s3Location": "s3://your/bucket/project_built_file...]" + } + ``` + + - [Template for the _Deploy to EC2_ step on AWS](https://docs.aws.amazon.com/codedeploy/latest/APIReference/API_CreateDeployment.html). + +1. Once you have completed these three templates based on your requirements, you + have two ways to pass in these JSON objects: + + - They can be three actual files located in your project. You must specify their path relative to + your project root in your `.gitlab-ci.yml` file, using the following variables. For example, if + your files are in a `<project_root>/aws` folder: + + ```yaml + variables: + CI_AWS_CF_CREATE_STACK_FILE: 'aws/cf_create_stack.json' + CI_AWS_S3_PUSH_FILE: 'aws/s3_push.json' + CI_AWS_EC2_DEPLOYMENT_FILE: 'aws/create_deployment.json' + ``` + + - Alternatively, you can provide these JSON objects as [file-typed environment variables](../variables/README.md#custom-environment-variables-of-type-file). + In your project, go to **Settings > CI / CD > Variables** and add + the three variables listed above as file-typed environment variables. + For each variable, set the value to its corresponding JSON object. + +1. Provide the name of the stack you're creating and/or targeting, as an environment variable: + + ```yaml + variables: + CI_AWS_CF_STACK_NAME: 'YourStackName' + ``` + +1. Add this CI template to your `.gitlab-ci.yml`: + + ```yaml + include: + - template: AWS/CF-Provision-and-Deploy-EC2.gitlab-ci.yml + ``` + +When running your project pipeline at this point: + +- Your AWS CloudFormation stack is created based on the content of your + `CI_AWS_CF_CREATE_STACK_FILE` file/variable. + If your stack already exists, this step is skipped, but the `provision` job it belongs to still + runs. +- Your built application is pushed to your S3 bucket then and deployed to your EC2 instance, based + on the related JSON object's content. The deployment job finishes whenever the deployment to EC2 + is done or has failed. diff --git a/doc/ci/docker/using_docker_build.md b/doc/ci/docker/using_docker_build.md index 045fcd39c4d..e3123cde1cd 100644 --- a/doc/ci/docker/using_docker_build.md +++ b/doc/ci/docker/using_docker_build.md @@ -48,7 +48,6 @@ The simplest approach is to install GitLab Runner in `shell` execution mode. GitLab Runner then executes job scripts as the `gitlab-runner` user. 1. Install [GitLab Runner](https://gitlab.com/gitlab-org/gitlab-runner/#installation). - 1. During GitLab Runner installation select `shell` as method of executing job scripts or use command: ```shell @@ -90,7 +89,6 @@ GitLab Runner then executes job scripts as the `gitlab-runner` user. 1. You can now use `docker` command (and **install** `docker-compose` if needed). -NOTE: **Note:** By adding `gitlab-runner` to the `docker` group you are effectively granting `gitlab-runner` full root permissions. For more information please read [On Docker security: `docker` group considered harmful](https://www.andreas-jung.com/contents/on-docker-security-docker-group-considered-harmful). @@ -101,7 +99,6 @@ The second approach is to use the special Docker-in-Docker (dind) (`docker`) and run the job script in context of that image in privileged mode. -NOTE: **Note:** `docker-compose` is not part of Docker-in-Docker (dind). To use `docker-compose` in your CI builds, follow the `docker-compose` [installation instructions](https://docs.docker.com/compose/install/). @@ -149,22 +146,17 @@ released. #### TLS enabled -NOTE: **Note:** -Requires GitLab Runner 11.11 or later, but is not supported if GitLab -Runner is installed using the [Helm -chart](https://docs.gitlab.com/runner/install/kubernetes.html). See the -[related -issue](https://gitlab.com/gitlab-org/charts/gitlab-runner/-/issues/83) for -details. - The Docker daemon supports connection over TLS and it's done by default for Docker 19.03.12 or higher. This is the **suggested** way to use the Docker-in-Docker service and [GitLab.com shared runners](../../user/gitlab_com/index.md#shared-runners) support this. -1. Install [GitLab Runner](https://docs.gitlab.com/runner/install/). +GitLab Runner 11.11 or later is required, but it is not supported if GitLab +Runner is installed using the [Helm chart](https://docs.gitlab.com/runner/install/kubernetes.html). +See the [related issue](https://gitlab.com/gitlab-org/charts/gitlab-runner/-/issues/83) for details. +1. Install [GitLab Runner](https://docs.gitlab.com/runner/install/). 1. Register GitLab Runner from the command line to use `docker` and `privileged` mode: @@ -225,7 +217,7 @@ support this. # The 'docker' hostname is the alias of the service container as described at # https://docs.gitlab.com/ee/ci/docker/using_docker_images.html#accessing-the-services. # - # Note that if you're using GitLab Runner 12.7 or earlier with the Kubernetes executor and Kubernetes 1.6 or earlier, + # If you're using GitLab Runner 12.7 or earlier with the Kubernetes executor and Kubernetes 1.6 or earlier, # the variable must be set to tcp://localhost:2376 because of how the # Kubernetes executor connects services to the job container # DOCKER_HOST: tcp://localhost:2376 @@ -287,7 +279,7 @@ variables: # The 'docker' hostname is the alias of the service container as described at # https://docs.gitlab.com/ee/ci/docker/using_docker_images.html#accessing-the-services # - # Note that if you're using GitLab Runner 12.7 or earlier with the Kubernetes executor and Kubernetes 1.6 or earlier, + # If you're using GitLab Runner 12.7 or earlier with the Kubernetes executor and Kubernetes 1.6 or earlier, # the variable must be set to tcp://localhost:2375 because of how the # Kubernetes executor connects services to the job container # DOCKER_HOST: tcp://localhost:2375 @@ -324,7 +316,6 @@ are done to the services as well, making these incompatible. In order to do that, follow the steps: 1. Install [GitLab Runner](https://docs.gitlab.com/runner/install/). - 1. Register GitLab Runner from the command line to use `docker` and share `/var/run/docker.sock`: ```shell @@ -506,14 +497,13 @@ environment = ["DOCKER_DRIVER=overlay2"] If you're running multiple runners, you have to modify all configuration files. -NOTE: **Note:** Read more about the [runner configuration](https://docs.gitlab.com/runner/configuration/) and [using the OverlayFS storage driver](https://docs.docker.com/engine/userguide/storagedriver/overlayfs-driver/). ## Using the GitLab Container Registry Once you've built a Docker image, you can push it up to the built-in -[GitLab Container Registry](../../user/packages/container_registry/index.md#build-and-push-images-using-gitlab-cicd). +[GitLab Container Registry](../../user/packages/container_registry/index.md#build-and-push-by-using-gitlab-cicd). ## Troubleshooting diff --git a/doc/ci/docker/using_docker_images.md b/doc/ci/docker/using_docker_images.md index 0fcd95c41ed..f7d54aa7d78 100644 --- a/doc/ci/docker/using_docker_images.md +++ b/doc/ci/docker/using_docker_images.md @@ -138,7 +138,6 @@ still succeeds even if that warning was printed. For example: As it was mentioned before, this feature is designed to provide **network accessible** services. A database is the simplest example of such a service. -NOTE: **Note:** The services feature is not designed to, and does not add any software from the defined `services` image(s) to the job's container. @@ -186,7 +185,6 @@ access to it from your build container under two hostnames to choose from: - `tutum-wordpress` - `tutum__wordpress` -NOTE: **Note:** Hostnames with underscores are not RFC valid and may cause problems in 3rd party applications. @@ -364,10 +362,9 @@ For example, the following two definitions are equal: | `name` | yes, when used with any other option | 9.4 | Full name of the image that should be used. It should contain the Registry part if needed. | | `entrypoint` | no | 9.4 |Command or script that should be executed as the container's entrypoint. It's translated to Docker's `--entrypoint` option while creating the container. The syntax is similar to [`Dockerfile`'s `ENTRYPOINT`](https://docs.docker.com/engine/reference/builder/#entrypoint) directive, where each shell token is a separate string in the array. | | `command` | no | 9.4 |Command or script that should be used as the container's command. It's translated to arguments passed to Docker after the image's name. The syntax is similar to [`Dockerfile`'s `CMD`](https://docs.docker.com/engine/reference/builder/#cmd) directive, where each shell token is a separate string in the array. | -| `alias` | no | 9.4 |Additional alias that can be used to access the service from the job's container. Read [Accessing the services](#accessing-the-services) for more information. | +| `alias` (1) | no | 9.4 |Additional alias that can be used to access the service from the job's container. Read [Accessing the services](#accessing-the-services) for more information. | -NOTE: **Note:** -Alias support for the Kubernetes executor was [introduced](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/2229) in GitLab Runner 12.8, and is only available for Kubernetes version 1.7 or later. +(1) Alias support for the Kubernetes executor was [introduced](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/2229) in GitLab Runner 12.8, and is only available for Kubernetes version 1.7 or later. ### Starting multiple services from the same image @@ -532,7 +529,6 @@ To define which should be used, the GitLab Runner process reads the configuratio If the `--user` flag is provided to run the GitLab Runner child processes as unprivileged user, the home directory of the main GitLab Runner process user is used. -NOTE: **Note:** GitLab Runner reads this configuration **only** from `config.toml` and ignores it if it's provided as an environment variable. This is because GitLab Runner uses **only** `config.toml` configuration and does not interpolate **ANY** environment variables at @@ -547,6 +543,7 @@ runtime. at least version **1.8** if you want to use private registries. - Available for [Kubernetes executor](https://docs.gitlab.com/runner/executors/kubernetes.html) in GitLab Runner 13.1 and later. +- [Credentials Store](#using-credentials-store) and [Credential Helpers](#using-credential-helpers) require binaries to be added to the GitLab Runner's `$PATH`, and require access to do so. Therefore, these features are not available on shared runners or any other runner where the user does not have access to the environment where the runner is installed. ### Using statically-defined credentials @@ -600,7 +597,7 @@ There are two ways to determine the value of `DOCKER_AUTH_CONFIG`: Open a terminal and execute the following command: ```shell - # Note the use of "-n" - it prevents encoding a newline in the password. + # The use of "-n" - prevents encoding a newline in the password. echo -n "my_username:my_password" | base64 # Example output to copy @@ -650,7 +647,6 @@ follow these steps: You can add configuration for as many registries as you want, adding more registries to the `"auths"` hash as described above. -NOTE: **Note:** The full `hostname:port` combination is required everywhere for the runner to match the `DOCKER_AUTH_CONFIG`. For example, if `registry.example.com:5000/namespace/image:tag` is specified in `.gitlab-ci.yml`, @@ -679,17 +675,14 @@ To add `DOCKER_AUTH_CONFIG` to a runner: environment = ["DOCKER_AUTH_CONFIG={\"auths\":{\"registry.example.com:5000\":{\"auth\":\"bXlfdXNlcm5hbWU6bXlfcGFzc3dvcmQ=\"}}}"] ``` -1. Restart the runner service. - -NOTE: **Note:** -The double quotes included in the `DOCKER_AUTH_CONFIG` -data must be escaped with backslashes. This prevents them from being -interpreted as TOML. + - The double quotes included in the `DOCKER_AUTH_CONFIG` + data must be escaped with backslashes. This prevents them from being + interpreted as TOML. + - The `environment` option is a list. Your runner may + have existing entries and you should add this to the list, not replace + it. -NOTE: **Note:** -The `environment` option is a list. So your runner may -have existing entries and you should add this to the list, not replace -it. +1. Restart the runner service. ### Using Credentials Store @@ -717,10 +710,9 @@ To configure credentials store, follow these steps: `${GITLAB_RUNNER_HOME}/.docker/config.json`. GitLab Runner reads this configuration file and uses the needed helper for this specific repository. -NOTE: **Note:** `credsStore` is used to access ALL the registries. -If you want to use both images from private registry and public images from DockerHub, -pulling from DockerHub would fail, because Docker daemon tries to use the same credentials for **ALL** the registries. +If you want to use both images from private registry and public images from Docker Hub, +pulling from Docker Hub would fail, because Docker daemon tries to use the same credentials for **ALL** the registries. ### Using Credential Helpers @@ -732,10 +724,8 @@ image which is private and requires you to log in into a private container regis To configure access for `aws_account_id.dkr.ecr.region.amazonaws.com`, follow these steps: 1. Make sure `docker-credential-ecr-login` is available in GitLab Runner's `$PATH`. - 1. Have any of the following [AWS credentials setup](https://github.com/awslabs/amazon-ecr-credential-helper#aws-credentials). Make sure that GitLab Runner can access the credentials. - 1. Make GitLab Runner use it. There are two ways to accomplish this. Either: - Create a [variable](../variables/README.md#gitlab-cicd-environment-variables) @@ -791,7 +781,6 @@ service containers. For all possible configuration variables check the documentation of each image provided in their corresponding Docker hub page. -NOTE: **Note:** All variables are passed to all services containers. It's not designed to distinguish which variable should go where. @@ -823,7 +812,6 @@ time. ## How to debug a job locally -NOTE: **Note:** The following commands are run without root privileges. You should be able to run Docker with your regular user account. diff --git a/doc/ci/docker/using_kaniko.md b/doc/ci/docker/using_kaniko.md index a62f4db4fe4..f9b09bada14 100644 --- a/doc/ci/docker/using_kaniko.md +++ b/doc/ci/docker/using_kaniko.md @@ -17,13 +17,13 @@ kaniko solves two problems with using the build](using_docker_build.md#use-docker-in-docker-workflow-with-docker-executor) method: - Docker-in-Docker requires [privileged mode](https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities) - in order to function, which is a significant security concern. + to function, which is a significant security concern. - Docker-in-Docker generally incurs a performance penalty and can be quite slow. ## Requirements -In order to utilize kaniko with GitLab, [a runner](https://docs.gitlab.com/runner/) -with one of the following executors is required: +To use kaniko with GitLab, [a runner](https://docs.gitlab.com/runner/) with one +of the following executors is required: - [Kubernetes](https://docs.gitlab.com/runner/executors/kubernetes.html). - [Docker](https://docs.gitlab.com/runner/executors/docker.html). diff --git a/doc/ci/enable_or_disable_ci.md b/doc/ci/enable_or_disable_ci.md index 46cf76637a4..8b88ec509e7 100644 --- a/doc/ci/enable_or_disable_ci.md +++ b/doc/ci/enable_or_disable_ci.md @@ -31,7 +31,6 @@ either: - Site-wide by modifying the settings in `gitlab.yml` and `gitlab.rb` for source and Omnibus installations respectively. -NOTE: **Note:** This only applies to pipelines run as part of GitLab CI/CD. This will not enable or disable pipelines that are run from an [external integration](../user/project/integrations/overview.md#integrations-listing). diff --git a/doc/ci/environments/index.md b/doc/ci/environments/index.md index 07d0dac6163..baf2156e64a 100644 --- a/doc/ci/environments/index.md +++ b/doc/ci/environments/index.md @@ -118,11 +118,9 @@ With this configuration, we: - Ensure that our app is able to be built successfully. - Lastly we deploy to the staging server. -NOTE: **Note:** -The `environment` keyword defines where the app is deployed. -The environment `name` and `url` is exposed in various places -within GitLab. Each time a job that has an environment specified -succeeds, a deployment is recorded, along with the Git SHA, and environment name. +Note that the `environment` keyword defines where the app is deployed. The environment `name` and +`url` is exposed in various places within GitLab. Each time a job that has an environment specified +succeeds, a deployment is recorded along with the Git SHA and environment name. CAUTION: **Caution:** Some characters are not allowed in environment names. Use only letters, @@ -288,12 +286,11 @@ You can find the "play" button in the pipelines, environments, deployments, and | Deployments |  | | Jobs |  | -Clicking on the play button in any view will trigger the `deploy_prod` job, and the -deployment will be recorded as a new environment named `production`. +Clicking the play button in any view triggers the `deploy_prod` job. The deployment is recorded as a +new environment named `production`. -NOTE: **Note:** -If your environment's name is `production` (all lowercase), -it will get recorded in [Value Stream Analytics](../../user/project/cycle_analytics.md). +If your environment's name is `production` (all lowercase), it's recorded in +[Value Stream Analytics](../../user/analytics/value_stream_analytics.md). ### Configuring dynamic environments @@ -371,9 +368,8 @@ For the value of: the example above: `https://$CI_COMMIT_REF_NAME.example.com`, which would give a URL of `https://100-do-the-thing.example.com`. -NOTE: **Note:** -You are not required to use the same prefix or only slashes (`/`) in the dynamic environments' -names. However, using this format will enable the [grouping similar environments](#grouping-similar-environments) +You aren't required to use the same prefix or only slashes (`/`) in the dynamic environments' names. +However, using this format enables the [grouping similar environments](#grouping-similar-environments) feature. ### Configuring Kubernetes deployments @@ -384,6 +380,12 @@ If you are deploying to a [Kubernetes cluster](../../user/project/clusters/index associated with your project, you can configure these deployments from your `gitlab-ci.yml` file. +NOTE: **Note:** +Kubernetes configuration isn't supported for Kubernetes clusters that are +[managed by GitLab](../../user/project/clusters/index.md#gitlab-managed-clusters). +To follow progress on support for GitLab-managed clusters, see the +[relevant issue](https://gitlab.com/gitlab-org/gitlab/-/issues/38054). + The following configuration options are supported: - [`namespace`](https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/) @@ -411,12 +413,6 @@ trace on the deployment job page:  -NOTE: **Note:** -Kubernetes configuration is not supported for Kubernetes clusters -that are [managed by GitLab](../../user/project/clusters/index.md#gitlab-managed-clusters). -To follow progress on support for GitLab-managed clusters, see the -[relevant issue](https://gitlab.com/gitlab-org/gitlab/-/issues/38054). - #### Configuring incremental rollouts Learn how to release production changes to only a portion of your Kubernetes pods with @@ -514,9 +510,8 @@ review_app: This example requires that NGINX and GitLab Runner are set up on the server this job will run on. -NOTE: **Note:** -See the [limitations](#limitations) section for some edge cases regarding the naming of -your branches and Review Apps. +See the [limitations](#limitations) section for some edge cases regarding the naming of your +branches and Review Apps. The complete example provides the following workflow to developers: @@ -617,13 +612,12 @@ To retry or rollback a deployment: #### What to expect with a rollback -Pressing the **Rollback** button on a specific commit will trigger a _new_ deployment with its -own unique job ID. - -This means that you will see a new deployment that points to the commit you are rolling back to. +Pressing the **Rollback** button on a specific commit triggers a _new_ deployment with its own +unique job ID. This means that you will see a new deployment that points to the commit you're +rolling back to. -NOTE: **Note:** -The defined deployment process in the job's `script` determines whether the rollback succeeds or not. +Note that the defined deployment process in the job's `script` determines whether the rollback +succeeds. ### Using the environment URL @@ -662,9 +656,8 @@ Stopping an environment: This is often used when multiple developers are working on a project at the same time, each of them pushing to their own branches, causing many dynamic environments to be created. -NOTE: **Note:** -Starting with GitLab 8.14, dynamic environments are stopped automatically -when their associated branch is deleted. +Starting with GitLab 8.14, dynamic environments stop automatically when their associated branch is +deleted. #### Automatically stopping an environment @@ -721,29 +714,25 @@ You can read more in the [`.gitlab-ci.yml` reference](../yaml/README.md#environm > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/20956) in GitLab 12.8. -You can set a expiry time to environments and stop them automatically after a certain period. +You can set an expiry time for environments and stop them automatically after a certain period. -For example, consider the use of this feature with Review Apps environments. -When you set up Review Apps, sometimes they keep running for a long time -because some merge requests are left as open. An example for this situation is when the author of the merge -request is not actively working on it, due to priority changes or a different approach was decided on, and the merge requests was simply forgotten. -Idle environments waste resources, therefore they -should be terminated as soon as possible. +For example, consider the use of this feature with Review App environments. When you set up Review +Apps, sometimes they keep running for a long time because some merge requests are left open and +forgotten. Such idle environments waste resources and should be terminated as soon as possible. -To address this problem, you can specify an optional expiration date for -Review Apps environments. When the expiry time is reached, GitLab will automatically trigger a job -to stop the environment, eliminating the need of manually doing so. In case an environment is updated, the expiration is renewed -ensuring that only active merge requests keep running Review Apps. +To address this problem, you can specify an optional expiration date for Review App environments. +When the expiry time is reached, GitLab automatically triggers a job to stop the environment, +eliminating the need of manually doing so. In case an environment is updated, the expiration is +renewed ensuring that only active merge requests keep running Review Apps. -To enable this feature, you need to specify the [`environment:auto_stop_in`](../yaml/README.md#environmentauto_stop_in) keyword in `.gitlab-ci.yml`. -You can specify a human-friendly date as the value, such as `1 hour and 30 minutes` or `1 day`. -`auto_stop_in` uses the same format of [`artifacts:expire_in` docs](../yaml/README.md#artifactsexpire_in). +To enable this feature, you must specify the [`environment:auto_stop_in`](../yaml/README.md#environmentauto_stop_in) +keyword in `.gitlab-ci.yml`. You can specify a human-friendly date as the value, such as +`1 hour and 30 minutes` or `1 day`. `auto_stop_in` uses the same format of +[`artifacts:expire_in` docs](../yaml/README.md#artifactsexpire_in). -NOTE: **Note:** -Due to the resource limitation, a background worker for stopping environments only -runs once every hour. This means environments will not be stopped at the exact -timestamp as the specified period, but will be stopped when the hourly cron worker -detects expired environments. +Note that due to resource limitation, a background worker for stopping environments only runs once +every hour. This means that environments aren't stopped at the exact timestamp specified, but are +instead stopped when the hourly cron worker detects expired environments. ##### Auto-stop example @@ -866,7 +855,7 @@ exist, you should see something like: ### Environment incident management -You have successfuly setup a Continous Delivery/Deployment workflow in your project. +You have successfully setup a Continuous Delivery/Deployment workflow in your project. Production environments can go down unexpectedly, including for reasons outside of your own control. For example, issues with external dependencies, infrastructure, or human error can cause major issues with an environment. This could include: @@ -903,8 +892,7 @@ you can monitor the behavior of your app running in each environment. For the mo dashboard to appear, you need to Configure Prometheus to collect at least one [supported metric](../../user/project/integrations/prometheus_library/index.md). -NOTE: **Note:** -Since GitLab 9.2, all deployments to an environment are shown directly on the monitoring dashboard. +In GitLab 9.2 and later, all deployments to an environment are shown directly on the monitoring dashboard. Once configured, GitLab will attempt to retrieve [supported performance metrics](../../user/project/integrations/prometheus_library/index.md) for any environment that has had a successful deployment. If monitoring data was @@ -938,6 +926,11 @@ This is a powerful feature that allows you to debug issues without leaving the c of your web browser. To enable it, just follow the instructions given in the service integration documentation. +NOTE: **Note:** +Container-based deployments often lack basic tools (like an editor), and may +be stopped or restarted at any time. If this happens, you will lose all your +changes. Treat this as a debugging tool, not a comprehensive online IDE. + Once enabled, your environments will gain a "terminal" button:  @@ -961,14 +954,9 @@ by your deployment so you can: You can open multiple terminals to the same environment, they each get their own shell session and even a multiplexer like `screen` or `tmux`. -NOTE: **Note:** -Container-based deployments often lack basic tools (like an editor), and may -be stopped or restarted at any time. If this happens, you will lose all your -changes. Treat this as a debugging tool, not a comprehensive online IDE. - ### Check out deployments locally -Since GitLab 8.13, a reference in the Git repository is saved for each deployment, so +In GitLab 8.13 and later, a reference in the Git repository is saved for each deployment, so knowing the state of your current environments is only a `git fetch` away. In your Git configuration, append the `[remote "<your-remote>"]` block with an extra @@ -1024,9 +1012,8 @@ As you can see, you can use specific matching for selecting a particular environ and also use wildcard matching (`*`) for selecting a particular environment group, such as [Review Apps](../review_apps/index.md) (`review/*`). -NOTE: **Note:** -The most _specific_ spec takes precedence over the other wildcard matching. -In this case, `review/feature-1` spec takes precedence over `review/*` and `*` specs. +Note that the most _specific_ spec takes precedence over the other wildcard matching. In this case, +the `review/feature-1` spec takes precedence over `review/*` and `*` specs. ### Environments Dashboard **(PREMIUM)** diff --git a/doc/ci/environments/protected_environments.md b/doc/ci/environments/protected_environments.md index 5dda0cc81f6..87bced29906 100644 --- a/doc/ci/environments/protected_environments.md +++ b/doc/ci/environments/protected_environments.md @@ -9,8 +9,6 @@ type: concepts, howto > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/6303) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.3. -## Overview - [Environments](../environments/index.md) can be used for different reasons: - Some of them are just for testing. diff --git a/doc/ci/examples/artifactory_and_gitlab/index.md b/doc/ci/examples/artifactory_and_gitlab/index.md index 2abb2cc1b0d..e2ee05923cd 100644 --- a/doc/ci/examples/artifactory_and_gitlab/index.md +++ b/doc/ci/examples/artifactory_and_gitlab/index.md @@ -23,10 +23,10 @@ We also assume that an Artifactory instance is available and reachable from the ## Create the simple Maven dependency -First of all, you need an application to work with: in this specific case we will -use a simple one, but it could be any Maven application. This will be the -dependency you want to package and deploy to Artifactory, in order to be -available to other projects. +First, you need an application to work with: in this specific case we'll use a +simple one, but it could be any Maven application. This will be the dependency +you want to package and deploy to Artifactory, to be available to other +projects. ### Prepare the dependency application @@ -58,7 +58,7 @@ The application is ready to use, but you need some additional steps to deploy it 1. Log in to Artifactory with your user's credentials. 1. From the main screen, click on the `libs-release-local` item in the **Set Me Up** panel. 1. Copy to clipboard the configuration snippet under the **Deploy** paragraph. -1. Change the `url` value in order to have it configurable via variables. +1. Change the `url` value to have it configurable by using variables. 1. Copy the snippet in the `pom.xml` file for your project, just after the `dependencies` section. The snippet should look like this: @@ -146,8 +146,9 @@ deploy: - master ``` -The runner will use the latest [Maven Docker image](https://hub.docker.com/_/maven/), which already contains all the tools and the dependencies you need to manage the project, -in order to run the jobs. +The runner uses the latest [Maven Docker image](https://hub.docker.com/_/maven/), +which contains all of the tools and dependencies needed to manage the project +and to run the jobs. Environment variables are set to instruct Maven to use the `homedir` of the repository instead of the user's home when searching for configuration and dependencies. diff --git a/doc/ci/examples/authenticating-with-hashicorp-vault/index.md b/doc/ci/examples/authenticating-with-hashicorp-vault/index.md index 73896547675..4a0ff2fa6ac 100644 --- a/doc/ci/examples/authenticating-with-hashicorp-vault/index.md +++ b/doc/ci/examples/authenticating-with-hashicorp-vault/index.md @@ -41,15 +41,14 @@ The JWT's payload looks like this: "nbf": 1585798372, # Not valid before "exp": 1585713886, # Expire at "sub": "job_1212", # Subject (job id) - "namespace_id": "1", - "namespace_path": "mygroup", - "project_id": "22", - "project_path": "mygroup/myproject", - "user_id": "42", - "user_login": "myuser", - "user_email": "myuser@example.com", - "pipeline_id": "1212", - "job_id": "1212", + "namespace_id": "1", # Use this to scope to group or user level namespace by id + "namespace_path": "mygroup", # Use this to scope to group or user level namespace by path + "project_id": "22", # + "project_path": "mygroup/myproject", # + "user_id": "42", # Id of the user executing the job + "user_email": "myuser@example.com", # Email of the user executing the job + "pipeline_id": "1212", # + "job_id": "1212", # "ref": "auto-deploy-2020-04-01", # Git ref for this job "ref_type": "branch", # Git ref type, branch or tag "ref_protected": "true" # true if this git ref is protected, false otherwise diff --git a/doc/ci/examples/end_to_end_testing_webdriverio/index.md b/doc/ci/examples/end_to_end_testing_webdriverio/index.md index 1f6c81a68aa..aa6e6f73a0d 100644 --- a/doc/ci/examples/end_to_end_testing_webdriverio/index.md +++ b/doc/ci/examples/end_to_end_testing_webdriverio/index.md @@ -9,7 +9,7 @@ type: tutorial [Review Apps](../../review_apps/index.md) are great: for every merge request (or branch, for that matter), the new code can be copied and deployed to a fresh production-like live -environment, making it incredibly low-effort to assess the impact of the changes. Thus, when we use a dependency manager like +environment, reducing the effort to assess the impact of changes. Thus, when we use a dependency manager like [Dependencies.io](https://www.dependencies.io/), it can submit a merge request with an updated dependency, and it will immediately be clear that the application can still be properly built and deployed. After all, you can _see_ it running! @@ -95,9 +95,10 @@ dependency upgrade did not break anything without even having to look at your we ## Running locally -We'll get to running the above test in CI/CD in a moment. When writing tests, however, it helps if -you do not have to wait for your pipelines to succeed in order to check whether they do what you -expect them to do. In other words, let's get it to run locally. +We'll get to running the above test in CI/CD in a moment. When writing tests, +however, it helps if you don't have to wait for your pipelines to succeed to +determine whether they do what you expect them to do. In other words, let's get +it to run locally. Make sure that your app is running locally. If you use Webpack, you can use [the Webpack Dev Server WebdriverIO plugin](https://www.npmjs.com/package/wdio-webpack-dev-server-service) diff --git a/doc/ci/examples/laravel_with_gitlab_and_envoy/index.md b/doc/ci/examples/laravel_with_gitlab_and_envoy/index.md index 8927c5c3480..aaa34afeddf 100644 --- a/doc/ci/examples/laravel_with_gitlab_and_envoy/index.md +++ b/doc/ci/examples/laravel_with_gitlab_and_envoy/index.md @@ -143,7 +143,6 @@ Now, let's clone our repository on the server just to make sure the `deployer` u git clone git@gitlab.example.com:<USERNAME>/laravel-sample.git ``` -NOTE: **Note:** Answer **yes** if asked `Are you sure you want to continue connecting (yes/no)?`. It adds GitLab.com to the known hosts. @@ -167,7 +166,6 @@ server { } ``` -NOTE: **Note:** You may replace the app's name in `/var/www/app/current/public` with the folder name of your application. ## Setting up Envoy @@ -397,8 +395,6 @@ To be able to build, test, and deploy our app with GitLab CI/CD, we need to prep To do that, we'll use a Docker image which has the minimum requirements that a Laravel app needs to run. [There are other ways](../php.md#test-php-projects-using-the-docker-executor) to do that as well, but they may lead our builds run slowly, which is not what we want when there are faster options to use. -With Docker images our builds run incredibly faster! - ### Create a Container Image Let's create a [Dockerfile](https://gitlab.com/mehranrasulian/laravel-sample/blob/master/Dockerfile) in the root directory of our app with the following content: @@ -441,9 +437,11 @@ On your GitLab project repository navigate to the **Registry** tab.  -You may need to [enable Container Registry](../../../user/packages/container_registry/index.md#enable-the-container-registry-for-your-project) to your project to see this tab. You'll find it under your project's **Settings > General > Visibility, project features, permissions**. +You may need to enable the Container Registry for your project to see this tab. You'll find it under your project's **Settings > General > Visibility, project features, permissions**. -To start using Container Registry on our machine, we first need to login to the GitLab registry using our GitLab username and password: +To start using Container Registry on our machine, we first need to sign in to the GitLab registry using our GitLab username and password. +Make sure you have [Docker](https://docs.docker.com/engine/installation/) installed on our machine, +then run the following commands: ```shell docker login registry.gitlab.com @@ -457,14 +455,10 @@ docker build -t registry.gitlab.com/<USERNAME>/laravel-sample . docker push registry.gitlab.com/<USERNAME>/laravel-sample ``` -NOTE: **Note:** -To run the above commands, we first need to have [Docker](https://docs.docker.com/engine/installation/) installed on our machine. - Congratulations! You just pushed the first Docker image to the GitLab Registry, and if you refresh the page you should be able to see it:  -NOTE: **Note:** You can also [use GitLab CI/CD](https://about.gitlab.com/blog/2016/05/23/gitlab-container-registry/#use-with-gitlab-ci) to build and push your Docker images, rather than doing that on your machine. We'll use this image further down in the `.gitlab-ci.yml` configuration file to handle the process of testing and deploying our app. @@ -544,7 +538,6 @@ services: ... ``` -NOTE: **Note:** If you wish to test your app with different PHP versions and [database management systems](../../services/README.md), you can define different `image` and `services` keywords for each test job. #### Variables diff --git a/doc/ci/examples/php.md b/doc/ci/examples/php.md index ab6ff1dc177..699d04f632c 100644 --- a/doc/ci/examples/php.md +++ b/doc/ci/examples/php.md @@ -141,12 +141,11 @@ Of course, `my_php.ini` must be present in the root directory of your repository ## Test PHP projects using the Shell executor -The shell executor runs your job in a terminal session on your server. -Thus, in order to test your projects you first need to make sure that all -dependencies are installed. +The shell executor runs your job in a terminal session on your server. To test +your projects, you must first ensure that all dependencies are installed. -For example, in a VM running Debian 8 we first update the cache, then we -install `phpunit` and `php5-mysql`: +For example, in a VM running Debian 8, first update the cache, and then install +`phpunit` and `php5-mysql`: ```shell sudo apt-get update -y @@ -219,8 +218,8 @@ test:atoum: ### Using Composer The majority of the PHP projects use Composer for managing their PHP packages. -In order to execute Composer before running your tests, simply add the -following in your `.gitlab-ci.yml`: +To execute Composer before running your tests, add the following to your +`.gitlab-ci.yml`: ```yaml # Composer stores all downloaded packages in the vendor/ directory. @@ -243,14 +242,14 @@ before_script: ## Access private packages or dependencies If your test suite needs to access a private repository, you need to configure -[the SSH keys](../ssh_keys/README.md) in order to be able to clone it. +the [SSH keys](../ssh_keys/README.md) to be able to clone it. ## Use databases or other services -Most of the time you will need a running database in order for your tests to -run. If you are using the Docker executor you can leverage Docker's ability to -link to other containers. With GitLab Runner, this can be achieved by -defining a `service`. +Most of the time, you need a running database for your tests to be able to +run. If you're using the Docker executor, you can leverage Docker's ability to +link to other containers. With GitLab Runner, this can be achieved by defining +a `service`. This functionality is covered in [the CI services](../services/README.md) documentation. diff --git a/doc/ci/examples/test-and-deploy-ruby-application-to-heroku.md b/doc/ci/examples/test-and-deploy-ruby-application-to-heroku.md index 066c0cf214f..86c979c489c 100644 --- a/doc/ci/examples/test-and-deploy-ruby-application-to-heroku.md +++ b/doc/ci/examples/test-and-deploy-ruby-application-to-heroku.md @@ -68,7 +68,7 @@ First install [Docker Engine](https://docs.docker.com/installation/). To build this project you also need to have [GitLab Runner](https://docs.gitlab.com/runner/). You can use public runners available on `gitlab.com` or register your own. Start by -creating a template configuration file in order to pass complex configuration: +creating a template configuration file to pass complex configuration: ```shell cat > /tmp/test-config.template.toml << EOF diff --git a/doc/ci/examples/test_phoenix_app_with_gitlab_ci_cd/index.md b/doc/ci/examples/test_phoenix_app_with_gitlab_ci_cd/index.md index ab6a4d3f507..c62f0dec598 100644 --- a/doc/ci/examples/test_phoenix_app_with_gitlab_ci_cd/index.md +++ b/doc/ci/examples/test_phoenix_app_with_gitlab_ci_cd/index.md @@ -117,7 +117,6 @@ Generated hello_gitlab_ci app The database for HelloGitlabCi.Repo has been created ``` -NOTE: **Note:** Phoenix assumes that our PostgreSQL database will have a `postgres` user account with the correct permissions and a password of `postgres`. If it's not your case, check [Ecto's instructions](https://hexdocs.pm/ecto/Ecto.html#module-repositories). @@ -205,7 +204,6 @@ when running our Phoenix in our `localhost`. Without `.gitkeep`, Git will not upload this empty directory and we'll got an error when running our test on GitLab. - NOTE: **Note:** If we add a folder via the GitLab UI, GitLab itself will add the `.gitkeep` to that new dir. Now, let's run a local test and see if everything we did didn't break anything. diff --git a/doc/ci/img/cf_ec2_diagram_v13_5.png b/doc/ci/img/cf_ec2_diagram_v13_5.png Binary files differnew file mode 100644 index 00000000000..1d49790c7b9 --- /dev/null +++ b/doc/ci/img/cf_ec2_diagram_v13_5.png diff --git a/doc/ci/img/ci_lint.png b/doc/ci/img/ci_lint.png Binary files differindex e62de011293..fdc3868cdce 100644 --- a/doc/ci/img/ci_lint.png +++ b/doc/ci/img/ci_lint.png diff --git a/doc/ci/img/ci_lint_dry_run.png b/doc/ci/img/ci_lint_dry_run.png Binary files differindex 4092b66d534..61d6379f70e 100644 --- a/doc/ci/img/ci_lint_dry_run.png +++ b/doc/ci/img/ci_lint_dry_run.png diff --git a/doc/ci/img/gitlab_vault_workflow_v13_4.png b/doc/ci/img/gitlab_vault_workflow_v13_4.png Binary files differnew file mode 100644 index 00000000000..80d07362bf4 --- /dev/null +++ b/doc/ci/img/gitlab_vault_workflow_v13_4.png diff --git a/doc/ci/introduction/index.md b/doc/ci/introduction/index.md index db2749233e8..d1f3e449e5b 100644 --- a/doc/ci/introduction/index.md +++ b/doc/ci/introduction/index.md @@ -12,7 +12,7 @@ In this document, we'll present an overview of the concepts of Continuous Integr Continuous Delivery, and Continuous Deployment, as well as an introduction to GitLab CI/CD. -NOTE: **Note:** +TIP: **Tip:** Out-of-the-box management systems can decrease hours spent on maintaining toolchains by 10% or more. Watch our ["Mastering continuous software development"](https://about.gitlab.com/webcast/mastering-ci-cd/) webcast to learn about continuous methods and how GitLab’s built-in CI can help you simplify and scale software development. @@ -154,7 +154,7 @@ commits to a feature branch in a remote repository in GitLab, the CI/CD pipeline set for your project is triggered. By doing so, GitLab CI/CD: -- Runs automated scripts (sequential or parallel) to: +- Runs automated scripts (sequentially or in parallel) to: - Build and test your app. - Preview the changes per merge request with Review Apps, as you would see in your `localhost`. diff --git a/doc/ci/merge_request_pipelines/index.md b/doc/ci/merge_request_pipelines/index.md index 3ce62936168..cc0b4ac1f86 100644 --- a/doc/ci/merge_request_pipelines/index.md +++ b/doc/ci/merge_request_pipelines/index.md @@ -24,7 +24,6 @@ can run a pipeline for merge requests.  -NOTE: **Note:** If you use this feature with [merge when pipeline succeeds](../../user/project/merge_requests/merge_when_pipeline_succeeds.md), pipelines for merge requests take precedence over the other regular pipelines. @@ -125,8 +124,9 @@ Therefore: - Since `C` specifies that it should only run for merge requests, it will not run for any pipeline except a merge request pipeline. -This helps you avoid having to add the `only:` rule to all of your jobs -in order to make them always run. You can use this format to set up a Review App, helping to save resources. +This helps you avoid having to add the `only:` rule to all of your jobs to make +them always run. You can use this format to set up a Review App, helping to +save resources. #### Excluding certain branches diff --git a/doc/ci/merge_request_pipelines/pipelines_for_merged_results/index.md b/doc/ci/merge_request_pipelines/pipelines_for_merged_results/index.md index dd08f9248f6..2330bdb4c7c 100644 --- a/doc/ci/merge_request_pipelines/pipelines_for_merged_results/index.md +++ b/doc/ci/merge_request_pipelines/pipelines_for_merged_results/index.md @@ -80,8 +80,8 @@ For more information, read the [documentation on Merge Trains](merge_trains/inde > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12996) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.3. -GitLab CI/CD can detect the presence of redundant pipelines, -and will cancel them automatically in order to conserve CI resources. +GitLab CI/CD can detect the presence of redundant pipelines, and cancels them +to conserve CI resources. When a user merges a merge request immediately within an ongoing merge train, the train will be reconstructed, as it will recreate the expected diff --git a/doc/ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md b/doc/ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md index 1f88e8f832f..45cae49377f 100644 --- a/doc/ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md +++ b/doc/ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md @@ -37,7 +37,6 @@ run. To add a merge request to a merge train, you need [permissions](../../../../user/permissions.md) to push to the target branch. -NOTE: **Note:** Each merge train can run a maximum of **twenty** pipelines in parallel. If more than twenty merge requests are added to the merge train, the merge requests will be queued until a slot in the merge train is free. There is no limit to the diff --git a/doc/ci/metrics_reports.md b/doc/ci/metrics_reports.md index 4f4471225a0..53e097760e6 100644 --- a/doc/ci/metrics_reports.md +++ b/doc/ci/metrics_reports.md @@ -9,8 +9,6 @@ type: reference > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9788) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.10. Requires GitLab Runner 11.10 and above. -## Overview - GitLab provides a lot of great reporting tools for [merge requests](../user/project/merge_requests/index.md) - [Unit test reports](unit_test_reports.md), [code quality](../user/project/merge_requests/code_quality.md), performance tests, etc. While JUnit is a great open framework for tests that "pass" or "fail", it is also important to see other types of metrics from a given change. You can configure your job to use custom Metrics Reports, and GitLab will display a report on the merge request so that it's easier and faster to identify changes without having to check the entire log. diff --git a/doc/ci/migration/jenkins.md b/doc/ci/migration/jenkins.md index 8dff99f7244..1130c11f472 100644 --- a/doc/ci/migration/jenkins.md +++ b/doc/ci/migration/jenkins.md @@ -23,7 +23,7 @@ that were able to quickly complete this migration: 1. Use the [Jenkins Wrapper](#jenkinsfile-wrapper) to temporarily maintain fragile Jenkins jobs. 1. Migrate the build and CI jobs and configure them to show results directly in your merge requests. They can use [Auto DevOps](../../topics/autodevops/index.md) as a starting point, and [customize](../../topics/autodevops/customize.md) or [decompose](../../topics/autodevops/customize.md#using-components-of-auto-devops) the configuration as needed. 1. Add [Review Apps](../review_apps/index.md). - 1. Migrate the deployment jobs using [cloud deployment templates](../cloud_deployment/index.md), adding [environments](../environments/index.md), and [deploy boards](../..//user/project/deploy_boards.md). + 1. Migrate the deployment jobs using [cloud deployment templates](../cloud_deployment/index.md), adding [environments](../environments/index.md), and [deploy boards](../../user/project/deploy_boards.md). 1. Work to unwrap any jobs still running with the use of the Jenkins wrapper. 1. Take stock of any common CI/CD job definitions then create and share [templates](#templates) for them. 1. Check the [pipeline efficiency documentation](../pipelines/pipeline_efficiency.md) @@ -83,7 +83,7 @@ There are some high level differences between the products worth mentioning: - You can control which jobs run in which cases, depending on how they are triggered, with the [`rules` syntax](../yaml/README.md#rules). -- GitLab [pipeline scheduling concepts](../pipelines/schedules.md) are also different than with Jenkins. +- GitLab [pipeline scheduling concepts](../pipelines/schedules.md) are also different from Jenkins. - You can reuse pipeline configurations using the [`include` keyword](../yaml/README.md#include) and [templates](#templates). Your templates can be kept in a central repository (with different permissions), and then any project can use them. This central project could also diff --git a/doc/ci/multi_project_pipelines.md b/doc/ci/multi_project_pipelines.md index 3f9a00b6cc8..378adcd35e9 100644 --- a/doc/ci/multi_project_pipelines.md +++ b/doc/ci/multi_project_pipelines.md @@ -163,6 +163,8 @@ have permission to run CI/CD pipelines against the protected branch, the pipelin ### Passing variables to a downstream pipeline +#### With the `variables` keyword + Sometimes you might want to pass variables to a downstream pipeline. You can do that using the `variables` keyword, just like you would when defining a regular job. @@ -211,10 +213,49 @@ In this scenario, the `UPSTREAM_BRANCH` variable with a value related to the upstream pipeline will be passed to the `downstream-job` job, and will be available within the context of all downstream builds. -NOTE: **Tip:** Upstream pipelines take precedence over downstream ones. If there are two variables with the same name defined in both upstream and downstream projects, -the ones defined in the upstream project will take precedence. +the ones defined in the upstream project take precedence. + +#### With variable inheritance + +You can pass variables to a downstream pipeline with [`dotenv` variable inheritance](variables/README.md#inherit-environment-variables) and [cross project artifact downloads](yaml/README.md#cross-project-artifact-downloads-with-needs). + +In the upstream pipeline: + +1. Save the variables in a `.env` file. +1. Save the `.env` file as a `dotenv` report. +1. Trigger the downstream pipeline. + +```yaml +build_vars: + stage: build + script: + - echo "BUILD_VERSION=hello" >> build.env + artifacts: + reports: + dotenv: build.env + +deploy: + stage: deploy + trigger: my/downstream_project +``` + +Set the `test` job in the downstream pipeline to inherit the variables from the `build_vars` +job in the upstream project with `needs:`. The `test` job inherits the variables in the +`dotenv` report and it can access `BUILD_VERSION` in the script: + +```yaml +test: + stage: test + script: + - echo $BUILD_VERSION + needs: + - project: my/upstream_project + job: build_vars + ref: master + artifacts: true +``` ### Mirroring status from triggered pipeline @@ -280,3 +321,11 @@ Any pipelines that complete successfully for new tags in the subscribed project will now trigger a pipeline on the current project's default branch. The maximum number of upstream pipeline subscriptions is 2 by default, for both the upstream and downstream projects. This [application limit](../administration/instance_limits.md#number-of-cicd-subscriptions-to-a-project) can be changed on self-managed instances by a GitLab administrator. + +## Downstream private projects confidentiality concern + +If you trigger a pipeline in a downstream private project, the name of the project +and the status of the pipeline is visible in the upstream project's pipelines page. + +If you have a public project that can trigger downstream pipelines in a private +project, make sure to check that there are no confidentiality problems. diff --git a/doc/ci/parent_child_pipelines.md b/doc/ci/parent_child_pipelines.md index 83fa1d355e6..a0965643970 100644 --- a/doc/ci/parent_child_pipelines.md +++ b/doc/ci/parent_child_pipelines.md @@ -68,11 +68,22 @@ microservice_a: trigger: include: - local: path/to/microservice_a.yml - - template: SAST.gitlab-ci.yml + - template: Security/SAST.gitlab-ci.yml ``` -NOTE: **Note:** -The max number of entries that are accepted for `trigger:include:` is three. +In [GitLab 13.5](https://gitlab.com/gitlab-org/gitlab/-/issues/205157) and later, +you can use [`include:file`](yaml/README.md#includefile) to trigger child pipelines +with a configuration file in a different project: + +```yaml +microservice_a: + trigger: + include: + - project: 'my-group/my-pipeline-library' + file: 'path/to/ci-config.yml' +``` + +The maximum number of entries that are accepted for `trigger:include:` is three. Similar to [multi-project pipelines](multi_project_pipelines.md#mirroring-status-from-triggered-pipeline), we can set the parent pipeline to depend on the status of the child pipeline upon completion: @@ -82,7 +93,7 @@ microservice_a: trigger: include: - local: path/to/microservice_a.yml - - template: SAST.gitlab-ci.yml + - template: Security/SAST.gitlab-ci.yml strategy: depend ``` @@ -153,32 +164,13 @@ This is [resolved in GitLab 12.10](https://gitlab.com/gitlab-org/gitlab/-/issues ## Nested child pipelines > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/29651) in GitLab 13.4. -> - It's [deployed behind a feature flag](../user/feature_flags.md), enabled by default. -> - It's enabled on GitLab.com. -> - It's recommended for production use. -> - For GitLab self-managed instances, GitLab administrators can opt to [disable it](#enable-or-disable-nested-child-pipelines). **(CORE ONLY)** +> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/243747) in GitLab 13.5. Parent and child pipelines were introduced with a maximum depth of one level of child pipelines, which was later increased to two. A parent pipeline can trigger many child pipelines, and these child pipelines can trigger their own child pipelines. It's not possible to trigger another level of child pipelines. -### Enable or disable nested child pipelines **(CORE ONLY)** +## Pass variables to a child pipeline -Nested child pipelines with a depth of two are under development but ready for -production use. This feature is deployed behind a feature flag that is **enabled by default**. - -[GitLab administrators with access to the GitLab Rails console](../administration/feature_flags.md) -can opt to disable it. - -To enable it: - -```ruby -Feature.enable(:ci_child_of_child_pipeline) -``` - -To disable it: - -```ruby -Feature.disable(:ci_child_of_child_pipeline) -``` +You can [pass variables to a downstream pipeline](multi_project_pipelines.md#passing-variables-to-a-downstream-pipeline). diff --git a/doc/ci/pipelines/img/ci_efficiency_pipeline_dag_critical_path.png b/doc/ci/pipelines/img/ci_efficiency_pipeline_dag_critical_path.png Binary files differindex 1715e8224ab..421fddaf38d 100644 --- a/doc/ci/pipelines/img/ci_efficiency_pipeline_dag_critical_path.png +++ b/doc/ci/pipelines/img/ci_efficiency_pipeline_dag_critical_path.png diff --git a/doc/ci/pipelines/img/ci_efficiency_pipeline_health_grafana_dashboard.png b/doc/ci/pipelines/img/ci_efficiency_pipeline_health_grafana_dashboard.png Binary files differindex 0956e76804e..59276bda727 100644 --- a/doc/ci/pipelines/img/ci_efficiency_pipeline_health_grafana_dashboard.png +++ b/doc/ci/pipelines/img/ci_efficiency_pipeline_health_grafana_dashboard.png diff --git a/doc/ci/pipelines/index.md b/doc/ci/pipelines/index.md index 1b9048089bd..f7e3698b6d4 100644 --- a/doc/ci/pipelines/index.md +++ b/doc/ci/pipelines/index.md @@ -10,7 +10,7 @@ type: reference > Introduced in GitLab 8.8. -NOTE: **Tip:** +TIP: **Tip:** Watch the ["Mastering continuous software development"](https://about.gitlab.com/webcast/mastering-ci-cd/) webcast to see a comprehensive demo of a GitLab CI/CD pipeline. @@ -101,8 +101,8 @@ you can filter the pipeline list by: - Trigger author - Branch name -- Status ([since GitLab 13.1](https://gitlab.com/gitlab-org/gitlab/-/issues/217617)) -- Tag ([since GitLab 13.1](https://gitlab.com/gitlab-org/gitlab/-/issues/217617)) +- Status ([GitLab 13.1 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/217617)) +- Tag ([GitLab 13.1 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/217617)) ### Run a pipeline manually @@ -114,7 +114,7 @@ operation of the pipeline. To execute a pipeline manually: 1. Navigate to your project's **CI/CD > Pipelines**. -1. Click on the **Run Pipeline** button. +1. Select the **Run Pipeline** button. 1. On the **Run Pipeline** page: 1. Select the branch to run the pipeline for in the **Create for** field. 1. Enter any [environment variables](../variables/README.md) required for the pipeline run. @@ -461,6 +461,28 @@ this line should be hidden when collapsed section_end:1560896353:my_first_section\r\e[0K ``` +#### Pre-collapse sections + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/198413) in GitLab 13.5. + +You can make the job log automatically collapse collapsible sections by adding the `collapsed` option to the section start. +Add `[collapsed=true]` after the section name and before the `\r`. The section end marker +remains unchanged: + +- Section start marker with `[collapsed=true]`: `section_start:UNIX_TIMESTAMP:SECTION_NAME[collapsed=true]\r\e[0K` + `TEXT_OF_SECTION_HEADER` +- Section end marker: `section_end:UNIX_TIMESTAMP:SECTION_NAME\r\e[0K` + +Add the updated section start text to the CI configuration. For example, +using `echo`: + +```yaml +job1: + script: + - echo -e "section_start:`date +%s`:my_first_section[collapsed=true]\r\e[0KHeader of the 1st collapsible section" + - echo 'this line should be hidden automatically after loading the job log' + - echo -e "section_end:`date +%s`:my_first_section\r\e[0K" +``` + ## Visualize pipelines > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/5742) in GitLab 8.11. @@ -473,7 +495,6 @@ and their statuses. Pipeline graphs can be displayed in two different ways, depending on the page you access the graph from. -NOTE: **Note:** GitLab capitalizes the stages' names in the pipeline graphs. ### Regular pipeline graphs diff --git a/doc/ci/pipelines/job_artifacts.md b/doc/ci/pipelines/job_artifacts.md index 750a76bfaa0..d904452a011 100644 --- a/doc/ci/pipelines/job_artifacts.md +++ b/doc/ci/pipelines/job_artifacts.md @@ -37,8 +37,8 @@ pdf: expire_in: 1 week ``` -A job named `pdf` calls the `xelatex` command in order to build a PDF file from -the latex source file `mycv.tex`. We then define the `artifacts` paths which in +A job named `pdf` calls the `xelatex` command to build a PDF file from the +latex source file `mycv.tex`. We then define the `artifacts` paths which in turn are defined with the `paths` keyword. All paths to files and directories are relative to the repository that was cloned during the build. @@ -61,12 +61,10 @@ The `artifacts:reports` keyword is used for collecting test reports, code qualit reports, and security reports from jobs. It also exposes these reports in GitLab's UI (merge requests, pipeline views, and security dashboards). -NOTE: **Note:** The test reports are collected regardless of the job results (success or failure). You can use [`artifacts:expire_in`](../yaml/README.md#artifactsexpire_in) to set up an expiration date for their artifacts. -NOTE: **Note:** If you also want the ability to browse the report output files, include the [`artifacts:paths`](../yaml/README.md#artifactspaths) keyword. @@ -96,7 +94,6 @@ rspec: The collected Unit test reports upload to GitLab as an artifact and display in merge requests. -NOTE: **Note:** If the JUnit tool you use exports to multiple XML files, specify multiple test report paths within a single job to concatenate them into a single file. Use a filename pattern (`junit: rspec-*.xml`), @@ -343,6 +340,11 @@ The latest artifacts are created by jobs in the **most recent** successful pipel for the specific ref. If you run two types of pipelines for the same ref, timing determines the latest artifact. For example, if a merge request creates a branch pipeline at the same time as a scheduled pipeline, the pipeline that completed most recently creates the latest artifact. +In [GitLab 13.5](https://gitlab.com/gitlab-org/gitlab/-/issues/201784) and later, artifacts +for [parent and child pipelines](../parent_child_pipelines.md) are searched in hierarchical +order from parent to child. For example, if both parent and child pipelines have a +job with the same name, the artifact from the parent pipeline is returned. + Artifacts for other pipelines can be accessed with direct access to them. The structure of the URL to download the whole artifacts archive is the following: @@ -429,7 +431,20 @@ To erase a job: ## Retrieve artifacts of private projects when using GitLab CI -In order to retrieve a job artifact of a different project, you might need to use a private token in order to [authenticate and download](../../api/job_artifacts.md#get-job-artifacts) the artifacts. +To retrieve a job artifact from a different project, you might need to use a +private token to [authenticate and download](../../api/job_artifacts.md#get-job-artifacts) +the artifact. + +## Troubleshooting + +### Error message `No files to upload` + +This is often preceded by other errors or warnings that specify the filename and why it wasn't +generated in the first place. Please check the entire job log for such messages. + +If you find no helpful messages, please retry the failed job after activating +[CI debug logging](../variables/README.md#debug-logging). +This provides useful information to investigate further. <!-- ## Troubleshooting diff --git a/doc/ci/pipelines/pipeline_artifacts.md b/doc/ci/pipelines/pipeline_artifacts.md new file mode 100644 index 00000000000..8d70fff4e03 --- /dev/null +++ b/doc/ci/pipelines/pipeline_artifacts.md @@ -0,0 +1,18 @@ +--- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: reference, howto +--- + +# Pipeline artifacts + +Pipeline artifacts are files created by GitLab after a pipeline finishes. These are different than [job artifacts](job_artifacts.md) because they are not explicitly managed by the `.gitlab-ci.yml` definitions. + +Pipeline artifacts are used by the [test coverage visualization feature](../../user/project/merge_requests/test_coverage_visualization.md) to collect coverage information. It uses the [`artifacts: reports`](../yaml/README.md#artifactsreports) CI/CD keyword. + +## Storage + +Pipeline artifacts are saved to disk or object storage. They count towards a project's [storage usage quota](../../user/group/index.md#storage-usage-quota). The **Artifacts** on the usage quote page is the sum of all job artifacts and pipeline artifacts. + +Pipeline artifacts are erased after one week. diff --git a/doc/ci/pipelines/schedules.md b/doc/ci/pipelines/schedules.md index e488179ffee..bcdb7c4c8b6 100644 --- a/doc/ci/pipelines/schedules.md +++ b/doc/ci/pipelines/schedules.md @@ -11,9 +11,6 @@ type: reference, howto > - Introduced in GitLab 9.1 as [Trigger Schedule](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/10533). > - [Renamed to Pipeline Schedule](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/10853) in GitLab 9.2. -NOTE: **Note:** -Cron notation is parsed by [Fugit](https://github.com/floraison/fugit). - Pipelines are normally run based on certain conditions being met. For example, when a branch is pushed to repository. Pipeline schedules can be used to also run [pipelines](index.md) at specific intervals. For example: @@ -24,6 +21,8 @@ Pipeline schedules can be used to also run [pipelines](index.md) at specific int In addition to using the GitLab UI, pipeline schedules can be maintained using the [Pipeline schedules API](../../api/pipeline_schedules.md). +Schedule timing is configured with cron notation, parsed by [Fugit](https://github.com/floraison/fugit). + ## Prerequisites In order for a scheduled pipeline to be created successfully: diff --git a/doc/ci/pipelines/settings.md b/doc/ci/pipelines/settings.md index 849eb66d07f..143a5346e88 100644 --- a/doc/ci/pipelines/settings.md +++ b/doc/ci/pipelines/settings.md @@ -36,13 +36,11 @@ in `.gitlab-ci.yml`. > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/28919) in GitLab 12.0. -NOTE: **Note:** -As of GitLab 12.0, newly created projects will automatically have a default -`git depth` value of `50`. +It is possible to limit the number of changes that GitLab CI/CD fetches when cloning +a repository. Setting a limit to `git depth` can speed up Pipelines execution. -It is possible to limit the number of changes that GitLab CI/CD will fetch when cloning -a repository. Setting a limit to `git depth` can speed up Pipelines execution. Maximum -allowed value is `1000`. +In GitLab 12.0 and later, newly created projects automatically have a default +`git depth` value of `50`. The maximum allowed value is `1000`. To disable shallow clone and make GitLab CI/CD fetch all branches and tags each time, keep the value empty or set to `0`. @@ -75,7 +73,7 @@ For information about setting a maximum artifact size for a project, see > - [Support for external `.gitlab-ci.yml` locations](https://gitlab.com/gitlab-org/gitlab/-/issues/14376) introduced in GitLab 12.6. By default we look for the `.gitlab-ci.yml` file in the project's root -directory. If needed, you can specify an alternate path and file name, including locations outside the project. +directory. If needed, you can specify an alternate path and filename, including locations outside the project. To customize the path: @@ -93,12 +91,12 @@ paths and file names include: - `my/path/.gitlab-ci.yml` - `my/path/.my-custom-file.yml` -If the CI configuration will be hosted on an external site, the URL link must end with `.yml`: +If hosting the CI configuration on an external site, the URL link must end with `.yml`: - `http://example.com/generate/ci/config.yml` -If the CI configuration will be hosted in a different project within GitLab, the path must be relative -to the root directory in the other project, with the group and project name added to the end: +If hosting the CI configuration in a different project within GitLab, the path must be relative +to the root directory in the other project. Include the group and project name at the end: - `.gitlab-ci.yml@mygroup/another-project` - `my/path/.my-custom-file.yml@mygroup/another-project` @@ -109,7 +107,7 @@ configuration file. For example: - Create a public project to host the configuration file. - Give write permissions on the project only to users who are allowed to edit the file. -Other users and projects will be able to access the configuration file without being +Other users and projects can access the configuration file without being able to edit it. ## Test coverage parsing @@ -125,8 +123,8 @@ can use <https://rubular.com> to test your regex. The regex returns the **last** match found in the output. If the pipeline succeeds, the coverage is shown in the merge request widget and -in the jobs table. If multiple jobs in the pipeline have coverage reports, they will -be averaged. +in the jobs table. If multiple jobs in the pipeline have coverage reports, they are +averaged.  @@ -140,7 +138,7 @@ in the pipelines settings page. > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/209121) the ability to download a `.csv` in GitLab 12.10. > - [Graph introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/33743) in GitLab 13.1. -If you want to see the evolution of your project code coverage over time, +To see the evolution of your project code coverage over time, you can view a graph or download a CSV file with this data. From your project: 1. Go to **{chart}** **Project Analytics > Repository** to see the historic data for each job listed in the dropdown above the graph. @@ -150,13 +148,13 @@ you can view a graph or download a CSV file with this data. From your project: ### Removing color codes -Some test coverage tools output with ANSI color codes that won't be -parsed correctly by the regular expression and will cause coverage +Some test coverage tools output with ANSI color codes that aren't +parsed correctly by the regular expression. This causes coverage parsing to fail. -If your coverage tool doesn't provide an option to disable color -codes in the output, you can pipe the output of the coverage tool through a -small one line script that will strip the color codes off. +Some coverage tools don't provide an option to disable color +codes in the output. If so, pipe the output of the coverage tool through a +small one line script that strips the color codes off. For example: @@ -172,7 +170,7 @@ Pipeline visibility is determined by: - The **Public pipelines** project setting under your project's **Settings > CI/CD > General pipelines**. NOTE: **Note:** -If the project visibility is set to **Private**, the [**Public pipelines** setting will have no effect](../enable_or_disable_ci.md#per-project-user-setting). +If the project visibility is set to **Private**, the [**Public pipelines** setting has no effect](../enable_or_disable_ci.md#per-project-user-setting). This also determines the visibility of these related features: @@ -180,8 +178,7 @@ This also determines the visibility of these related features: - Job artifacts - The [pipeline security dashboard](../../user/application_security/security_dashboard/index.md#pipeline-security) **(ULTIMATE)** -NOTE: **Note:** -Currently, job logs and artifacts are [not yet visible for guest users and non-project members](https://gitlab.com/gitlab-org/gitlab/-/issues/25649). +Job logs and artifacts are [not visible for guest users and non-project members](https://gitlab.com/gitlab-org/gitlab/-/issues/25649). If **Public pipelines** is enabled (default): @@ -204,16 +201,14 @@ If **Public pipelines** is disabled: > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/9362) in GitLab 9.1. -If you want all pending non-HEAD pipelines on branches to auto-cancel each time -a new pipeline is created, such as after a Git push or manually from the UI, -you can enable this in the project settings: +You can set pending or running pipelines to cancel automatically when a new pipeline runs on the same branch. You can enable this in the project settings: 1. Go to **Settings > CI / CD**. 1. Expand **General Pipelines**. 1. Check the **Auto-cancel redundant, pending pipelines** checkbox. 1. Click **Save changes**. -Note that only jobs with [interruptible](../yaml/README.md#interruptible) set to `true` will be cancelled. +Note that only jobs with [interruptible](../yaml/README.md#interruptible) set to `true` are cancelled. ## Skip outdated deployment jobs @@ -232,18 +227,18 @@ To avoid this scenario: 1. Check the **Skip outdated deployment jobs** checkbox. 1. Click **Save changes**. -The pending deployment jobs will be skipped. +When enabled, any older deployments job are skipped when a new deployment starts. For more information, see [Deployment safety](../environments/deployment_safety.md). ## Pipeline Badges In the pipelines settings page you can find pipeline status and test coverage -badges for your project. The latest successful pipeline will be used to read +badges for your project. The latest successful pipeline is used to read the pipeline status and test coverage values. Visit the pipelines settings page in your project to see the exact link to -your badges, as well as ways to embed the badge image in your HTML or Markdown +your badges. You can also see ways to embed the badge image in your HTML or Markdown pages.  @@ -276,8 +271,8 @@ https://gitlab.example.com/<namespace>/<project>/badges/<branch>/pipeline.svg?ig ### Test coverage report badge -GitLab makes it possible to define the regular expression for [coverage report](#test-coverage-parsing), -that each job log will be matched against. This means that each job in the +GitLab makes it possible to define the regular expression for the [coverage report](#test-coverage-parsing), +that each job log is matched against. This means that each job in the pipeline can have the test coverage percentage value defined. The test coverage badge can be accessed using following link: @@ -288,7 +283,7 @@ https://gitlab.example.com/<namespace>/<project>/badges/<branch>/coverage.svg If you would like to get the coverage report from a specific job, you can add the `job=coverage_job_name` parameter to the URL. For example, the following -Markdown code will embed the test coverage report badge of the `coverage` job +Markdown code embeds the test coverage report badge of the `coverage` job into your `README.md`: ```markdown @@ -297,7 +292,7 @@ into your `README.md`: ### Badge styles -Pipeline badges can be rendered in different styles by adding the `style=style_name` parameter to the URL. Currently two styles are available: +Pipeline badges can be rendered in different styles by adding the `style=style_name` parameter to the URL. Two styles are available: #### Flat (default) @@ -324,10 +319,10 @@ https://gitlab.example.com/<namespace>/<project>/badges/<branch>/coverage.svg?st The text for a badge can be customized. This can be useful to differentiate between multiple coverage jobs that run in the same pipeline. Customize the badge text and width by adding the `key_text=custom_text` and `key_width=custom_key_width` parameters to the URL: ```plaintext -https://gitlab.com/gitlab-org/gitlab/badges/master/coverage.svg?job=karma&key_text=Frontend+Coverage&key_width=100 +https://gitlab.com/gitlab-org/gitlab/badges/master/coverage.svg?job=karma&key_text=Frontend+Coverage&key_width=130 ``` - + ## Environment Variables diff --git a/doc/ci/quick_start/README.md b/doc/ci/quick_start/README.md index fa16614b0e0..246430a6458 100644 --- a/doc/ci/quick_start/README.md +++ b/doc/ci/quick_start/README.md @@ -68,11 +68,6 @@ blog about it](https://about.gitlab.com/blog/2015/05/06/why-were-replacing-gitla ### Creating a simple `.gitlab-ci.yml` file -NOTE: **Note:** -A GitLab team member has made an [unofficial visual pipeline editor](https://unofficial.gitlab.tools/visual-pipelines/). -There is a [plan to make it an official part of GitLab](https://gitlab.com/groups/gitlab-org/-/epics/4069) -in the future, but it's available for anyone who wants to try it at the above link. - You need to create a file named `.gitlab-ci.yml` in the root directory of your repository. This is a [YAML](https://en.wikipedia.org/wiki/YAML) file so you have to pay extra attention to indentation. Always use spaces, not tabs. @@ -117,9 +112,17 @@ What is important is that each job is run independently from each other. If you want to check whether the `.gitlab-ci.yml` of your project is valid, there is a [CI Lint tool](../lint.md) available in every project. +You can use the [CI/CD configuration visualization](../yaml/visualization.md) to +see a graphical representation of your `.gitlab-ci.yml`. + For more information and a complete `.gitlab-ci.yml` syntax, please read [the reference documentation on `.gitlab-ci.yml`](../yaml/README.md). +TIP: **Tip:** +A GitLab team member has made an [unofficial visual pipeline editor](https://unofficial.gitlab.tools/visual-pipelines/). +There is a [plan to make it an official part of GitLab](https://gitlab.com/groups/gitlab-org/-/epics/4069) +in the future, but it's available for anyone who wants to try it at the above link. + ### Push `.gitlab-ci.yml` to GitLab Once you've created `.gitlab-ci.yml`, you should add it to your Git repository diff --git a/doc/ci/review_apps/img/toolbar_feeback_form.png b/doc/ci/review_apps/img/toolbar_feeback_form.png Binary files differdeleted file mode 100644 index fe1c7e6e611..00000000000 --- a/doc/ci/review_apps/img/toolbar_feeback_form.png +++ /dev/null diff --git a/doc/ci/review_apps/img/toolbar_feedback_form_v13_5.png b/doc/ci/review_apps/img/toolbar_feedback_form_v13_5.png Binary files differnew file mode 100644 index 00000000000..dc4a13b2152 --- /dev/null +++ b/doc/ci/review_apps/img/toolbar_feedback_form_v13_5.png diff --git a/doc/ci/review_apps/index.md b/doc/ci/review_apps/index.md index e2d5cbcbea4..7110117709f 100644 --- a/doc/ci/review_apps/index.md +++ b/doc/ci/review_apps/index.md @@ -33,7 +33,7 @@ In the above example: ## How Review Apps work A Review App is a mapping of a branch with an [environment](../environments/index.md). -Access to the Review App is made available as a link on the [merge request](../../user/project/merge_requests.md) relevant to the branch. +Access to the Review App is made available as a link on the [merge request](../../user/project/merge_requests/index.md) relevant to the branch. The following is an example of a merge request with an environment set dynamically. @@ -188,20 +188,35 @@ Once you have the route mapping set up, it will take effect in the following loc ## Visual Reviews **(STARTER)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/10761) in GitLab Starter 12.0. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/10761) in GitLab Starter 12.0. +> - It's [deployed behind a feature flag](../../user/feature_flags.md), enabled by default. +> - It's enabled on GitLab.com. +> - For GitLab self-managed instances, GitLab administrators can opt to [disable it](#enable-or-disable-visual-reviews). **(STARTER ONLY)** -With Visual Reviews, you can provide a feedback form to your Review Apps so -that reviewers can post comments directly from the app back to the merge request -that spawned the Review App. +With Visual Reviews, members of any team (Product, Design, Quality, and so on) can provide feedback comments through a form in your review apps. The comments are added to the merge request that triggered the review app. -### Configuring Visual Reviews +### Using Visual Reviews -Ensure that the `anonymous_visual_review_feedback` feature flag is enabled. -Administrators can enable with a Rails console as follows: +After Visual Reviews has been [configured](#configure-review-apps-for-visual-reviews) for the +Review App, the Visual Reviews feedback form is overlaid on the right side of every page. -```ruby -Feature.enable(:anonymous_visual_review_feedback) -``` + + +To use the feedback form to make a comment in the merge request: + +1. Click the **Review** tab on the right side of a page. +1. Make a comment on the visual review. You can make use of all the + [Markdown annotations](../../user/markdown.md) that are also available in + merge request comments. +1. Enter your personal information: + - If [`data-require-auth`](#authentication-for-visual-reviews) is `true`, you must enter your [personal access token](../../user/profile/personal_access_tokens.md). + - Otherwise, enter your name, and optionally your email. +1. Click **Send feedback**. + +<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> +To see Visual reviews in action, see the [Visual Reviews Walk through](https://youtu.be/1_tvWTlPfM4). + +### Configure Review Apps for Visual Reviews The feedback form is served through a script you add to pages in your Review App. If you have [Developer permissions](../../user/permissions.md) to the project, @@ -212,18 +227,18 @@ if [route maps](#route-maps) are configured in the project.  The provided script should be added to the `<head>` of your application and -consists of some project and merge request specific values. Here's what it -looks like: +consists of some project and merge request specific values. Here's how it +looks for a project with code hosted in a project on GitLab.com: ```html <script data-project-id='11790219' data-merge-request-id='1' - data-mr-url='https://gitlab.example.com' + data-mr-url='https://gitlab.com' data-project-path='sarah/review-app-tester' data-require-auth='true' id='review-app-toolbar-script' - src='https://gitlab.example.com/assets/webpack/visual_review_toolbar.js'> + src='https://gitlab.com/assets/webpack/visual_review_toolbar.js'> </script> ``` @@ -239,21 +254,21 @@ to replace those values at runtime when each review app is created: - `data-mr-url` is the URL of the GitLab instance and will be the same for all review apps. - `data-project-path` is the project's path, which can be found by `CI_PROJECT_PATH`. -- `data-require-auth` is optional for public projects but required for [private and internal ones](#visual-reviews-in-private-or-internal-projects). If this is set to `true`, the user will be required to enter their [personal access token](../../user/profile/personal_access_tokens.md) instead of their name and email. +- `data-require-auth` is optional for public projects but required for [private and internal ones](#authentication-for-visual-reviews). If this is set to `true`, the user will be required to enter their [personal access token](../../user/profile/personal_access_tokens.md) instead of their name and email. - `id` is always `review-app-toolbar-script`, you don't need to change that. - `src` is the source of the review toolbar script, which resides in the respective GitLab instance and will be the same for all review apps. -For example, in a Ruby application, you would need to have this script: +For example, in a Ruby application with code hosted on in a project GitLab.com, you would need to have this script: ```html <script data-project-id="ENV['CI_PROJECT_ID']" data-merge-request-id="ENV['CI_MERGE_REQUEST_IID']" - data-mr-url='https://gitlab.example.com' + data-mr-url='https://gitlab.com' data-project-path="ENV['CI_PROJECT_PATH']" id='review-app-toolbar-script' - src='https://gitlab.example.com/assets/webpack/visual_review_toolbar.js'> + src='https://gitlab.com/assets/webpack/visual_review_toolbar.js'> </script> ``` @@ -273,33 +288,34 @@ can supply the ID by either: - Dynamically adding the `data-merge-request-id` value during the build of the app. - Supplying it manually through the visual review form in the app. -### Visual Reviews in private or internal projects +### Enable or disable Visual Reviews **(STARTER ONLY)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/42750#note_317271120) in GitLab 12.10. +Visual Reviews is deployed behind a feature flag that is **enabled by default**. +[GitLab administrators with access to the GitLab Rails console](../../administration/feature_flags.md) +can opt to disable it. -To enable visual reviews for private and internal projects, set the -[`data-require-auth` variable](#configuring-visual-reviews) to `true`. When enabled, -the user must enter a [personal access token](../../user/profile/personal_access_tokens.md) -with `api` scope before submitting feedback. +To disable it: -### Using Visual Reviews +```ruby +Feature.disable(:anonymous_visual_review_feedback) +``` -After Visual Reviews has been [enabled](#configuring-visual-reviews) for the -Review App, the Visual Reviews feedback form is overlaid on the app's pages at -the bottom-right corner. +To enable it: - +```ruby +Feature.enable(:anonymous_visual_review_feedback) +``` -To use the feedback form: +### Authentication for Visual Reviews -1. Make a comment on the visual review. You can make use of all the - [Markdown annotations](../../user/markdown.md) that are also available in - merge request comments. -1. If `data-require-auth` is `true`, you must enter your [personal access token](../../user/profile/personal_access_tokens.md). Otherwise, you must enter your name, and optionally, your email. -1. Finally, click **Send feedback**. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/42750#note_317271120) in GitLab 12.10. + +To enable visual reviews for private and internal projects, set the +[`data-require-auth` variable](#enable-or-disable-visual-reviews) to `true`. When enabled, +the user must enter a [personal access token](../../user/profile/personal_access_tokens.md) +with `api` scope before submitting feedback. -After you make and submit a comment in the visual review box, it will appear -automatically in the respective merge request. +This same method can be used to require authentication for any public projects. ## Limitations diff --git a/doc/ci/runners/README.md b/doc/ci/runners/README.md index 32561e6b98c..a3cc46f59bf 100644 --- a/doc/ci/runners/README.md +++ b/doc/ci/runners/README.md @@ -18,16 +18,12 @@ Runners can be specific to certain projects or available to all projects. ## Types of runners -There are three types of runners: +In the GitLab UI there are three types of runners, based on who you want to have access: -- [Shared](#shared-runners) (for all projects) -- [Group](#group-runners) (for all projects in a group) -- [Specific](#specific-runners) (for specific projects) - -If you are running self-managed GitLab, you can create your own runners. - -If you are using GitLab.com, you can use the shared runners provided by GitLab or -create your own group or specific runners. +- [Shared runners](#shared-runners) are available to all groups and projects in a GitLab instance. +- [Group runners](#group-runners) are available to all projects and subgroups in a group. +- [Specific runners](#specific-runners) are associated with specific projects. + Typically, specific runners are used for one project at a time. ### Shared runners @@ -39,11 +35,10 @@ multiple projects. If you are using a self-managed instance of GitLab: -- Your administrator can install and register shared runners by viewing the instructions - [here](https://docs.gitlab.com/runner/install/index.html). +- Your administrator can install and register shared runners by [following the documentation](https://docs.gitlab.com/runner/install/index.html). <!-- going to your project's <!-- **Settings > CI / CD**, expanding the **Runners** section, and clicking **Show runner installation instructions**.--> - <!-- These instructions are also available [here](https://docs.gitlab.com/runner/install/index.html).--> + <!-- These instructions are also available [in the documentation](https://docs.gitlab.com/runner/install/index.html).--> - The administrator can also configure a maximum number of shared runner [pipeline minutes for each group](../../user/admin_area/settings/continuous_integration.md#shared-runners-pipeline-minutes-quota). @@ -123,21 +118,20 @@ To enable shared runners: #### Disable shared runners -You can disable shared runners for individual projects<!-- or for groups-->. -You must have Owner permissions for the project<!-- or group-->. +You can disable shared runners for individual projects or for groups. +You must have Owner permissions for the project or group. To disable shared runners for a project: 1. Go to the project's **Settings > CI/CD** and expand the **Runners** section. 1. In the **Shared runners** area, click **Disable shared runners**. -<!--To disable shared runners for a group: +To disable shared runners for a group: 1. Go to the group's **Settings > CI/CD** and expand the **Runners** section. -1. In the **Shared runners** area, click **Disable shared runners globally**. +1. In the **Shared runners** area, click **Enable shared runners for this group**. 1. Optionally, to allow shared runners to be enabled for individual projects or subgroups, - click **Allow projects/subgroups to override the global setting**. ---> + click **Allow projects and subgroups to override the group setting**. ### Group runners diff --git a/doc/ci/secrets/index.md b/doc/ci/secrets/index.md index 6d561fe00a3..fb1183cd68b 100644 --- a/doc/ci/secrets/index.md +++ b/doc/ci/secrets/index.md @@ -17,23 +17,36 @@ Unlike CI variables, which are always presented to a job, secrets must be explic required by a job. Read [GitLab CI/CD pipeline configuration reference](../yaml/README.md#secrets) for more information about the syntax. -GitLab has selected [Vault by Hashicorp](https://www.vaultproject.io) as the +GitLab has selected [Vault by HashiCorp](https://www.vaultproject.io) as the first supported provider, and [KV-V2](https://www.vaultproject.io/docs/secrets/kv/kv-v2) as the first supported secrets engine. GitLab authenticates using Vault's -[JWT Auth method](https://www.vaultproject.io/docs/auth/jwt#jwt-authentication), using +[JSON Web Token (JWT) authentication method](https://www.vaultproject.io/docs/auth/jwt#jwt-authentication), using the [JSON Web Token](https://gitlab.com/gitlab-org/gitlab/-/issues/207125) (`CI_JOB_JWT`) introduced in GitLab 12.10. You must [configure your Vault server](#configure-your-vault-server) before you can use [use Vault secrets in a CI job](#use-vault-secrets-in-a-ci-job). +The flow for using GitLab with HashiCorp Vault +is summarized by this diagram: + + + +1. Configure your vault and secrets. +1. Generate your JWT and provide it to your CI job. +1. Runner contacts HashiCorp Vault and authenticates using the JWT. +1. HashiCorp Vault verifies the JWT. +1. HashiCorp Vault checks the bounded claims and attaches policies. +1. HashiCorp Vault returns the token. +1. Runner reads secrets from the HashiCorp Vault. + NOTE: **Note:** -Read the [Authenticating and Reading Secrets With Hashicorp Vault](../examples/authenticating-with-hashicorp-vault/index.md) -tutorial for a version of this feature that is available to all +Read the [Authenticating and Reading Secrets With HashiCorp Vault](../examples/authenticating-with-hashicorp-vault/index.md) +tutorial for a version of this feature. It's available to all subscription levels, supports writing secrets to and deleting secrets from Vault, -and multiple secrets engines. +and supports multiple secrets engines. ## Configure your Vault server @@ -90,7 +103,7 @@ the secrets stored in Vault by defining them with the `vault` keyword: ```yaml secrets: DATABASE_PASSWORD: - vault: production/db/password@ops # translates to secret `ops/data/production/db`, field `password` + vault: production/db/password@ops # translates to secret `ops/data/production/db`, field `password` ``` In this example: @@ -149,7 +162,7 @@ generated by this GitLab instance may be allowed to authenticate using this role For a full list of `CI_JOB_JWT` claims, read the [How it works](../examples/authenticating-with-hashicorp-vault/index.md#how-it-works) section of the -[Authenticating and Reading Secrets With Hashicorp Vault](../examples/authenticating-with-hashicorp-vault/index.md) tutorial. +[Authenticating and Reading Secrets With HashiCorp Vault](../examples/authenticating-with-hashicorp-vault/index.md) tutorial. You can also specify some attributes for the resulting Vault tokens, such as time-to-live, IP address range, and number of uses. The full list of options is available in diff --git a/doc/ci/services/postgres.md b/doc/ci/services/postgres.md index aadbce5a50a..96552ab1245 100644 --- a/doc/ci/services/postgres.md +++ b/doc/ci/services/postgres.md @@ -13,7 +13,7 @@ do this with the Docker and Shell executors of GitLab Runner. ## Use PostgreSQL with the Docker executor -If you are using [GitLab Runner](../runners/README.md) with the Docker executor +If you're using [GitLab Runner](../runners/README.md) with the Docker executor, you basically have everything set up already. First, in your `.gitlab-ci.yml` add: @@ -29,12 +29,11 @@ variables: POSTGRES_HOST_AUTH_METHOD: trust ``` -NOTE: **Note:** -The `POSTGRES_DB`, `POSTGRES_USER`, `POSTGRES_PASSWORD` and `POSTGRES_HOST_AUTH_METHOD` -variables can't be set in the GitLab UI. To set them, assign them to a variable -[in the UI](../variables/README.md#create-a-custom-variable-in-the-ui), and then assign that -variable to the `POSTGRES_DB`, `POSTGRES_USER`, `POSTGRES_PASSWORD` and `POSTGRES_HOST_AUTH_METHOD` -variables in your `.gitlab-ci.yml`. +To set values for the `POSTGRES_DB`, `POSTGRES_USER`, +`POSTGRES_PASSWORD` and `POSTGRES_HOST_AUTH_METHOD`, +[assign them to a variable in the user interface](../variables/README.md#create-a-custom-variable-in-the-ui), +then assign that variable to the corresponding variable in your +`.gitlab-ci.yml` file. And then configure your application to use the database, for example: @@ -45,14 +44,14 @@ Password: '' Database: nice_marmot ``` -If you are wondering why we used `postgres` for the `Host`, read more at +If you're wondering why we used `postgres` for the `Host`, read more at [How services are linked to the job](../docker/using_docker_images.md#how-services-are-linked-to-the-job). You can also use any other Docker image available on [Docker Hub](https://hub.docker.com/_/postgres). -For example, to use PostgreSQL 9.3 the service becomes `postgres:9.3`. +For example, to use PostgreSQL 9.3, the service becomes `postgres:9.3`. -The `postgres` image can accept some environment variables. For more details -check the documentation on [Docker Hub](https://hub.docker.com/_/postgres). +The `postgres` image can accept some environment variables. For more details, +see the documentation on [Docker Hub](https://hub.docker.com/_/postgres). ## Use PostgreSQL with the Shell executor @@ -65,7 +64,7 @@ First install the PostgreSQL server: sudo apt-get install -y postgresql postgresql-client libpq-dev ``` -The next step is to create a user, so login to PostgreSQL: +The next step is to create a user, so sign in to PostgreSQL: ```shell sudo -u postgres psql -d template1 @@ -74,24 +73,26 @@ sudo -u postgres psql -d template1 Then create a user (in our case `runner`) which will be used by your application. Change `$password` in the command below to a real strong password. -*__Note:__ Do not type `template1=#`, this is part of the PostgreSQL prompt.* +NOTE: **Note:** +Be sure to not enter `template1=#` in the following commands, as that's part of +the PostgreSQL prompt. ```shell template1=# CREATE USER runner WITH PASSWORD '$password' CREATEDB; ``` -*__Note:__ Notice that we created the user with the privilege to be able to -create databases (`CREATEDB`). In the following steps we will create a database -explicitly for that user but having that privilege can be useful if in your -testing framework you have tools that drop and create databases.* +The created user has the privilege to create databases (`CREATEDB`). The +following steps describe how to create a database explicitly for that user, but +having that privilege can be useful if in your testing framework you have tools +that drop and create databases. -Create the database and grant all privileges on it for the user `runner`: +Create the database and grant all privileges to it for the user `runner`: ```shell template1=# CREATE DATABASE nice_marmot OWNER runner; ``` -If all went well you can now quit the database session: +If all went well, you can now quit the database session: ```shell template1=# \q @@ -104,8 +105,8 @@ check that everything is in place. psql -U runner -h localhost -d nice_marmot -W ``` -*__Note:__ We are explicitly telling `psql` to connect to localhost in order -to use the md5 authentication. If you omit this step you will be denied access.* +This command explicitly directs `psql` to connect to localhost to use the md5 +authentication. If you omit this step, you'll be denied access. Finally, configure your application to use the database, for example: @@ -122,5 +123,5 @@ We have set up an [Example PostgreSQL Project](https://gitlab.com/gitlab-example convenience that runs on [GitLab.com](https://gitlab.com) using our publicly available [shared runners](../runners/README.md). -Want to hack on it? Simply fork it, commit and push your changes. Within a few +Want to hack on it? Fork it, commit, and push your changes. Within a few moments the changes will be picked by a public runner and the job will begin. diff --git a/doc/ci/ssh_keys/README.md b/doc/ci/ssh_keys/README.md index d8280316f19..12478000a0a 100644 --- a/doc/ci/ssh_keys/README.md +++ b/doc/ci/ssh_keys/README.md @@ -36,7 +36,6 @@ with any type of [executor](https://docs.gitlab.com/runner/executors/) `~/.ssh/authorized_keys`) or add it as a [deploy key](../../ssh/README.md#deploy-keys) if you are accessing a private GitLab repository. -NOTE: **Note:** The private key will not be displayed in the job log, unless you enable [debug logging](../variables/README.md#debug-logging). You might also want to check the [visibility of your pipelines](../pipelines/settings.md#visibility-of-pipelines). @@ -94,7 +93,6 @@ to access it. This is where an SSH key pair comes in handy. # - git config --global user.name "User name" ``` - NOTE: **Note:** The [`before_script`](../yaml/README.md#before_script-and-after_script) can be set globally or per-job. @@ -134,7 +132,8 @@ on, and use that key for all projects that are run on this machine. If you are accessing a private GitLab repository you need to add it as a [deploy key](../../ssh/README.md#deploy-keys). -Once done, try to log in to the remote server in order to accept the fingerprint: +After generating the key, try to sign in to the remote server to accept the +fingerprint: ```shell ssh example.com @@ -163,7 +162,6 @@ ssh-keyscan 1.2.3.4 Create a new [variable](../variables/README.md#gitlab-cicd-environment-variables) with `SSH_KNOWN_HOSTS` as "Key", and as a "Value" add the output of `ssh-keyscan`. -NOTE: **Note:** If you need to connect to multiple servers, all the server host keys need to be collected in the **Value** of the variable, one key per line. diff --git a/doc/ci/triggers/README.md b/doc/ci/triggers/README.md index 2b006b8779b..bcd19f0de6f 100644 --- a/doc/ci/triggers/README.md +++ b/doc/ci/triggers/README.md @@ -95,9 +95,9 @@ Read more about the [jobs API](../../api/job_artifacts.md#download-the-artifacts ## Adding a new trigger -You can add a new trigger by going to your project's -**Settings ➔ CI/CD** under **Triggers**. The **Add trigger** button will -create a new token which you can then use to trigger a rerun of this +Go to your +**Settings ➔ CI/CD** under **Triggers** to add a new trigger. The **Add trigger** button creates +a new token which you can then use to trigger a rerun of this particular project's pipeline. Every new trigger you create, gets assigned a different token which you can @@ -121,7 +121,7 @@ POST /projects/:id/trigger/pipeline ``` The required parameters are the [trigger's `token`](#authentication-tokens) -and the Git `ref` on which the trigger will be performed. Valid refs are +and the Git `ref` on which the trigger is performed. Valid refs are branches or tags. The `:id` of a project can be found by [querying the API](../../api/projects.md) or by visiting the **CI/CD** settings page which provides self-explanatory examples. @@ -146,7 +146,7 @@ curl --request POST \ https://gitlab.example.com/api/v4/projects/9/trigger/pipeline ``` -In this case, the project with ID `9` will get rebuilt on `master` branch. +In this case, the project with ID `9` gets rebuilt on `master` branch. Alternatively, you can pass the `token` and `ref` arguments in the query string: @@ -169,9 +169,9 @@ build_docs: - tags ``` -This means that whenever a new tag is pushed on project A, the job will run and the -`build_docs` job will be executed, triggering a rebuild of project B. The -`stage: deploy` ensures that this job will run only after all jobs with +This means that whenever a new tag is pushed on project A, the job runs and the +`build_docs` job is executed, triggering a rebuild of project B. The +`stage: deploy` ensures that this job runs only after all jobs with `stage: test` complete successfully. ## Triggering a pipeline from a webhook @@ -183,14 +183,14 @@ webhook URL for Push and Tag events (change the project ID, ref and token): https://gitlab.example.com/api/v4/projects/9/ref/master/trigger/pipeline?token=TOKEN ``` -`ref` should be passed as part of the URL in order to take precedence over -`ref` from the webhook body that designates the branch ref that fired the -trigger in the source repository. `ref` should be URL-encoded if it contains slashes. +You should pass `ref` as part of the URL, to take precedence over `ref` from +the webhook body that designates the branch ref that fired the trigger in the +source repository. Be sure to URL-encode `ref` if it contains slashes. ## Making use of trigger variables You can pass any number of arbitrary variables in the trigger API call and they -will be available in GitLab CI/CD so that they can be used in your `.gitlab-ci.yml` +are available in GitLab CI/CD so that they can be used in your `.gitlab-ci.yml` file. The parameter is of the form: ```plaintext @@ -237,7 +237,7 @@ upload_package: ``` You can then trigger a rebuild while you pass the `UPLOAD_TO_S3` variable -and the script of the `upload_package` job will run: +and the script of the `upload_package` job is run: ```shell curl --request POST \ @@ -252,10 +252,6 @@ of all types of variables. ## Using cron to trigger nightly pipelines -NOTE: **Note:** -The following behavior can also be achieved through GitLab's UI with -[pipeline schedules](../pipelines/schedules.md). - Whether you craft a script or just run cURL directly, you can trigger jobs in conjunction with cron. The example below triggers a job on the `master` branch of project with ID `9` every night at `00:30`: @@ -264,9 +260,12 @@ branch of project with ID `9` every night at `00:30`: 30 0 * * * curl --request POST --form token=TOKEN --form ref=master https://gitlab.example.com/api/v4/projects/9/trigger/pipeline ``` +This behavior can also be achieved through GitLab's UI with +[pipeline schedules](../pipelines/schedules.md). + ## Legacy triggers -Old triggers, created before GitLab 9.0 will be marked as legacy. +Old triggers, created before GitLab 9.0 are marked as legacy. Triggers with the legacy label do not have an associated user and only have access to the current project. They are considered deprecated and will be diff --git a/doc/ci/unit_test_reports.md b/doc/ci/unit_test_reports.md index 5a59a175a89..b9c1809bf0d 100644 --- a/doc/ci/unit_test_reports.md +++ b/doc/ci/unit_test_reports.md @@ -10,8 +10,6 @@ type: reference > - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/45318) in GitLab 11.2. Requires GitLab Runner 11.2 and above. > - [Renamed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/39737) from JUnit test reports to Unit test reports in GitLab 13.4. -## Overview - It is very common that a [CI/CD pipeline](pipelines/index.md) contains a test job that will verify your code. If the tests fail, the pipeline fails and users get notified. The person that @@ -20,12 +18,12 @@ tests failed so that they can fix them. You can configure your job to use Unit test reports, and GitLab will display a report on the merge request so that it's easier and faster to identify the -failure without having to check the entire log. Unit test reports currently +failure without having to check the entire log. Unit test reports currently only support test reports in the JUnit report format. -If you don't use Merge Requests but still want to see the unit test report -output without searching through job logs, the full -[Unit test reports](#viewing-unit-test-reports-on-gitlab) are available +If you don't use Merge Requests but still want to see the unit test report +output without searching through job logs, the full +[Unit test reports](#viewing-unit-test-reports-on-gitlab) are available in the pipeline detail view. Consider the following workflow: @@ -84,7 +82,6 @@ To make the Unit test report output files browsable, include them with the To upload the report even if the job fails (for example if the tests do not pass), use the [`artifacts:when:always`](yaml/README.md#artifactswhen) keyword. -NOTE: **Note:** You cannot have multiple tests with the same name and class in your JUnit report format XML file. ### Ruby example @@ -146,8 +143,8 @@ java: junit: build/test-results/test/**/TEST-*.xml ``` -NOTE: **Note:** -Support for `**` was added in [GitLab Runner 13.0](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/2620). +In [GitLab Runner 13.0](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/2620) +and later, you can use `**`. #### Maven diff --git a/doc/ci/variables/README.md b/doc/ci/variables/README.md index 1a982fa4e19..9c8fb994bf7 100644 --- a/doc/ci/variables/README.md +++ b/doc/ci/variables/README.md @@ -12,10 +12,11 @@ affect the way running processes behave on an operating system. Environment variables are part of the environment in which a process runs. -For example, a running process can query the value of the -`TEMP` environment variable to discover a suitable location -to store temporary files, or to define a `URL` for a database -that can be reused in different scripts. +For example, a running process could: + +- Use the value of a `TEMP` environment variable to know the correct location + to store temporary files. +- Use a `DATABASE_URL` variable for the URL to a database that can be reused in different scripts. Variables are useful for customizing your jobs in GitLab CI/CD. When you use variables, you don't have to hard-code values. @@ -62,22 +63,6 @@ job `test_variable`, which is `test`:  -As another example, let's say you're using your own GitLab -instance and you want to know what domain your GitLab Pages are -served under. You can call it by using the predefined -variable `$CI_PAGES_DOMAIN` in your script: - -```yaml -pages: - script: - - ... - - echo $CI_PAGES_DOMAIN -``` - -For GitLab.com users, the output is `gitlab.io`. For your -private instance, the output is whatever your sysadmin has -defined. - ## Custom environment variables When you need a specific custom environment variable, you can @@ -103,8 +88,8 @@ variables: You can then call its value in your script: ```yaml - script: - - echo "$TEST" +script: + - echo "$TEST" ``` For more details, see [`.gitlab-ci.yml` defined variables](#gitlab-ciyml-defined-variables). @@ -181,8 +166,8 @@ You can use tools like [the AWS CLI](https://docs.aws.amazon.com/cli/latest/user and [`kubectl`](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/#the-kubeconfig-environment-variable) to customize your configuration by using **File** type variables. -In the past, a common pattern was to read the value of a CI variable, save it in a file, and then -use the newly created file in your script: +Previously, a common pattern was to read the value of a CI variable, save it in a file, and then +use that file in your script: ```shell # Read certificate stored in $KUBE_CA_PEM variable and save it in a new file @@ -258,7 +243,7 @@ Some variables are listed in the UI so you can choose them more quickly. | `AWS_DEFAULT_REGION` | Any | 12.10 | | `AWS_SECRET_ACCESS_KEY` | Any | 12.10 | -NOTE: **Note:** +CAUTION: **Caution:** When you store credentials, there are security implications. If you are using AWS keys, for example, follow their [best practices](https://docs.aws.amazon.com/general/latest/gr/aws-access-keys-best-practices.html). @@ -288,11 +273,11 @@ job_name: ### PowerShell -To access environment variables in a **Windows PowerShell** environment, prefix -the variable name with (`$env:`). For environment variables set by GitLab CI, including those set by [`variables`](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/ci/yaml/README.md#variables) -parameter, they can also be accessed by prefixing the variable name with (`$`) -as of [GitLab Runner 1.0.0](https://gitlab.com/gitlab-org/gitlab-runner/-/commit/abc44bb158008cd3a49c0d8173717c38dadb29ae#47afd7e8f12afdb8f0246262488f24e6dd071a22). -System set environment variables however must be accessed using (`$env:`). +To access variables in a **Windows PowerShell** environment, including system set +environment variables, prefix the variable name with (`$env:`). Environment variables +set by GitLab CI can also be accessed by prefixing the variable name with (`$`) with +[GitLab Runner 1.0.0](https://gitlab.com/gitlab-org/gitlab-runner/-/merge_requests/68) +and later. ```yaml job_name: @@ -386,9 +371,6 @@ export GITLAB_USER_ID="42" ## `.gitlab-ci.yml` defined variables -NOTE: **Note:** -This feature requires GitLab Runner 0.5.0 or higher and GitLab 7.14 or higher. - You can add variables that are set in the build environment to `.gitlab-ci.yml`. These variables are saved in the repository, and they are meant to store non-sensitive project configuration, like `RAILS_ENV` or @@ -428,10 +410,12 @@ script: > Introduced in GitLab 9.4. -You can define per-project or per-group variables -that are set in the pipeline environment. Group-level variables are stored out of -the repository (not in `.gitlab-ci.yml`) and are securely passed to GitLab Runner, -which makes them available during a pipeline run. For Premium users who do **not** use an external key store or who use GitLab's [integration with HashiCorp Vault](../secrets/index.md), we recommend using group environment variables to store secrets like passwords, SSH keys, and credentials. +You can define per-project or per-group variables that are set in the pipeline environment. Group-level variables are stored out of the repository (not in `.gitlab-ci.yml`). They are securely passed to GitLab Runner, which makes them available during a pipeline run. + +We recommend using group environment variables to store secrets (like passwords, SSH keys, and credentials) for Premium users who: + +- Do **not** use an external key store. +- Use GitLab's [integration with HashiCorp Vault](../secrets/index.md). Group-level variables can be added by: @@ -452,8 +436,7 @@ Once you set them, they are available for all subsequent pipelines. Any group-le Instance variables are useful for no longer needing to manually enter the same credentials repeatedly for all your projects. Instance-level variables are available to all projects and groups on the instance. -NOTE: **Note:** -The maximum number of instance-level variables is [planned to be 25](https://gitlab.com/gitlab-org/gitlab/-/issues/216097). +In GitLab 13.1 and later, the [maximum number of instance-level variables is 25](https://gitlab.com/gitlab-org/gitlab/-/issues/216097). You can define instance-level variables via the UI or [API](../../api/instance_level_ci_variables.md). @@ -463,7 +446,7 @@ To add an instance-level variable: 1. Click the **Add variable** button, and fill in the details: - **Key**: Must be one line, using only letters, numbers, or `_` (underscore), with no spaces. - - **Value**: [Since GitLab 13.3](https://gitlab.com/gitlab-org/gitlab/-/issues/220028), 10,000 characters allowed. This is also bounded by the limits of the selected runner operating system. In GitLab 13.0 to 13.2, 700 characters allowed. + - **Value**: [In GitLab 13.3 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/220028), 10,000 characters allowed. This is also bounded by the limits of the selected runner operating system. In GitLab 13.0 to 13.2, 700 characters allowed. - **Type**: `File` or `Variable`. - **Protect variable** (Optional): If selected, the variable is only available in pipelines that run on protected branches or tags. - **Mask variable** (Optional): If selected, the variable's **Value** is not shown in job logs. The variable is not saved if the value does not meet the [masking requirements](#masked-variable-requirements). @@ -566,12 +549,11 @@ variables take precedence over those defined in `.gitlab-ci.yml`. Variable names are limited by the underlying shell used to execute scripts (see [available shells](https://docs.gitlab.com/runner/shells/index.html). Each shell has its own unique set of reserved variable names. -You also want to keep in mind the [scope of environment variables](where_variables_can_be_used.md) to ensure a variable is defined in the scope -in which you wish to use it. +Keep in mind the [scope of environment variables](where_variables_can_be_used.md) to ensure a variable is defined in the scope in which you wish to use it. ## Where variables can be used -Click [here](where_variables_can_be_used.md) for a section that describes where and how the different types of variables can be used. +[This section](where_variables_can_be_used.md) describes where and how the different types of variables can be used. ## Advanced use @@ -609,8 +591,8 @@ then available as environment variables on the running application container. CAUTION: **Caution:** -Variables with multi-line values are not currently supported due to -limitations with the current Auto DevOps scripting environment. +Variables with multi-line values are not supported due to +limitations with the Auto DevOps scripting environment. ### Override a variable by manually running a pipeline @@ -701,9 +683,10 @@ Examples: - `$VARIABLE == ""` - `$VARIABLE != ""` (introduced in GitLab 11.11) -If you want to check whether a variable is defined, but is empty, you can -simply compare it against an empty string, like `$VAR == ''` or non-empty -string `$VARIABLE != ""`. +To check if a variable is defined but empty, compare it to: + +- An empty string: `$VARIABLE == ''` +- A non-empty string: `$VARIABLE != ""` #### Comparing two variables @@ -719,9 +702,8 @@ of these variables. Example: `$STAGING` -If you only want to create a job when there is some variable present, -which means that it is defined and non-empty, you can simply use -variable name as an expression, like `$STAGING`. If `$STAGING` variable +To create a job when there is some variable present, meaning it is defined and non-empty, +use the variable name as an expression, like `$STAGING`. If the `$STAGING` variable is defined, and is non empty, expression evaluates to `true`. `$STAGING` value needs to be a string, with length higher than zero. Variable that contains only whitespace characters is not an empty variable. @@ -785,7 +767,7 @@ Examples: ##### Enable or disable parenthesis support for variables **(CORE ONLY)** -The feature is currently deployed behind a feature flag that is **enabled by default**. +The feature is deployed behind a feature flag that is **enabled by default**. [GitLab administrators with access to the GitLab Rails console](../../administration/feature_flags.md) can opt to disable it for your instance. @@ -820,8 +802,7 @@ NOTE: **Note:** The available regular expression syntax is limited. See [related issue](https://gitlab.com/gitlab-org/gitlab/-/issues/35438) for more details. -If needed, you can use a test pipeline to determine whether a regular expression will -work in a variable. The example below tests the `^mast.*` regular expression directly, +If needed, you can use a test pipeline to determine whether a regular expression works in a variable. The example below tests the `^mast.*` regular expression directly, as well as from within a variable: ```yaml @@ -856,9 +837,7 @@ from being leaked into the log unless your script writes them to the screen. If a job isn't working as expected, this can make the problem difficult to investigate; in these cases, you can enable debug tracing in `.gitlab-ci.yml`. -Available on GitLab Runner v1.7+, this feature enables the shell's execution -log, resulting in a verbose job log listing all commands that were run, -variables that were set, and so on. +Available on GitLab Runner v1.7+, this feature enables the shell's execution log. This results in a verbose job log listing all commands that were run, variables that were set, and so on. Before enabling this, you should ensure jobs are visible to [team members only](../../user/permissions.md#project-features). You should diff --git a/doc/ci/variables/deprecated_variables.md b/doc/ci/variables/deprecated_variables.md index 94ec8439605..71e2b5b2e44 100644 --- a/doc/ci/variables/deprecated_variables.md +++ b/doc/ci/variables/deprecated_variables.md @@ -16,7 +16,6 @@ To follow conventions of naming across GitLab, and to further move away from the `build` term and toward `job`, some [CI/CD environment variables](README.md#predefined-environment-variables) were renamed for GitLab 9.0 release. -NOTE: **Note:** Starting with GitLab 9.0, we have deprecated the `$CI_BUILD_*` variables. **You are strongly advised to use the new variables as we will remove the old ones in future GitLab releases.** diff --git a/doc/ci/variables/predefined_variables.md b/doc/ci/variables/predefined_variables.md index 915041b71a6..08aaacd2620 100644 --- a/doc/ci/variables/predefined_variables.md +++ b/doc/ci/variables/predefined_variables.md @@ -69,7 +69,8 @@ Kubernetes-specific environment variables are detailed in the | `CI_JOB_MANUAL` | 8.12 | all | The flag to indicate that job was manually started | | `CI_JOB_NAME` | 9.0 | 0.5 | The name of the job as defined in `.gitlab-ci.yml` | | `CI_JOB_STAGE` | 9.0 | 0.5 | The name of the stage as defined in `.gitlab-ci.yml` | -| `CI_JOB_TOKEN` | 9.0 | 1.2 | Token used for authenticating with the [GitLab Container Registry](../../user/packages/container_registry/index.md), downloading [dependent repositories](../../user/project/new_ci_build_permissions_model.md#dependent-repositories), and accessing [GitLab-managed Terraform state](../../user/infrastructure/index.md#gitlab-managed-terraform-state). | +| `CI_JOB_STATUS` | all | 13.5 | The state of the job as each runner stage is executed. Use with [`after_script`](../yaml/README.md#before_script-and-after_script) where `CI_JOB_STATUS` can be either: `success`, `failed` or `canceled`. | +| `CI_JOB_TOKEN` | 9.0 | 1.2 | Token used for authenticating with [a few API endpoints](../../api/README.md#gitlab-ci-job-token) and downloading [dependent repositories](../../user/project/new_ci_build_permissions_model.md#dependent-repositories). The token is valid as long as the job is running. | | `CI_JOB_JWT` | 12.10 | all | RS256 JSON web token that can be used for authenticating with third party systems that support JWT authentication, for example [HashiCorp's Vault](../secrets/index.md). | | `CI_JOB_URL` | 11.1 | 0.5 | Job details URL | | `CI_KUBERNETES_ACTIVE` | 13.0 | all | Included with the value `true` only if the pipeline has a Kubernetes cluster available for deployments. Not included if no cluster is available. Can be used as an alternative to [`only:kubernetes`/`except:kubernetes`](../yaml/README.md#onlykubernetesexceptkubernetes) with [`rules:if`](../yaml/README.md#rulesif) | @@ -104,7 +105,7 @@ Kubernetes-specific environment variables are detailed in the | `CI_PROJECT_ID` | all | all | The unique ID of the current project that GitLab CI/CD uses internally | | `CI_PROJECT_NAME` | 8.10 | 0.5 | The name of the directory for the project that is currently being built. For example, if the project URL is `gitlab.example.com/group-name/project-1`, the `CI_PROJECT_NAME` would be `project-1`. | | `CI_PROJECT_NAMESPACE` | 8.10 | 0.5 | The project namespace (username or group name) that is currently being built | -| `CI_PROJECT_ROOT_NAMESPACE` | 13.2 | 0.5 | The **root** project namespace (username or group name) that is currently being built. For example, if `CI_PROJECT_NAME` is `root-group/child-group/grandchild-group`, `CI_PROJECT_ROOT_NAMESPACE` would be `root-group`. | +| `CI_PROJECT_ROOT_NAMESPACE` | 13.2 | 0.5 | The **root** project namespace (username or group name) that is currently being built. For example, if `CI_PROJECT_NAMESPACE` is `root-group/child-group/grandchild-group`, `CI_PROJECT_ROOT_NAMESPACE` would be `root-group`. | | `CI_PROJECT_PATH` | 8.10 | 0.5 | The namespace with project name | | `CI_PROJECT_PATH_SLUG` | 9.3 | all | `$CI_PROJECT_PATH` lowercased and with everything except `0-9` and `a-z` replaced with `-`. Use in URLs and domain names. | | `CI_PROJECT_REPOSITORY_LANGUAGES` | 12.3 | all | Comma-separated, lowercased list of the languages used in the repository (e.g. `ruby,javascript,html,css`) | diff --git a/doc/ci/variables/where_variables_can_be_used.md b/doc/ci/variables/where_variables_can_be_used.md index 330b960ca9a..b4236ca34c2 100644 --- a/doc/ci/variables/where_variables_can_be_used.md +++ b/doc/ci/variables/where_variables_can_be_used.md @@ -38,15 +38,14 @@ There are two places defined variables can be used. On the: ### `config.toml` file -NOTE: **Note:** -You can read more about `config.toml` in the [GitLab Runner docs](https://docs.gitlab.com/runner/configuration/advanced-configuration.html). - | Definition | Can be expanded? | Description | |:-------------------------------------|:-----------------|:---------------------------------------------------------------------------------------------------------------------------------------------| | `runners.environment` | yes | The variable expansion is made by GitLab Runner's [internal variable expansion mechanism](#gitlab-runner-internal-variable-expansion-mechanism) | | `runners.kubernetes.pod_labels` | yes | The Variable expansion is made by GitLab Runner's [internal variable expansion mechanism](#gitlab-runner-internal-variable-expansion-mechanism) | | `runners.kubernetes.pod_annotations` | yes | The Variable expansion is made by GitLab Runner's [internal variable expansion mechanism](#gitlab-runner-internal-variable-expansion-mechanism) | +You can read more about `config.toml` in the [GitLab Runner docs](https://docs.gitlab.com/runner/configuration/advanced-configuration.html). + ## Expansion mechanisms There are three expansion mechanisms: @@ -104,10 +103,6 @@ These restrictions are because `after_script` scripts are executed in a ## Persisted variables -NOTE: **Note:** -Some of the persisted variables contain tokens and cannot be used by some definitions -due to security reasons. - The following variables are known as "persisted": - `CI_PIPELINE_ID` @@ -130,6 +125,9 @@ They are: - For definitions where the ["Expansion place"](#gitlab-ciyml-file) is GitLab. - In the `only` and `except` [variables expressions](README.md#environment-variables-expressions). +Some of the persisted variables contain tokens and cannot be used by some definitions +due to security reasons. + ## Variables with an environment scope Variables defined with an environment scope are supported. Given that diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md index 40df9c7c986..2e4ab68a0e8 100644 --- a/doc/ci/yaml/README.md +++ b/doc/ci/yaml/README.md @@ -33,11 +33,8 @@ We have complete examples of configuring pipelines: > - <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> Learn how [Verizon reduced rebuilds](https://about.gitlab.com/blog/2019/02/14/verizon-customer-story/) > from 30 days to under 8 hours with GitLab. -NOTE: **Note:** If you have a [mirrored repository that GitLab pulls from](../../user/project/repository/repository_mirroring.md#pulling-from-a-remote-repository), -you may need to enable pipeline triggering. Go to your project's - -**Settings > Repository > Pull from a remote repository > Trigger pipelines for mirror updates**. +you may need to enable pipeline triggering. Go to your project's **Settings > Repository > Pull from a remote repository > Trigger pipelines for mirror updates**. ## Introduction @@ -107,12 +104,12 @@ The following table lists available parameters for jobs: | [`script`](#script) | Shell script that is executed by a runner. | | [`after_script`](#before_script-and-after_script) | Override a set of commands that are executed after job. | | [`allow_failure`](#allow_failure) | Allow job to fail. Failed job does not contribute to commit status. | -| [`artifacts`](#artifacts) | List of files and directories to attach to a job on success. Also available: `artifacts:paths`, `artifacts:exclude`, `artifacts:expose_as`, `artifacts:name`, `artifacts:untracked`, `artifacts:when`, `artifacts:expire_in`, `artifacts:reports`. | +| [`artifacts`](#artifacts) | List of files and directories to attach to a job on success. Also available: `artifacts:paths`, `artifacts:exclude`, `artifacts:expose_as`, `artifacts:name`, `artifacts:untracked`, `artifacts:when`, `artifacts:expire_in`, and `artifacts:reports`. | | [`before_script`](#before_script-and-after_script) | Override a set of commands that are executed before job. | -| [`cache`](#cache) | List of files that should be cached between subsequent runs. Also available: `cache:paths`, `cache:key`, `cache:untracked`, and `cache:policy`. | +| [`cache`](#cache) | List of files that should be cached between subsequent runs. Also available: `cache:paths`, `cache:key`, `cache:untracked`, `cache:when`, and `cache:policy`. | | [`coverage`](#coverage) | Code coverage settings for a given job. | | [`dependencies`](#dependencies) | Restrict which artifacts are passed to a specific job by providing a list of jobs to fetch artifacts from. | -| [`environment`](#environment) | Name of an environment to which the job deploys. Also available: `environment:name`, `environment:url`, `environment:on_stop`, `environment:auto_stop_in` and `environment:action`. | +| [`environment`](#environment) | Name of an environment to which the job deploys. Also available: `environment:name`, `environment:url`, `environment:on_stop`, `environment:auto_stop_in`, and `environment:action`. | | [`except`](#onlyexcept-basic) | Limit when jobs are not created. Also available: [`except:refs`, `except:kubernetes`, `except:variables`, and `except:changes`](#onlyexcept-advanced). | | [`extends`](#extends) | Configuration entries that this job inherits from. | | [`image`](#image) | Use Docker images. Also available: `image:name` and `image:entrypoint`. | @@ -184,7 +181,7 @@ To enable or disable the inheritance of all `variables:` or `default:` parameter - `variables: true` or `variables: false` To inherit only a subset of `default:` parameters or `variables:`, specify what -you wish to inherit, and any not listed will **not** be inherited. Use +you wish to inherit. Anything not listed is **not** inherited. Use one of the following formats: ```yaml @@ -208,16 +205,16 @@ inherit: In the example below: - `rubocop`: - - **will** inherit: Nothing. + - inherits: Nothing. - `rspec`: - - **will** inherit: the default `image` and the `WEBHOOK_URL` variable. - - will **not** inherit: the default `before_script` and the `DOMAIN` variable. + - inherits: the default `image` and the `WEBHOOK_URL` variable. + - does **not** inherit: the default `before_script` and the `DOMAIN` variable. - `capybara`: - - **will** inherit: the default `before_script` and `image`. - - will **not** inherit: the `DOMAIN` and `WEBHOOK_URL` variables. + - inherits: the default `before_script` and `image`. + - does **not** inherit: the `DOMAIN` and `WEBHOOK_URL` variables. - `karma`: - - **will** inherit: the default `image` and `before_script`, and the `DOMAIN` variable. - - will **not** inherit: `WEBHOOK_URL` variable. + - inherits: the default `image` and `before_script`, and the `DOMAIN` variable. + - does **not** inherit: `WEBHOOK_URL` variable. ```yaml default: @@ -347,23 +344,23 @@ workflow: This example never allows pipelines for schedules or `push` (branches and tags) pipelines, but does allow pipelines in **all** other cases, *including* merge request pipelines. -As with `rules` defined in jobs, be careful not to use a configuration that allows -merge request pipelines and branch pipelines to run at the same time, or you could -have [duplicate pipelines](#prevent-duplicate-pipelines). +Be careful not to use a configuration that might run +merge request pipelines and branch pipelines at the same time. As with `rules` defined in jobs, +it can cause [duplicate pipelines](#prevent-duplicate-pipelines). #### `workflow:rules` templates > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/217732) in GitLab 13.0. -We provide pre-made templates for use with your pipelines that set up `workflow: rules` -for common scenarios. Usage of these will make things easier and prevent duplicate pipelines from running. +We provide templates that set up `workflow: rules` +for common scenarios. These templates help prevent duplicate pipelines. The [`Branch-Pipelines` template](https://gitlab.com/gitlab-org/gitlab/-/tree/master/lib/gitlab/ci/templates/Workflows/Branch-Pipelines.gitlab-ci.yml) makes your pipelines run for branches and tags. -Branch pipeline status will be displayed within merge requests that use that branch -as a source, but this pipeline type does not support any features offered by -[Merge Request Pipelines](../merge_request_pipelines/) like +Branch pipeline status is displayed within merge requests that use the branch +as a source. However, this pipeline type does not support any features offered by +[Merge Request Pipelines](../merge_request_pipelines/), like [Pipelines for Merge Results](../merge_request_pipelines/#pipelines-for-merged-results) or [Merge Trains](../merge_request_pipelines/pipelines_for_merged_results/merge_trains/). Use this template if you are intentionally avoiding those features. @@ -391,7 +388,7 @@ include: ### `include` > - Introduced in [GitLab Premium](https://about.gitlab.com/pricing/) 10.5. -> - Available for Starter, Premium and Ultimate since 10.6. +> - Available for Starter, Premium, and Ultimate in GitLab 10.6 and later. > - [Moved](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/42861) to GitLab Core in 11.4. Using the `include` keyword allows the inclusion of external YAML files. This helps @@ -402,6 +399,10 @@ configuration files. This helps avoid duplicated configuration, for example, glo `include` requires the external YAML file to have the extensions `.yml` or `.yaml`, otherwise the external file is not included. +Using [YAML anchors](#anchors) across different YAML files sourced by `include` is not +supported. You must only refer to anchors in the same file. Instead +of using YAML anchors, you can use the [`extends` keyword](#extends). + `include` supports the following inclusion methods: | Method | Description | @@ -413,7 +414,6 @@ otherwise the external file is not included. The `include` methods do not support [variable expansion](../variables/where_variables_can_be_used.md#variables-usage). -NOTE: **Note:** `.gitlab-ci.yml` configuration included by all methods is evaluated at pipeline creation. The configuration is a snapshot in time and persisted in the database. Any changes to referenced `.gitlab-ci.yml` configuration is not reflected in GitLab until the next pipeline is created. @@ -428,11 +428,6 @@ TIP: **Tip:** Use merging to customize and override included CI/CD configurations with local definitions. Local definitions in `.gitlab-ci.yml` override included definitions. -NOTE: **Note:** -Using [YAML anchors](#anchors) across different YAML files sourced by `include` is not -supported. You must only refer to anchors in the same file. Instead -of using YAML anchors, you can use the [`extends` keyword](#extends). - #### `include:local` `include:local` includes a file from the same repository as `.gitlab-ci.yml`. @@ -442,12 +437,11 @@ You can only use files that are tracked by Git on the same branch your configuration file is on. In other words, when using a `include:local`, make sure that both `.gitlab-ci.yml` and the local file are on the same branch. +Including local files through Git submodules paths is not supported. + All [nested includes](#nested-includes) are executed in the scope of the same project, so it's possible to use local, project, remote, or template includes. -NOTE: **Note:** -Including local files through Git submodules paths is not supported. - Example: ```yaml @@ -455,7 +449,6 @@ include: - local: '/templates/.gitlab-ci-template.yml' ``` -TIP: **Tip:** Local includes can be used as a replacement for symbolic links that are not followed. This can be defined as a short local include: @@ -495,8 +488,8 @@ include: file: '/templates/.gitlab-ci-template.yml' ``` -All [nested includes](#nested-includes) are executed in the scope of the target project, -so it's possible to use local (relative to target project), project, remote +All [nested includes](#nested-includes) are executed in the scope of the target project. +This means you can use local (relative to target project), project, remote, or template includes. #### `include:remote` @@ -548,7 +541,7 @@ Nested includes allow you to compose a set of includes. A total of 100 includes is allowed, but duplicate includes are considered a configuration error. -Since [GitLab 12.4](https://gitlab.com/gitlab-org/gitlab/-/issues/28212), the time limit +In [GitLab 12.4](https://gitlab.com/gitlab-org/gitlab/-/issues/28212) and later, the time limit for resolving all files is 30 seconds. #### Additional `includes` examples @@ -635,10 +628,9 @@ job: - bundle exec rspec ``` -NOTE: **Note:** Sometimes, `script` commands must be wrapped in single or double quotes. -For example, commands that contain a colon (`:`) must be wrapped in quotes so -that the YAML parser knows to interpret the whole thing as a string rather than +For example, commands that contain a colon (`:`) must be wrapped in quotes. +The YAML parser needs to interpret the text as a string rather than a "key: value" pair. Be careful when using special characters: `:`, `{`, `}`, `[`, `]`, `,`, `&`, `*`, `#`, `?`, `|`, `-`, `<`, `>`, `=`, `!`, `%`, `@`, `` ` ``. @@ -657,15 +649,18 @@ job: > Introduced in GitLab 8.7 and requires GitLab Runner v1.2. -`before_script` is used to define a command that should be run before each +`before_script` is used to define commands that should be run before each job, including deploy jobs, but after the restoration of any [artifacts](#artifacts). This must be an array. Scripts specified in `before_script` are concatenated with any scripts specified in the main [`script`](#script), and executed together in a single shell. -`after_script` is used to define the command that runs after each -job, including failed ones. This must be an array. +`after_script` is used to define commands that run after each +job, including failed jobs. This must be an array. If a job times out or is cancelled, +the `after_script` commands are not executed. Support for executing `after_script` +commands for timed-out or cancelled jobs +[is planned](https://gitlab.com/gitlab-org/gitlab/-/issues/15603). Scripts specified in `after_script` are executed in a new shell, separate from any `before_script` or `script` scripts. As a result, they: @@ -744,11 +739,11 @@ using [`|` (literal) and `>` (folded) YAML multi-line block scalar indicators](h CAUTION: **Warning:** If multiple commands are combined into one command string, only the last command's -failure or success is reported, -[incorrectly ignoring failures from earlier commands due to a bug](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/25394). -If the success of the job depends on the success or failure of these commands, -you can run the commands as separate `script:` items, or add `exit 1` commands -as appropriate to the command string where needed. +failure or success is reported. +[Failures from earlier commands are ignored due to a bug](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/25394). +To work around this, +run each command as a separate `script:` item, or add an `exit 1` command +to each command string. You can use the `|` (literal) YAML multiline block scalar indicator to write commands over multiple lines in the `script` section of a job description. @@ -808,7 +803,7 @@ Second command line. ``` When you omit the `>` or `|` block scalar indicators, GitLab forms the command -by concatenating non-empty lines, so make sure the lines can run when concatenated. +by concatenating non-empty lines. Make sure the lines can run when concatenated. Shell [here documents](https://en.wikipedia.org/wiki/Here_document) work with the `|` and `>` operators as well. The example below transliterates the lower case letters @@ -895,6 +890,8 @@ The following stages are available to every pipeline: User-defined stages are executed after `.pre` and before `.post`. +A pipeline is not created if all jobs are in `.pre` or `.post` stages. + The order of `.pre` and `.post` can't be changed, even if defined out of order in `.gitlab-ci.yml`. For example, the following are equivalent configuration: @@ -926,9 +923,6 @@ For example, the following are equivalent configuration: - b ``` -NOTE: **Note:** -A pipeline is not created if all jobs are in `.pre` or `.post` stages. - ### `extends` > Introduced in GitLab 11.3. @@ -961,7 +955,7 @@ GitLab performs a reverse deep merge based on the keys. GitLab: - Merges the `rspec` contents into `.tests` recursively. - Doesn't merge the values of the keys. -The result is this `rspec` job: +The result is this `rspec` job, where `script: rake test` is overwritten by `script: rake rspec`: ```yaml rspec: @@ -974,9 +968,6 @@ rspec: - $RSPEC ``` -NOTE: **Note:** -Note that `script: rake test` has been overwritten by `script: rake rspec`. - If you do want to include the `rake test`, see [`before_script` and `after_script`](#before_script-and-after_script). `.tests` in this example is a [hidden job](#hide-jobs), but it's @@ -1113,7 +1104,6 @@ is either included or excluded from the pipeline, depending on the configuration If included, the job also has [certain attributes](#rules-attributes) added to it. -CAUTION: **Caution:** `rules` replaces [`only/except`](#onlyexcept-basic) and can't be used in conjunction with it. If you attempt to use both keywords in the same job, the linter returns a `key may not be used with rules` error. @@ -1500,9 +1490,8 @@ job: - spec/**.rb ``` -NOTE: **Note:** -For performance reasons, using `exists` with patterns is limited to 10000 -checks. After the 10000th check, rules with patterned globs always match. +For performance reasons, using `exists` with patterns is limited to 10,000 +checks. After the 10,000th check, rules with patterned globs always match. #### `rules:allow_failure` @@ -1543,15 +1532,15 @@ docker build: script: docker build -t my-image:$CI_COMMIT_REF_SLUG . rules: - if: '$VAR == "string value"' - changes: # Will include the job and set to when:manual if any of the follow paths match a modified file. + changes: # Include the job and set to when:manual if any of the follow paths match a modified file. - Dockerfile - docker/scripts/* when: manual - # - when: never would be redundant here, this is implied any time rules are listed. + # - when: never would be redundant here, this is implied any time rules are listed. ``` -Keywords such as `branches` or `refs` that are currently available for -`only`/`except` are not yet available in `rules` as they are being individually +Keywords such as `branches` or `refs` that are available for +`only`/`except` are not available in `rules`. They are being individually considered for their usage and behavior in this context. Future keyword improvements are being discussed in our [epic for improving `rules`](https://gitlab.com/groups/gitlab-org/-/epics/2783), where anyone can add suggestions or requests. @@ -1567,10 +1556,9 @@ job1: if: ($CI_COMMIT_BRANCH == "master" || $CI_COMMIT_BRANCH == "develop") && $MY_VARIABLE ``` -NOTE: **Note:** -In GitLab 13.2 and older, the order of operations when mixing `||` and `&&` in a single rule may not have executed -in the expected order. This is [fixed](https://gitlab.com/gitlab-org/gitlab/-/issues/230938) -in GitLab 13.3. +CAUTION: **Caution:** +[Before GitLab 13.3](https://gitlab.com/gitlab-org/gitlab/-/issues/230938), +rules that use both `||` and `&&` may evaluate with an unexpected order of operations. ### `only`/`except` (basic) @@ -1611,8 +1599,8 @@ In addition, `only` and `except` allow the use of special keywords: | `triggers` | For pipelines created by using a [trigger token](../triggers/README.md#trigger-token). | | `web` | For pipelines created by using **Run pipeline** button in the GitLab UI, from the project's **CI/CD > Pipelines** section. | -In the example below, `job` will run only for refs that start with `issue-`, -whereas all branches will be skipped: +In the example below, `job` runs only for refs that start with `issue-`, +whereas all branches are skipped: ```yaml job: @@ -1637,8 +1625,8 @@ job: - branches ``` -In this example, `job` will run only for refs that are tagged, or if a build is -explicitly requested via an API trigger or a [Pipeline Schedule](../pipelines/schedules.md): +In this example, `job` runs only for refs that are tagged, or if a build is +explicitly requested by an API trigger or a [Pipeline Schedule](../pipelines/schedules.md): ```yaml job: @@ -1684,19 +1672,19 @@ job: #### Regular expressions -Because `@` is used to denote the beginning of a ref's repository path, -matching a ref name containing the `@` character in a regular expression -requires the use of the hex character code match `\x40`. +The `@` symbol denotes the beginning of a ref's repository path. +To match a ref name that contains the `@` character in a regular expression, +you must use the hex character code match `\x40`. Only the tag or branch name can be matched by a regular expression. The repository path, if given, is always matched literally. -If a regular expression shall be used to match the tag or branch name, -the entire ref name part of the pattern has to be a regular expression, -and must be surrounded by `/`. -(With regular expression flags appended after the closing `/`.) -So `issue-/.*/` won't work to match all tag names or branch names -that begin with `issue-`. +To match the tag or branch name, +the entire ref name part of the pattern must be a regular expression surrounded by `/`. +For example, you can't use `issue-/.*/` to match all tag names or branch names +that begin with `issue-`, but you can use `/issue-.*/`. + +Regular expression flags must be appended after the closing `/`. TIP: **Tip:** Use anchors `^` and `$` to avoid the regular expression @@ -1706,20 +1694,17 @@ while just `/issue/` would also match a branch called `severe-issues`. #### Supported `only`/`except` regexp syntax -CAUTION: **Warning:** -This is a breaking change that was introduced with GitLab 11.9.4. - -In GitLab 11.9.4, GitLab begun internally converting regexp used +In GitLab 11.9.4, GitLab began internally converting the regexp used in `only` and `except` parameters to [RE2](https://github.com/google/re2/wiki/Syntax). -This means that only subset of features provided by [Ruby Regexp](https://ruby-doc.org/core/Regexp.html) -is supported. [RE2](https://github.com/google/re2/wiki/Syntax) limits the set of features -provided due to computational complexity, which means some features became unavailable in GitLab 11.9.4. -For example, negative lookaheads. +[RE2](https://github.com/google/re2/wiki/Syntax) limits the set of available features +due to computational complexity, and some features, like negative lookaheads, became unavailable. +Only a subset of features provided by [Ruby Regexp](https://ruby-doc.org/core/Regexp.html) +are now supported. -For GitLab versions from 11.9.7 and up to GitLab 12.0, GitLab provides a feature flag that can be -enabled by administrators that allows users to use unsafe regexp syntax. This brings compatibility -with previously allowed syntax version and allows users to gracefully migrate to the new syntax. +From GitLab 11.9.7 to GitLab 12.0, GitLab provided a feature flag to +let you use the unsafe regexp syntax. This flag allowed +compatibility with the previous syntax version so you could gracefully migrate to the new syntax. ```ruby Feature.enable(:allow_unsafe_ruby_regexp) @@ -1727,10 +1712,6 @@ Feature.enable(:allow_unsafe_ruby_regexp) ### `only`/`except` (advanced) -CAUTION: **Warning:** -This is an _alpha_ feature, and is subject to change at any time without -prior notice! - GitLab supports both simple and complex strategies, so it's possible to use an array and a hash configuration scheme. @@ -1741,7 +1722,7 @@ Four keys are available: - `changes` - `kubernetes` -If you use multiple keys under `only` or `except`, the keys will be evaluated as a +If you use multiple keys under `only` or `except`, the keys are evaluated as a single conjoined expression. That is: - `only:` includes the job if **all** of the keys have at least one condition that matches. @@ -1752,9 +1733,9 @@ the pipeline if the following is true: - `(any listed refs are true) AND (any listed variables are true) AND (any listed changes are true) AND (any chosen Kubernetes status matches)` -In the example below, the `test` job will `only` be created when **all** of the following are true: +In the example below, the `test` job is `only` created when **all** of the following are true: -- The pipeline has been [scheduled](../../user/project/pipelines/schedules.md) **or** runs for `master`. +- The pipeline has been [scheduled](../pipelines/schedules.md) **or** runs for `master`. - The `variables` keyword matches. - The `kubernetes` service is active on the project. @@ -1775,7 +1756,7 @@ added if the following is true: - `(any listed refs are true) OR (any listed variables are true) OR (any listed changes are true) OR (a chosen Kubernetes status matches)` -In the example below, the `test` job will **not** be created when **any** of the following are true: +In the example below, the `test` job is **not** created when **any** of the following are true: - The pipeline runs for the `master` branch. - There are changes to the `README.md` file in the root directory of the repository. @@ -1797,8 +1778,8 @@ test: The `refs` strategy can take the same values as the [simplified only/except configuration](#onlyexcept-basic). -In the example below, the `deploy` job is going to be created only when the -pipeline has been [scheduled](../pipelines/schedules.md) or runs for the `master` branch: +In the example below, the `deploy` job is created only when the +pipeline is [scheduled](../pipelines/schedules.md) or runs for the `master` branch: ```yaml deploy: @@ -1814,7 +1795,7 @@ deploy: The `kubernetes` strategy accepts only the `active` keyword. -In the example below, the `deploy` job is going to be created only when the +In the example below, the `deploy` job is created only when the Kubernetes service is active in the project: ```yaml @@ -1827,12 +1808,11 @@ deploy: > `variables` policy introduced in GitLab 10.7. -The `variables` keyword is used to define variables expressions. In other words, -you can use predefined variables / project / group or -environment-scoped variables to define an expression GitLab is going to -evaluate in order to decide whether a job should be created or not. +The `variables` keyword defines variable expressions. + +These expressions determine whether or not a job should be created. -Examples of using variables expressions: +Examples of using variable expressions: ```yaml deploy: @@ -1902,22 +1882,21 @@ docker build: - more_scripts/*.{rb,py,sh} ``` -In the scenario above, when pushing commits to an existing branch in GitLab, -it creates and triggers the `docker build` job, provided that one of the -commits contains changes to any of the following: +When you push commits to an existing branch, +the `docker build` job is created, but only if changes were made to any of the following: - The `Dockerfile` file. -- Any of the files inside `docker/scripts/` directory. -- Any of the files and subdirectories inside the `dockerfiles` directory. -- Any of the files with `rb`, `py`, `sh` extensions inside the `more_scripts` directory. +- Any of the files in the `docker/scripts/` directory. +- Any of the files and subdirectories in the `dockerfiles` directory. +- Any of the files with `rb`, `py`, `sh` extensions in the `more_scripts` directory. CAUTION: **Warning:** -If using `only:changes` with [only allow merge requests to be merged if the pipeline succeeds](../../user/project/merge_requests/merge_when_pipeline_succeeds.md#only-allow-merge-requests-to-be-merged-if-the-pipeline-succeeds), -undesired behavior could result if you don't [also use `only:merge_requests`](#using-onlychanges-with-pipelines-for-merge-requests). +If you use `only:changes` with [only allow merge requests to be merged if the pipeline succeeds](../../user/project/merge_requests/merge_when_pipeline_succeeds.md#only-allow-merge-requests-to-be-merged-if-the-pipeline-succeeds), +you should [also use `only:merge_requests`](#using-onlychanges-with-pipelines-for-merge-requests). Otherwise it may not work as expected. You can also use glob patterns to match multiple files in either the root directory -of the repository, or in _any_ directory within the repository, but they must be wrapped -in double quotes or GitLab will fail to parse the `.gitlab-ci.yml`. For example: +of the repository, or in _any_ directory within the repository. However, they must be wrapped +in double quotes or GitLab can't parse them. For example: ```yaml test: @@ -1930,10 +1909,8 @@ test: - "**/*.sql" ``` -The following example will skip the `build` job if a change is detected in any file -in the root directory of the repository with a `.md` extension. This mean that if you change multiple files, -but only one file is a `.md` file, the `build` job will still be skipped and will -not run for the other files. +You can skip a job if a change is detected in any file with a +`.md` extension in the root directory of the repository: ```yaml build: @@ -1943,13 +1920,13 @@ build: - "*.md" ``` -CAUTION: **Warning:** -There are some points to be aware of when -[using this feature with new branches or tags *without* pipelines for merge requests](#using-onlychanges-without-pipelines-for-merge-requests). +If you change multiple files, but only one file ends in `.md`, +the `build` job is still skipped. The job does not run for any of the files. -CAUTION: **Warning:** -There are some points to be aware of when -[using this feature with scheduled pipelines](#using-onlychanges-with-scheduled-pipelines). +Read more about how to use this feature with: + +- [New branches or tags *without* pipelines for merge requests](#using-onlychanges-without-pipelines-for-merge-requests). +- [Scheduled pipelines](#using-onlychanges-with-scheduled-pipelines). ##### Using `only:changes` with pipelines for merge requests @@ -1957,7 +1934,7 @@ With [pipelines for merge requests](../merge_request_pipelines/index.md), it's possible to define a job to be created based on files modified in a merge request. -In order to deduce the correct base SHA of the source branch, we recommend combining +To deduce the correct base SHA of the source branch, we recommend combining this keyword with `only: [merge_requests]`. This way, file differences are correctly calculated from any further commits, thus all changes in the merge requests are properly tested in pipelines. @@ -1975,13 +1952,9 @@ docker build service one: - service-one/**/* ``` -In the scenario above, if a merge request is created or updated that changes -either files in `service-one` directory or the `Dockerfile`, GitLab creates -and triggers the `docker build service one` job. - -Note that if [pipelines for merge requests](../merge_request_pipelines/index.md) is -combined with `only: [change]`, but `only: [merge_requests]` is omitted, there could be -unwanted behavior. +In this scenario, if a merge request changes +files in the `service-one` directory or the `Dockerfile`, GitLab creates +the `docker build service one` job. For example: @@ -1994,15 +1967,17 @@ docker build service one: - service-one/**/* ``` -In the example above, a pipeline could fail due to changes to a file in `service-one/**/*`. -A later commit could then be pushed that does not include any changes to this file, -but includes changes to the `Dockerfile`, and this pipeline could pass because it's only -testing the changes to the `Dockerfile`. GitLab checks the **most recent pipeline**, -that **passed**, and will show the merge request as mergeable, despite the earlier -failed pipeline caused by a change that was not yet corrected. +In the example above, the pipeline might fail because of changes to a file in `service-one/**/*`. + +A later commit that doesn't have changes in `service-one/**/*` +but does have changes to the `Dockerfile` can pass. The job +only tests the changes to the `Dockerfile`. -With this configuration, care must be taken to check that the most recent pipeline -properly corrected any failures from previous pipelines. +GitLab checks the **most recent pipeline** that **passed**. If the merge request is mergeable, +it doesn't matter that an earlier pipeline failed because of a change that has not been corrected. + +When you use this configuration, ensure that the most recent pipeline +properly corrects any failures from previous pipelines. ##### Using `only:changes` without pipelines for merge requests @@ -2068,15 +2043,15 @@ production: This example creates four paths of execution: -- Linter: the `lint` job will run immediately without waiting for the `build` stage to complete because it has no needs (`needs: []`). +- Linter: the `lint` job runs immediately without waiting for the `build` stage to complete because it has no needs (`needs: []`). -- Linux path: the `linux:rspec` and `linux:rubocop` jobs will be run as soon +- Linux path: the `linux:rspec` and `linux:rubocop` jobs runs as soon as the `linux:build` job finishes without waiting for `mac:build` to finish. -- macOS path: the `mac:rspec` and `mac:rubocop` jobs will be run as soon +- macOS path: the `mac:rspec` and `mac:rubocop` jobs runs as soon as the `mac:build` job finishes, without waiting for `linux:build` to finish. -- The `production` job will be executed as soon as all previous jobs +- The `production` job runs as soon as all previous jobs finish; in this case: `linux:build`, `linux:rspec`, `linux:rubocop`, `mac:build`, `mac:rspec`, `mac:rubocop`. @@ -2084,14 +2059,14 @@ This example creates four paths of execution: - If `needs:` is set to point to a job that is not instantiated because of `only/except` rules or otherwise does not exist, the - pipeline will be created with YAML error. + pipeline is not created and a YAML error is shown. - The maximum number of jobs that a single job can need in the `needs:` array is limited: - For GitLab.com, the limit is 50. For more information, see our [infrastructure issue](https://gitlab.com/gitlab-com/gl-infra/infrastructure/-/issues/7541). - For self-managed instances, the limit is: 50. This limit [can be changed](#changing-the-needs-job-limit). - If `needs:` refers to a job that is marked as `parallel:`. - the current job will depend on all parallel jobs created. -- `needs:` is similar to `dependencies:` in that it needs to use jobs from prior stages, + the current job depends on all parallel jobs being created. +- `needs:` is similar to `dependencies:` in that it must use jobs from prior stages, meaning it's impossible to create circular dependencies. Depending on jobs in the current stage is not possible either, but support [is planned](https://gitlab.com/gitlab-org/gitlab/-/issues/30632). - Related to the above, stages must be explicitly defined for all jobs @@ -2108,8 +2083,7 @@ can choose a custom limit. For example, to set the limit to 100: Plan.default.actual_limits.update!(ci_needs_size_limit: 100) ``` -NOTE: **Note:** -To disable the ability to use DAG, set the limit to `0`. +To disable directed acyclic graphs (DAG), set the limit to `0`. #### Artifact downloads with `needs` @@ -2117,12 +2091,12 @@ To disable the ability to use DAG, set the limit to `0`. When using `needs`, artifact downloads are controlled with `artifacts: true` (default) or `artifacts: false`. -Since GitLab 12.6, you can't combine the [`dependencies`](#dependencies) keyword +In GitLab 12.6 and later, you can't combine the [`dependencies`](#dependencies) keyword with `needs` to control artifact downloads in jobs. `dependencies` is still valid in jobs that do not use `needs`. -In the example below, the `rspec` job will download the `build_job` artifacts, while the -`rubocop` job won't: +In the example below, the `rspec` job downloads the `build_job` artifacts, while the +`rubocop` job doesn't: ```yaml build_job: @@ -2144,9 +2118,9 @@ rubocop: artifacts: false ``` -Additionally, in the three syntax examples below, the `rspec` job will download the artifacts -from all three `build_jobs`, as `artifacts` is true for `build_job_1`, and will -**default** to true for both `build_job_2` and `build_job_3`. +Additionally, in the three syntax examples below, the `rspec` job downloads the artifacts +from all three `build_jobs`. `artifacts` is true for `build_job_1` and +**defaults** to true for both `build_job_2` and `build_job_3`. ```yaml rspec: @@ -2161,9 +2135,10 @@ rspec: > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/14311) in GitLab v12.7. -`needs` can be used to download artifacts from up to five jobs in pipelines on -[other refs in the same project](#artifact-downloads-between-pipelines-in-the-same-project), -or pipelines in different projects, groups and namespaces: +Use `needs` to download artifacts from up to five jobs in pipelines: + +- [On other refs in the same project](#artifact-downloads-between-pipelines-in-the-same-project). +- In different projects, groups and namespaces. ```yaml build_job: @@ -2177,7 +2152,7 @@ build_job: artifacts: true ``` -`build_job` will download the artifacts from the latest successful `build-1` job +`build_job` downloads the artifacts from the latest successful `build-1` job on the `master` branch in the `group/project-name` project. If the project is in the same group or namespace, you can omit them from the `project:` key. For example, `project: group/project-name` or `project: project-name`. @@ -2186,9 +2161,10 @@ The user running the pipeline must have at least `reporter` access to the group ##### Artifact downloads between pipelines in the same project -`needs` can be used to download artifacts from different pipelines in the current project -by setting the `project` keyword as the current project's name, and specifying a ref. -In the example below, `build_job` will download the artifacts for the latest successful +Use `needs` to download artifacts from different pipelines in the current project. +Set the `project` keyword as the current project's name, and specify a ref. + +In this example, `build_job` downloads the artifacts for the latest successful `build-1` job with the `other-ref` ref: ```yaml @@ -2220,7 +2196,6 @@ build_job: artifacts: true ``` -NOTE: **Note:** Downloading artifacts from jobs that are run in [`parallel:`](#parallel) is not supported. ### `tags` @@ -2232,7 +2207,7 @@ When you register a runner, you can specify the runner's tags, for example `ruby`, `postgres`, `development`. In this example, the job is run by a runner that -has both `ruby` AND `postgres` tags defined. +has both `ruby` and `postgres` tags defined. ```yaml job: @@ -2271,16 +2246,16 @@ The default value is `false`, except for [manual](#whenmanual) jobs using the `when: manual` syntax, unless using [`rules:`](#rules) syntax, where all jobs default to false, *including* `when: manual` jobs. -When enabled and the job fails, the job will show an orange warning in the UI. -However, the logical flow of the pipeline will consider the job a +When `allow_failure` is enabled and the job fails, the job shows an orange warning in the UI. +However, the logical flow of the pipeline considers the job a success/passed, and is not blocked. -Assuming all other jobs are successful, the job's stage and its pipeline will -show the same orange warning. However, the associated commit will be marked +Assuming all other jobs are successful, the job's stage and its pipeline +show the same orange warning. However, the associated commit is marked as "passed", without warnings. -In the example below, `job1` and `job2` will run in parallel, but if `job1` -fails, it won't stop the next stage from running, since it's marked with +In the example below, `job1` and `job2` run in parallel, but if `job1` +fails, it doesn't stop the next stage from running, because it's marked with `allow_failure: true`: ```yaml @@ -2309,15 +2284,15 @@ failure. `when` can be set to one of the following values: 1. `on_success` - execute job only when all jobs from prior stages - succeed (or are considered succeeding because they are marked - `allow_failure`). This is the default. + succeed (or are considered succeeding because they have `allow_failure: true`). + This is the default. 1. `on_failure` - execute job only when at least one job from prior stages fails. 1. `always` - execute job regardless of the status of jobs from prior stages. 1. `manual` - execute job manually (added in GitLab 8.10). Read about - [manual actions](#whenmanual) below. + [manual jobs](#whenmanual) below. 1. `delayed` - execute job after a certain period (added in GitLab 11.14). - Read about [delayed actions](#whendelayed) below. + Read about [delayed jobs](#whendelayed) below. 1. `never`: - With [`rules`](#rules), don't execute job. - With [`workflow:rules`](#workflowrules), don't run pipeline. @@ -2371,45 +2346,44 @@ The above script: #### `when:manual` > - Introduced in GitLab 8.10. -> - Blocking manual actions were introduced in GitLab 9.0. +> - Blocking manual jobs were introduced in GitLab 9.0. > - Protected actions were introduced in GitLab 9.2. -Manual actions are a special type of job that are not executed automatically, -they need to be explicitly started by a user. An example usage of manual actions -would be a deployment to a production environment. Manual actions can be started -from the pipeline, job, environment, and deployment views. Read more at the -[environments documentation](../environments/index.md#configuring-manual-deployments). +A manual job is a type of job that is not executed automatically and must be explicitly +started by a user. You might want to use manual jobs for things like deploying to production. -Manual actions can be either optional or blocking. Blocking manual actions will -block the execution of the pipeline at the stage this action is defined in. It's -possible to resume execution of the pipeline when someone executes a blocking -manual action by clicking a _play_ button. +To make a job manual, add `when: manual` to its configuration. -When a pipeline is blocked, it won't be merged if Merge When Pipeline Succeeds -is set. Blocked pipelines also do have a special status, called _manual_. -When the `when:manual` syntax is used, manual actions are non-blocking by -default. If you want to make manual action blocking, it's necessary to add -`allow_failure: false` to the job's definition in `.gitlab-ci.yml`. +Manual jobs can be started from the pipeline, job, [environment](../environments/index.md#configuring-manual-deployments), +and deployment views. -Optional manual actions have `allow_failure: true` set by default and their -Statuses don't contribute to the overall pipeline status. So, if a manual -action fails, the pipeline will eventually succeed. +Manual jobs can be either optional or blocking: -NOTE: **Note:** -When using [`rules:`](#rules), `allow_failure` defaults to `false`, including for manual jobs. +- **Optional**: Manual jobs have [`allow_failure: true](#allow_failure) set by default + and are considered optional. The status of an optional manual job does not contribute + to the overall pipeline status. A pipeline can succeed even if all its manual jobs fail. -Manual actions are considered to be write actions, so permissions for -[protected branches](../../user/project/protected_branches.md) are used when -a user wants to trigger an action. In other words, in order to trigger a manual -action assigned to a branch that the pipeline is running for, the user needs to -have the ability to merge to this branch. It's possible to use protected environments -to more strictly [protect manual deployments](#protecting-manual-jobs) from being -run by unauthorized users. +- **Blocking**: To make a blocking manual job, add `allow_failure: false` to its configuration. + Blocking manual jobs stop further execution of the pipeline at the stage where the + job is defined. To let the pipeline continue running, click **{play}** (play) on + the blocking manual job. -NOTE: **Note:** -Using `when:manual` and `trigger` together results in the error `jobs:#{job-name} when -should be on_success, on_failure or always`, because `when:manual` prevents triggers -being used. + Merge requests in projects with [merge when pipeline succeeds](../../user/project/merge_requests/merge_when_pipeline_succeeds.md) + enabled can't be merged with a blocked pipeline. Blocked pipelines show a status + of **blocked**. + +When you use [`rules:`](#rules), `allow_failure` defaults to `false`, including for manual jobs. + +To trigger a manual job, a user must have permission to merge to the assigned branch. +You can use [protected branches](../../user/project/protected_branches.md) to more strictly +[protect manual deployments](#protecting-manual-jobs) from being run by unauthorized users. + +In [GitLab 13.5](https://gitlab.com/gitlab-org/gitlab/-/issues/201938) and later, you +can use `when:manual` in the same job as [`trigger`](#trigger). In GitLab 13.4 and +earlier, using them together causes the error `jobs:#{job-name} when should be on_success, on_failure or always`. +It is deployed behind the `:ci_manual_bridges` [feature flag](../../user/feature_flags.md), which is **enabled by default**. +[GitLab administrators with access to the Rails console](../../administration/feature_flags.md) +can opt to disable it. ##### Protecting manual jobs **(PREMIUM)** @@ -2441,12 +2415,12 @@ To do this, you must: 1. In the [protected environments settings](../environments/protected_environments.md#protecting-environments), select the environment (`production` in the example above) and add the users, roles or groups that are authorized to trigger the manual job to the **Allowed to Deploy** list. Only those in - this list will be able to trigger this manual job, as well as GitLab administrators + this list can trigger this manual job, as well as GitLab administrators who are always able to use protected environments. -Additionally, if a manual job is defined as blocking by adding `allow_failure: false`, -the next stages of the pipeline won't run until the manual job is triggered. This -can be used as a way to have a defined list of users allowed to "approve" later pipeline +Additionally, if you define a manual job as blocking by adding `allow_failure: false`, +the pipeline's next stages don't run until the manual job is triggered. You can use this +to define a list of users allowed to "approve" later pipeline stages by triggering the blocking manual job. #### `when:delayed` @@ -2465,11 +2439,11 @@ provided. `start_in` key must be less than or equal to one week. Examples of val - `1 day` - `1 week` -When there is a delayed job in a stage, the pipeline won't progress until the delayed job has finished. +When there is a delayed job in a stage, the pipeline doesn't progress until the delayed job has finished. This means this keyword can also be used for inserting delays between different stages. The timer of a delayed job starts immediately after the previous stage has completed. -Similar to other types of jobs, a delayed job's timer won't start unless the previous stage passed. +Similar to other types of jobs, a delayed job's timer doesn't start unless the previous stage passed. The following example creates a job named `timed rollout 10%` that is executed 30 minutes after the previous stage has completed: @@ -2482,7 +2456,7 @@ timed rollout 10%: ``` You can stop the active timer of a delayed job by clicking the **{time-out}** (**Unschedule**) button. -This job will never be executed in the future unless you execute the job manually. +This job can no longer be scheduled to run automatically. You can, however, execute the job manually. To start a delayed job immediately, click the **Play** button. Soon GitLab Runner picks up and starts the job. @@ -2495,7 +2469,7 @@ Soon GitLab Runner picks up and starts the job. `environment` is used to define that a job deploys to a specific environment. If `environment` is specified and no environment under that name exists, a new -one will be created automatically. +one is created automatically. In its simplest form, the `environment` keyword can be defined like: @@ -2506,7 +2480,7 @@ deploy to production: environment: production ``` -In the above example, the `deploy to production` job will be marked as doing a +In the above example, the `deploy to production` job is marked as doing a deployment to the `production` environment. #### `environment:name` @@ -2574,12 +2548,12 @@ deploy to production: > - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/22191) in GitLab 8.13. > - Starting with GitLab 8.14, when you have an environment that has a stop action -> defined, GitLab will automatically trigger a stop action when the associated +> defined, GitLab automatically triggers a stop action when the associated > branch is deleted. -Closing (stopping) environments can be achieved with the `on_stop` keyword defined under -`environment`. It declares a different job that runs in order to close -the environment. +Closing (stopping) environments can be achieved with the `on_stop` keyword +defined under `environment`. It declares a different job that runs to close the +environment. Read the `environment:action` section for an example. @@ -2591,7 +2565,7 @@ The `action` keyword can be used to specify jobs that prepare, start, or stop en | **Value** | **Description** | |-----------|---------------------------------------------------------------------------------------------------------------------------------------------------------------| -| start | Default value. Indicates that job starts the environment. Deployment will be created after job starts. | +| start | Default value. Indicates that job starts the environment. The deployment is created after the job starts. | | prepare | Indicates that job is only preparing the environment. Does not affect deployments. [Read more about environments](../environments/index.md#prepare-an-environment) | | stop | Indicates that job stops deployment. See the example below. | @@ -2617,21 +2591,20 @@ stop_review_app: action: stop ``` -In the above example we set up the `review_app` job to deploy to the `review` -environment, and we also defined a new `stop_review_app` job under `on_stop`. -Once the `review_app` job is successfully finished, it will trigger the -`stop_review_app` job based on what is defined under `when`. In this case we -set it up to `manual` so it will need a [manual action](#whenmanual) via -GitLab's web interface in order to run. +In the above example, the `review_app` job deploys to the `review` +environment. A new `stop_review_app` job is listed under `on_stop`. +After the `review_app` job is finished, it triggers the +`stop_review_app` job based on what is defined under `when`. In this case, +it is set to `manual`, so it needs a [manual action](#whenmanual) from +GitLab's user interface to run. -Also in the example, `GIT_STRATEGY` is set to `none` so that GitLab Runner won’t -try to check out the code after the branch is deleted when the `stop_review_app` -job is [automatically triggered](../environments/index.md#automatically-stopping-an-environment). +Also in the example, `GIT_STRATEGY` is set to `none`. If the +`stop_review_app` job is [automatically triggered](../environments/index.md#automatically-stopping-an-environment), +the runner won’t try to check out the code after the branch is deleted. -NOTE: **Note:** -The above example overwrites global variables. If your stop environment job depends -on global variables, you can use [anchor variables](#yaml-anchors-for-variables) when setting the `GIT_STRATEGY` -to change it without overriding the global variables. +The example also overwrites global variables. If your `stop` `environment` job depends +on global variables, you can use [anchor variables](#yaml-anchors-for-variables) when you set the `GIT_STRATEGY`. +This changes the job without overriding the global variables. The `stop_review_app` job is **required** to have the following keywords defined: @@ -2640,10 +2613,12 @@ The `stop_review_app` job is **required** to have the following keywords defined - `environment:action` Additionally, both jobs should have matching [`rules`](../yaml/README.md#onlyexcept-basic) -or [`only/except`](../yaml/README.md#onlyexcept-basic) configuration. In the example -above, if the configuration is not identical, the `stop_review_app` job might not be -included in all pipelines that include the `review_app` job, and it will not be -possible to trigger the `action: stop` to stop the environment automatically. +or [`only/except`](../yaml/README.md#onlyexcept-basic) configuration. + +In the example above, if the configuration is not identical: + +- The `stop_review_app` job might not be included in all pipelines that include the `review_app` job. +- It is not possible to trigger the `action: stop` to stop the environment automatically. #### `environment:auto_stop_in` @@ -2687,7 +2662,7 @@ deploy: namespace: production ``` -This will set up the `deploy` job to deploy to the `production` +This configuration sets up the `deploy` job to deploy to the `production` environment, using the `production` [Kubernetes namespace](https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/). @@ -2719,11 +2694,11 @@ deploy as review app: url: https://$CI_ENVIRONMENT_SLUG.example.com/ ``` -The `deploy as review app` job will be marked as deployment to dynamically +The `deploy as review app` job is marked as deployment to dynamically create the `review/$CI_COMMIT_REF_NAME` environment, where `$CI_COMMIT_REF_NAME` is an [environment variable](../variables/README.md) set by the runner. The `$CI_ENVIRONMENT_SLUG` variable is based on the environment name, but suitable -for inclusion in URLs. In this case, if the `deploy as review app` job was run +for inclusion in URLs. In this case, if the `deploy as review app` job is run in a branch named `pow`, this environment would be accessible with an URL like `https://review-pow.example.com/`. @@ -2742,16 +2717,15 @@ as Review Apps. You can see a simple example using Review Apps at > by default. > - From GitLab 9.2, caches are restored before [artifacts](#artifacts). -TIP: **Learn more:** -Read how caching works and find out some good practices in the -[caching dependencies documentation](../caching/index.md). - `cache` is used to specify a list of files and directories that should be cached between jobs. You can only use paths that are within the local working copy. If `cache` is defined outside the scope of jobs, it means it's set -globally and all jobs will use that definition. +globally and all jobs use that definition. + +Read how caching works and find out some good practices in the +[caching dependencies documentation](../caching/index.md). #### `cache:paths` @@ -2777,7 +2751,7 @@ rspec: ``` Locally defined cache overrides globally defined options. The following `rspec` -job will cache only `binaries/`: +job caches only `binaries/`: ```yaml cache: @@ -2792,20 +2766,20 @@ rspec: - binaries/ ``` -Note that since cache is shared between jobs, if you're using different -paths for different jobs, you should also set a different **cache:key** -otherwise cache content can be overwritten. +The cache is shared between jobs, so if you're using different +paths for different jobs, you should also set a different `cache:key`. +Otherwise cache content can be overwritten. #### `cache:key` > Introduced in GitLab Runner v1.0.0. -Since the cache is shared between jobs, if you're using different -paths for different jobs, you should also set a different `cache:key` -otherwise cache content can be overwritten. +The cache is shared between jobs, so if you're using different +paths for different jobs, you should also set a different `cache:key`. +Otherwise cache content can be overwritten. -The `key` parameter defines the affinity of caching between jobs, -to have a single cache for all jobs, cache per-job, cache per-branch +The `key` parameter defines the affinity of caching between jobs. +You can have a single cache for all jobs, cache per-job, cache per-branch, or any other way that fits your workflow. This way, you can fine tune caching, including caching data between different jobs or even different branches. @@ -2814,10 +2788,6 @@ The `cache:key` variable can use any of the set, is just literal `default`, which means everything is shared between pipelines and jobs by default, starting from GitLab 9.0. -NOTE: **Note:** -The `cache:key` variable can't contain the `/` character, or the equivalent -URI-encoded `%2F`; a value made only of dots (`.`, `%2E`) is also forbidden. - For example, to enable per-branch caching: ```yaml @@ -2837,18 +2807,23 @@ cache: - binaries/ ``` +The `cache:key` variable can't contain the `/` character, or the equivalent +URI-encoded `%2F`. A value made only of dots (`.`, `%2E`) is also forbidden. + +You can specify a [fallback cache key](#fallback-cache-key) to use if the specified `cache:key` is not found. + ##### `cache:key:files` > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/18986) in GitLab v12.5. The `cache:key:files` keyword extends the `cache:key` functionality by making it easier -to reuse some caches, and rebuild them less often, which will speed up subsequent pipeline +to reuse some caches, and rebuild them less often, which speeds up subsequent pipeline runs. -When you include `cache:key:files`, you must also list the project files that will be used to generate the key, up to a maximum of two files. -The cache `key` will be a SHA checksum computed from the most recent commits (up to two, if two files are listed) +When you include `cache:key:files`, you must also list the project files that are used to generate the key, up to a maximum of two files. +The cache `key` is a SHA checksum computed from the most recent commits (up to two, if two files are listed) that changed the given files. If neither file was changed in any commits, -the fallback key will be `default`. +the fallback key is `default`. ```yaml cache: @@ -2864,7 +2839,7 @@ cache: In this example we're creating a cache for Ruby and Node.js dependencies that is tied to current versions of the `Gemfile.lock` and `package.json` files. Whenever one of these files changes, a new cache key is computed and a new cache is created. Any future -job runs using the same `Gemfile.lock` and `package.json` with `cache:key:files` will +job runs that use the same `Gemfile.lock` and `package.json` with `cache:key:files` use the new cache, instead of rebuilding the dependencies. ##### `cache:key:prefix` @@ -2897,11 +2872,11 @@ rspec: - bundle exec rspec ``` -For example, adding a `prefix` of `$CI_JOB_NAME` will -cause the key to look like: `rspec-feef9576d21ee9b6a32e30c5c79d0a0ceb68d1e5` and +For example, adding a `prefix` of `$CI_JOB_NAME` +causes the key to look like: `rspec-feef9576d21ee9b6a32e30c5c79d0a0ceb68d1e5` and the job cache is shared across different branches. If a branch changes -`Gemfile.lock`, that branch will have a new SHA checksum for `cache:key:files`. A new cache key -will be generated, and a new cache will be created for that key. +`Gemfile.lock`, that branch has a new SHA checksum for `cache:key:files`. A new cache key +is generated, and a new cache is created for that key. If `Gemfile.lock` is not found, the prefix is added to `default`, so the key in the example would be `rspec-default`. @@ -2928,6 +2903,28 @@ rspec: - binaries/ ``` +#### `cache:when` + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/18969) in GitLab 13.5 and GitLab Runner v13.5.0. + +`cache:when` defines when to save the cache, based on the status of the job. You can +set `cache:when` to: + +- `on_success` - save the cache only when the job succeeds. This is the default. +- `on_failure` - save the cache only when the job fails. +- `always` - save the cache regardless of the job status. + +For example, to store a cache whether or not the job fails or succeeds: + +```yaml +rspec: + script: rspec + cache: + paths: + - rspec/ + when: 'always' +``` + #### `cache:policy` > Introduced in GitLab 9.4. @@ -2967,13 +2964,13 @@ rspec: - bundle exec rspec ... ``` -This helps to speed up job execution and reduce load on the cache server, -especially when you have a large number of cache-using jobs executing in +This helps to speed up job execution and reduce load on the cache server. +It is especially helpful when you have a large number of cache-using jobs executing in parallel. -Additionally, if you have a job that unconditionally recreates the cache without -reference to its previous contents, you can use `policy: push` in that job to -skip the download step. +If you have a job that unconditionally recreates the cache without +referring to its previous contents, you can skip the download step. +To do so, add `policy: push` to the job. ### `artifacts` @@ -2986,8 +2983,8 @@ skip the download step. `artifacts` is used to specify a list of files and directories that are attached to the job when it [succeeds, fails, or always](#artifactswhen). -The artifacts will be sent to GitLab after the job finishes and will -be available for download in the GitLab UI provided that the size is not +The artifacts are sent to GitLab after the job finishes. They are +available for download in the GitLab UI if the size is not larger than the [maximum artifact size](../../user/gitlab_com/index.md#gitlab-cicd). [Read more about artifacts](../pipelines/job_artifacts.md). @@ -3003,7 +3000,7 @@ patterns and: - In GitLab Runner 12.10 and earlier, [`filepath.Match`](https://pkg.go.dev/path/filepath/#Match). -To restrict which jobs a specific job will fetch artifacts from, see [dependencies](#dependencies). +To restrict which jobs a specific job fetches artifacts from, see [dependencies](#dependencies). Send all files in `binaries` and `.config`: @@ -3026,7 +3023,7 @@ job: You may want to create artifacts only for tagged releases to avoid filling the build server storage with temporary build artifacts. -Create artifacts only for tags (`default-job` won't create artifacts): +Create artifacts only for tags (`default-job` doesn't create artifacts): ```yaml default-job: @@ -3098,10 +3095,10 @@ test: paths: ['file.txt'] ``` -With this configuration, GitLab will add a link **artifact 1** to the relevant merge request +With this configuration, GitLab adds a link **artifact 1** to the relevant merge request that points to `file1.txt`. -An example that will match an entire directory: +An example that matches an entire directory: ```yaml test: @@ -3116,12 +3113,12 @@ Note the following: - Artifacts do not display in the merge request UI when using variables to define the `artifacts:paths`. - A maximum of 10 job artifacts per merge request can be exposed. - Glob patterns are unsupported. -- If a directory is specified, the link will be to the job [artifacts browser](../pipelines/job_artifacts.md#browsing-artifacts) if there is more than +- If a directory is specified, the link is to the job [artifacts browser](../pipelines/job_artifacts.md#browsing-artifacts) if there is more than one file in the directory. - For exposed single file artifacts with `.html`, `.htm`, `.txt`, `.json`, `.xml`, and `.log` extensions, if [GitLab Pages](../../administration/pages/index.md) is: - - Enabled, GitLab will automatically render the artifact. - - Not enabled, you will see the file in the artifacts browser. + - Enabled, GitLab automatically renders the artifact. + - Not enabled, the file is displayed in the artifacts browser. #### `artifacts:name` @@ -3133,11 +3130,6 @@ useful when you want to download the archive from GitLab. The `artifacts:name` variable can make use of any of the [predefined variables](../variables/README.md). The default name is `artifacts`, which becomes `artifacts.zip` when you download it. -NOTE: **Note:** -If your branch-name contains forward slashes -(for example `feature/my-feature`) it's advised to use `$CI_COMMIT_REF_SLUG` -instead of `$CI_COMMIT_REF_NAME` for proper naming of the artifact. - To create an archive with a name of the current job: ```yaml @@ -3159,6 +3151,10 @@ job: - binaries/ ``` +If your branch-name contains forward slashes +(for example `feature/my-feature`) it's advised to use `$CI_COMMIT_REF_SLUG` +instead of `$CI_COMMIT_REF_NAME` for proper naming of the artifact. + To create an archive with a name of the current job and the current branch or tag including only the binaries directory: @@ -3207,10 +3203,8 @@ job: #### `artifacts:untracked` `artifacts:untracked` is used to add all Git untracked files as artifacts (along -to the paths defined in `artifacts:paths`). - -NOTE: **Note:** -`artifacts:untracked` ignores configuration in the repository's `.gitignore` file. +to the paths defined in `artifacts:paths`). `artifacts:untracked` ignores configuration +in the repository's `.gitignore` file. Send all Git untracked files: @@ -3250,7 +3244,7 @@ failure. 1. `on_failure` - upload artifacts only when the job fails. 1. `always` - upload artifacts regardless of the job status. -To upload artifacts only when job fails: +For example, to upload artifacts only when a job fails: ```yaml job: @@ -3300,7 +3294,6 @@ job: expire_in: 1 week ``` -NOTE: **Note:** The latest artifacts for refs are locked against deletion, and kept regardless of the expiry time. [Introduced in](https://gitlab.com/gitlab-org/gitlab/-/issues/16267) GitLab 13.0 behind a disabled feature flag, and [made the default behavior](https://gitlab.com/gitlab-org/gitlab/-/issues/229936) @@ -3335,24 +3328,27 @@ These are the available report types: > Introduced in GitLab 8.6 and GitLab Runner v1.1.1. -By default, all [`artifacts`](#artifacts) from all previous [stages](#stages) -are passed, but you can use the `dependencies` parameter to define a limited -list of jobs (or no jobs) to fetch artifacts from. +By default, all [`artifacts`](#artifacts) from previous [stages](#stages) +are passed to each job. However, you can use the `dependencies` parameter to +define a limited list of jobs to fetch artifacts from. You can also set a job to download no artifacts at all. To use this feature, define `dependencies` in context of the job and pass a list of all previous jobs the artifacts should be downloaded from. -You can only define jobs from stages that are executed before the current one. -An error will be shown if you define jobs from the current stage or next ones. -Defining an empty array will skip downloading any artifacts for that job. -The status of the previous job is not considered when using `dependencies`, so -if it failed or it's a manual job that was not run, no error occurs. -In the following example, we define two jobs with artifacts, `build:osx` and +You can define jobs from stages that were executed before the current one. +An error occurs if you define jobs from the current or an upcoming stage. + +To prevent a job from downloading artifacts, define an empty array. + +When you use `dependencies`, the status of the previous job is not considered. +If a job fails or it's a manual job that was not run, no error occurs. + +The following example defines two jobs with artifacts: `build:osx` and `build:linux`. When the `test:osx` is executed, the artifacts from `build:osx` -will be downloaded and extracted in the context of the build. The same happens +are downloaded and extracted in the context of the build. The same happens for `test:linux` and artifacts from `build:linux`. -The job `deploy` will download artifacts from all previous jobs because of +The job `deploy` downloads artifacts from all previous jobs because of the [stage](#stages) precedence: ```yaml @@ -3387,16 +3383,15 @@ deploy: script: make deploy ``` -##### When a dependent job will fail +##### When a dependent job fails > Introduced in GitLab 10.3. If the artifacts of the job that is set as a dependency have been [expired](#artifactsexpire_in) or [erased](../pipelines/job_artifacts.md#erasing-artifacts), then -the dependent job will fail. +the dependent job fails. -NOTE: **Note:** You can ask your administrator to [flip this switch](../../administration/job_artifacts.md#validation-for-dependencies) and bring back the old behavior. @@ -3409,7 +3404,7 @@ Use `coverage` to configure how code coverage is extracted from the job output. Regular expressions are the only valid kind of value expected here. So, using -surrounding `/` is mandatory in order to consistently and explicitly represent +surrounding `/` is mandatory to consistently and explicitly represent a regular expression string. You must escape special characters if you want to match them literally. @@ -3426,17 +3421,17 @@ job1: > - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/3442) in GitLab 9.5. > - [Behavior expanded](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/3515) in GitLab 11.5 to control which failures to retry on. -Use `retry` to configure how many times a job is going to be retried in +Use `retry` to configure how many times a job is retried in case of a failure. -When a job fails and has `retry` configured, it's going to be processed again -up to the amount of times specified by the `retry` keyword. +When a job fails, the job is processed again, +until the limit specified by the `retry` keyword is reached. -If `retry` is set to 2, and a job succeeds in a second run (first retry), it won't be retried -again. `retry` value has to be a positive integer, equal or larger than 0, but -lower or equal to 2 (two retries maximum, three runs in total). +If `retry` is set to `2`, and a job succeeds in a second run (first retry), it is not retried. +The `retry` value must be a positive integer, from `0` to `2` +(two retries maximum, three runs in total). -A simple example to retry in all failure cases: +This example retries all failure cases: ```yaml test: @@ -3444,7 +3439,7 @@ test: retry: 2 ``` -By default, a job will be retried on all failure cases. To have a better control +By default, a job is retried on all failure cases. To have better control over which failures to retry, `retry` can be a hash with the following keys: - `max`: The maximum number of retries. @@ -3460,8 +3455,8 @@ test: when: runner_system_failure ``` -If there is another failure, other than a runner system failure, the job will -not be retried. +If there is another failure, other than a runner system failure, the job +is not retried. To retry on multiple failure cases, `when` can also be an array of failures: @@ -3478,10 +3473,10 @@ test: Possible values for `when` are: <!-- - Please make sure to update `RETRY_WHEN_IN_DOCUMENTATION` array in - `spec/lib/gitlab/ci/config/entry/retry_spec.rb` if you change any of - the documented values below. The test there makes sure that all documented - values are really valid as a configuration option and therefore should always + If you change any of the values below, make sure to update the `RETRY_WHEN_IN_DOCUMENTATION` + array in `spec/lib/gitlab/ci/config/entry/retry_spec.rb`. + The test there makes sure that all documented + values are valid as a configuration option and therefore should always stay in sync with this documentation. --> @@ -3543,7 +3538,6 @@ test: parallel: 5 ``` -TIP: **Tip:** Parallelize tests suites across parallel jobs. Different languages have different tools to facilitate this. @@ -3578,6 +3572,12 @@ job split into three separate jobs. Use `matrix:` to configure different variables for jobs that are running in parallel. There can be from 2 to 50 jobs. +[In GitLab 13.5](https://gitlab.com/gitlab-org/gitlab/-/issues/26362) and later, +you can have one-dimensional matrices with a single job. +The ability to have one-dimensional matrices is [deployed behind a feature flag](../../user/feature_flags.md), +enabled by default. It's enabled on GitLab.com. For self-managed GitLab instances, +administrators can opt to disable it by [disabling the `one_dimensional_matrix:` feature flag](../../administration/feature_flags.md). **(CORE ONLY)** + Every job gets the same `CI_NODE_TOTAL` [environment variable](../variables/README.md#predefined-environment-variables) value, and a unique `CI_NODE_INDEX` value. ```yaml @@ -3623,19 +3623,25 @@ Job naming style [was improved](https://gitlab.com/gitlab-org/gitlab/-/issues/23 Use `trigger` to define a downstream pipeline trigger. When GitLab starts a job created with a `trigger` definition, a downstream pipeline is created. +Jobs with `trigger` can only use a [limited set of keywords](../multi_project_pipelines.md#limitations). +For example, you can't run commands with [`script`](#script), [`before_script`](#before_script-and-after_script), +or [`after_script`](#before_script-and-after_script). + You can use this keyword to create two different types of downstream pipelines: - [Multi-project pipelines](../multi_project_pipelines.md#creating-multi-project-pipelines-from-gitlab-ciyml) - [Child pipelines](../parent_child_pipelines.md) -[Since GitLab 13.2](https://gitlab.com/gitlab-org/gitlab/-/issues/197140/), you can -see which job triggered a downstream pipeline by hovering your mouse cursor over -the downstream pipeline job in the [pipeline graph](../pipelines/index.md#visualize-pipelines). +[In GitLab 13.2](https://gitlab.com/gitlab-org/gitlab/-/issues/197140/) and later, you can +view which job triggered a downstream pipeline. In the [pipeline graph](../pipelines/index.md#visualize-pipelines), +hover over the downstream pipeline job. -NOTE: **Note:** -Using a `trigger` with `when:manual` together results in the error `jobs:#{job-name} -when should be on_success, on_failure or always`, because `when:manual` prevents -triggers being used. +In [GitLab 13.5](https://gitlab.com/gitlab-org/gitlab/-/issues/201938) and later, you +can use [`when:manual`](#whenmanual) in the same job as `trigger`. In GitLab 13.4 and +earlier, using them together causes the error `jobs:#{job-name} when should be on_success, on_failure or always`. +It is deployed behind the `:ci_manual_bridges` [feature flag](../../user/feature_flags.md), which is **enabled by default**. +[GitLab administrators with access to the Rails console](../../administration/feature_flags.md) +can opt to disable it. #### Simple `trigger` syntax for multi-project pipelines @@ -3654,7 +3660,7 @@ staging: #### Complex `trigger` syntax for multi-project pipelines -It's possible to configure a branch name that GitLab will use to create +You can configure a branch name that GitLab uses to create a downstream pipeline with: ```yaml @@ -3669,7 +3675,7 @@ staging: branch: stable ``` -It's possible to mirror the status from a triggered pipeline: +To mirror the status from a triggered pipeline: ```yaml trigger_job: @@ -3678,7 +3684,7 @@ trigger_job: strategy: depend ``` -It's possible to mirror the status from an upstream pipeline: +To mirror the status from an upstream pipeline: ```yaml upstream_bridge: @@ -3736,14 +3742,30 @@ child-pipeline: The `generated-config.yml` is extracted from the artifacts and used as the configuration for triggering the child pipeline. +##### Trigger child pipeline with files from another project + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/205157) in GitLab 13.5. + +To trigger child pipelines with files from another private project under the same +GitLab instance, use [`include:file`](#includefile): + +```yaml +child-pipeline: + trigger: + include: + - project: 'my-group/my-pipeline-library' + ref: 'master' + file: '/path/to/child-pipeline.yml' +``` + #### Linking pipelines with `trigger:strategy` By default, the `trigger` job completes with the `success` status as soon as the downstream pipeline is created. To force the `trigger` job to wait for the downstream (multi-project or child) pipeline to complete, use -`strategy: depend`. This will make the trigger job wait with a "running" status until the triggered -pipeline completes. At that point, the `trigger` job will complete and display the same status as +`strategy: depend`. This setting makes the trigger job wait with a "running" status until the triggered +pipeline completes. At that point, the `trigger` job completes and displays the same status as the downstream job. ```yaml @@ -3753,16 +3775,16 @@ trigger_job: strategy: depend ``` -This can help keep your pipeline execution linear. In the example above, jobs from -subsequent stages will wait for the triggered pipeline to successfully complete before -starting, at the cost of reduced parallelization. +This setting can help keep your pipeline execution linear. In the example above, jobs from +subsequent stages wait for the triggered pipeline to successfully complete before +starting, which reduces parallelization. #### Trigger a pipeline by API call -Triggers can be used to force a rebuild of a specific branch, tag or commit, -with an API call when a pipeline gets created using a trigger token. +To force a rebuild of a specific branch, tag, or commit, you can use an API call +with a trigger token. -Not to be confused with the [`trigger`](#trigger) parameter. +The trigger token is different than the [`trigger`](#trigger) parameter. [Read more in the triggers documentation.](../triggers/README.md) @@ -3771,19 +3793,18 @@ Not to be confused with the [`trigger`](#trigger) parameter. > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/32022) in GitLab 12.3. `interruptible` is used to indicate that a job should be canceled if made redundant by a newer pipeline run. Defaults to `false`. -This value will only be used if the [automatic cancellation of redundant pipelines feature](../pipelines/settings.md#auto-cancel-pending-pipelines) +This value is used only if the [automatic cancellation of redundant pipelines feature](../pipelines/settings.md#auto-cancel-pending-pipelines) is enabled. -When enabled, a pipeline on the same branch will be canceled when: +When enabled, a pipeline on the same branch is canceled when: - It's made redundant by a newer pipeline run. - Either all jobs are set as interruptible, or any uninterruptible jobs haven't started. -Pending jobs are always considered interruptible. - -TIP: **Tip:** Set jobs as interruptible that can be safely canceled once started (for instance, a build job). +Pending jobs are always considered interruptible. + Here is a simple example: ```yaml @@ -3806,17 +3827,16 @@ step-2: step-3: stage: stage3 script: - - echo "Because step-2 can not be canceled, this step will never be canceled, even though set as interruptible." + - echo "Because step-2 can not be canceled, this step can never be canceled, even though it's set as interruptible." interruptible: true ``` -In the example above, a new pipeline run will cause an existing running pipeline to be: +In the example above, a new pipeline run causes an existing running pipeline to be: - Canceled, if only `step-1` is running or pending. - Not canceled, once `step-2` starts running. -NOTE: **Note:** -Once an uninterruptible job is running, the pipeline will never be canceled, regardless of the final job's state. +When an uninterruptible job is running, the pipeline can never be canceled, regardless of the final job's state. ### `resource_group` @@ -3826,12 +3846,13 @@ Sometimes running multiple jobs or pipelines at the same time in an environment can lead to errors during the deployment. To avoid these errors, the `resource_group` attribute can be used to ensure that -the runner doesn't run certain jobs simultaneously. +the runner doesn't run certain jobs simultaneously. Resource groups behave similar +to semaphores in other programming languages. When the `resource_group` key is defined for a job in `.gitlab-ci.yml`, job executions are mutually exclusive across different pipelines for the same project. If multiple jobs belonging to the same resource group are enqueued simultaneously, -only one of the jobs is picked by the runner, and the other jobs wait until the +only one of the jobs is picked by the runner. The other jobs wait until the `resource_group` is free. Here is a simple example: @@ -3842,17 +3863,14 @@ deploy-to-production: resource_group: production ``` -In this case, if a `deploy-to-production` job is running in a pipeline, and a new -`deploy-to-production` job is created in a different pipeline, it won't run until -the currently running/pending `deploy-to-production` job is finished. As a result, -you can ensure that concurrent deployments will never happen to the production environment. +In this case, two `deploy-to-production` jobs in two separate pipelines can never run at the same time. As a result, +you can ensure that concurrent deployments never happen to the production environment. There can be multiple `resource_group`s defined per environment. A good use case for this -is when deploying to physical devices. You may have more than one physical device, and each -one can be deployed to, but there can be only one deployment per device at any given time. +is when deploying to physical devices. You may have multiple physical devices that +can be deployed to, but there can be only one deployment per device at any given time. -NOTE: **Note:** -This key can only contain letters, digits, `-`, `_`, `/`, `$`, `{`, `}`, `.`, and spaces. +The `resource_group` value can only contain letters, digits, `-`, `_`, `/`, `$`, `{`, `}`, `.`, and spaces. It can't start or end with `/`. For more information, see [Deployments Safety](../environments/deployment_safety.md). @@ -3962,7 +3980,11 @@ The title of each milestone the release is associated with. #### `release:released_at` The date and time when the release is ready. Defaults to the current date and time if not -defined. Expected in ISO 8601 format (2019-03-15T08:00:00Z). +defined. Should be enclosed in quotes and expressed in ISO 8601 format. + +```json +released_at: '2021-03-15T08:00:00Z' +``` #### Complete example for `release` @@ -3990,7 +4012,7 @@ tags. These options cannot be used together, so choose one: - 'm1' - 'm2' - 'm3' - released_at: '2020-07-15T08:00:00Z' # Optional, will auto generate if not defined, or can use a variable. + released_at: '2020-07-15T08:00:00Z' # Optional, is auto generated if not defined, or can use a variable. ``` - To create a release automatically when commits are pushed or merged to the default branch, @@ -4036,9 +4058,14 @@ tags. These options cannot be used together, so choose one: - 'm1' - 'm2' - 'm3' - released_at: '2020-07-15T08:00:00Z' # Optional, will auto generate if not defined, or can use a variable. + released_at: '2020-07-15T08:00:00Z' # Optional, is auto generated if not defined, or can use a variable. ``` +#### Release assets as Generic packages + +You can use [Generic packages](../../user/packages/generic_packages/) to host your release assets. +For a complete example of how to do this, see the [example in the repository](https://gitlab.com/gitlab-org/release-cli/-/tree/master/docs/examples/release-assets-as-generic-package/). + #### `releaser-cli` command line The entries under the `:release` node are transformed into a `bash` command line and sent @@ -4056,14 +4083,16 @@ release-cli create --name "Release $CI_COMMIT_SHA" --description "Created using > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/33014) in GitLab 13.4. `secrets` indicates the [CI Secrets](../secrets/index.md) this job needs. It should be a hash, -and the keys should be the names of the environment variables the job needs to access the secrets. +and the keys should be the names of the environment variables that are made available to the job. +The value of each secret is saved in a temporary file. This file's path is stored in these +environment variables. #### `secrets:vault` **(PREMIUM)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/28321) in GitLab 13.4. `vault` keyword specifies secrets provided by [Hashicorp's Vault](https://www.vaultproject.io/). -This syntax has multiple forms. The shortest form asssumes the use of the +This syntax has multiple forms. The shortest form assumes the use of the [KV-V2](https://www.vaultproject.io/docs/secrets/kv/kv-v2) secrets engine, mounted at the default path `kv-v2`. The last part of the secret's path is the field to fetch the value for: @@ -4072,7 +4101,7 @@ field to fetch the value for: job: secrets: DATABASE_PASSWORD: - vault: production/db/password # translates to secret `kv-v2/data/production/db`, field `password` + vault: production/db/password # translates to secret `kv-v2/data/production/db`, field `password` ``` You can specify a custom secrets engine path by adding a suffix starting with `@`: @@ -4081,7 +4110,7 @@ You can specify a custom secrets engine path by adding a suffix starting with `@ job: secrets: DATABASE_PASSWORD: - vault: production/db/password@ops # translates to secret `ops/data/production/db`, field `password` + vault: production/db/password@ops # translates to secret `ops/data/production/db`, field `password` ``` In the detailed form of the syntax, you can specify all details explicitly: @@ -4131,34 +4160,40 @@ Read more on [GitLab Pages user documentation](../../user/project/pages/index.md > Introduced in GitLab Runner v0.5.0. -NOTE: **Note:** -Integers (as well as strings) are legal both for variable's name and value. -Floats are not legal and can't be used. +Variables are configurable values that are passed to jobs. They can be set +globally and per-job. + +There are two types of variables. + +- [Custom variables](../variables/README.md#custom-environment-variables): + You can define their values in the `.gitlab-ci.yml` file, in the GitLab UI, + or by using the API. +- [Predefined variables](../variables/predefined_variables.md): + These values are set by the runner itself. + One example is `CI_COMMIT_REF_NAME`, which is the branch or tag the project is built for. -Variables are configurable values in `.gitlab-ci.yml` that are passed to jobs. -They can be set globally and per-job. -When you use the `variables` keyword in jobs, it overrides the global -YAML variables and predefined ones of the same name. +After you define a variable, you can use it in all executed commands and scripts. -Variables are stored in the Git repository and are meant for non-sensitive -project configuration, for example: +Variables are meant for non-sensitive project configuration, for example: ```yaml variables: DATABASE_URL: "postgres://postgres@postgres/my_database" ``` -You can use these variables later in all executed commands and scripts. -The YAML-defined variables are also set to all created service containers, -so that you can fine tune them. +You can use integers and strings for the variable's name and value. +You cannot use floats. + +If you define a variable at the top level of the `gitlab-ci.yml` file, it is global, +meaning it applies to all jobs. + +If you define a variable within a job, it's available to that job only. -Except for the user-defined variables, there are also variables [set up by the -runner itself](../variables/README.md#predefined-environment-variables). -One example would be `CI_COMMIT_REF_NAME`, which has the value of -the branch or tag name the project is built for. Apart from the variables -you can set in `.gitlab-ci.yml`, there are also environment -[variables](../variables/README.md#gitlab-cicd-environment-variables), -which can be set in the GitLab UI. +If a variable of the same name is defined globally and for a specific job, the +[job-specific variable is used](../variables/README.md#priority-of-environment-variables). + +All YAML-defined variables are also set to any linked +[service containers](../docker/using_docker_images.md#what-is-a-service). [YAML anchors for variables](#yaml-anchors-for-variables) are available. @@ -4169,12 +4204,9 @@ Learn more about [variables and their priority](../variables/README.md). > - Introduced in GitLab 8.9 as an experimental feature. > - `GIT_STRATEGY=none` requires GitLab Runner v1.7+. -CAUTION: **Caution:** -May change or be removed completely in future releases. - You can set the `GIT_STRATEGY` used for getting recent application code, either globally or per-job in the [`variables`](#variables) section. If left -unspecified, the default from project settings will be used. +unspecified, the default from the project settings is used. There are three possible values: `clone`, `fetch`, and `none`. @@ -4195,10 +4227,11 @@ variables: GIT_STRATEGY: fetch ``` -`none` also re-uses the local working copy, but skips all Git operations -(including GitLab Runner's pre-clone script, if present). It's mostly useful -for jobs that operate exclusively on artifacts (for example, `deploy`). Git repository -data may be present, but it's certain to be out of date, so you should only +`none` also re-uses the local working copy. However, it skips all Git operations, +including GitLab Runner's pre-clone script, if present. + +It's useful for jobs that operate exclusively on artifacts, like a deployment job. +Git repository data may be present, but it's likely out-of-date. You should only rely on files brought into the local working copy from cache or artifacts. ```yaml @@ -4222,10 +4255,10 @@ globally or per-job in the [`variables`](#variables) section. There are three possible values: `none`, `normal`, and `recursive`: -- `none` means that submodules won't be included when fetching the project +- `none` means that submodules are not included when fetching the project code. This is the default, which matches the pre-v1.10 behavior. -- `normal` means that only the top-level submodules will be included. It's +- `normal` means that only the top-level submodules are included. It's equivalent to: ```shell @@ -4234,7 +4267,7 @@ There are three possible values: `none`, `normal`, and `recursive`: ``` - `recursive` means that all submodules (including submodules of submodules) - will be included. This feature needs Git v1.8.1 and later. When using a + are included. This feature needs Git v1.8.1 and later. When using a GitLab Runner with an executor not based on Docker, make sure the Git version meets that requirement. It's equivalent to: @@ -4243,7 +4276,7 @@ There are three possible values: `none`, `normal`, and `recursive`: git submodule update --init --recursive ``` -Note that for this feature to work correctly, the submodules must be configured +For this feature to work correctly, the submodules must be configured (in `.gitmodules`) with either: - the HTTP(S) URL of a publicly-accessible repository, or @@ -4259,15 +4292,15 @@ The `GIT_CHECKOUT` variable can be used when the `GIT_STRATEGY` is set to either specified, it defaults to true. You can set them globally or per-job in the [`variables`](#variables) section. -If set to `false`, the runner will: +If set to `false`, the runner: -- when doing `fetch` - update the repository and leave working copy on +- when doing `fetch` - updates the repository and leaves the working copy on the current revision, -- when doing `clone` - clone the repository and leave working copy on the +- when doing `clone` - clones the repository and leaves the working copy on the default branch. -Having this setting set to `true` will mean that for both `clone` and `fetch` -strategies the runner will checkout the working copy to a revision related +If `GIT_CHECKOUT` is set to `true`, both `clone` and `fetch` work the same way. +The runner checks out the working copy of a revision related to the CI pipeline: ```yaml @@ -4313,7 +4346,7 @@ script: The `GIT_FETCH_EXTRA_FLAGS` variable is used to control the behavior of `git fetch`. You can set it globally or per-job in the [`variables`](#variables) section. -`GIT_FETCH_EXTRA_FLAGS` accepts all possible options of the [`git fetch`](https://git-scm.com/docs/git-fetch) command, but please note that `GIT_FETCH_EXTRA_FLAGS` flags will be appended after the default flags that can't be modified. +`GIT_FETCH_EXTRA_FLAGS` accepts all options of the [`git fetch`](https://git-scm.com/docs/git-fetch) command. However, `GIT_FETCH_EXTRA_FLAGS` flags are appended after the default flags that can't be modified. The default flags are: @@ -4335,7 +4368,7 @@ script: - ls -al cache/ ``` -The configuration above will result in `git fetch` being called this way: +The configuration above results in `git fetch` being called this way: ```shell git fetch origin $REFSPECS --depth 50 --prune @@ -4347,13 +4380,13 @@ Where `$REFSPECS` is a value provided to the runner internally by GitLab. > Introduced in GitLab, it requires GitLab Runner v1.9+. -You can set the number for attempts the running job will try to execute each -of the following stages: +You can set the number of attempts that the running job tries to execute +the following stages: | Variable | Description | |-----------------------------------|--------------------------------------------------------| | **ARTIFACT_DOWNLOAD_ATTEMPTS** | Number of attempts to download artifacts running a job | -| **EXECUTOR_JOB_SECTION_ATTEMPTS** | [Since GitLab 12.10](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/4450), the number of attempts to run a section in a job after a [`No Such Container`](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/4450) error ([Docker executor](https://docs.gitlab.com/runner/executors/docker.html) only). | +| **EXECUTOR_JOB_SECTION_ATTEMPTS** | [In GitLab 12.10](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/4450) and later, the number of attempts to run a section in a job after a [`No Such Container`](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/4450) error ([Docker executor](https://docs.gitlab.com/runner/executors/docker.html) only). | | **GET_SOURCES_ATTEMPTS** | Number of attempts to fetch sources running a job | | **RESTORE_CACHE_ATTEMPTS** | Number of attempts to restore the cache running a job | @@ -4368,27 +4401,52 @@ variables: You can set them globally or per-job in the [`variables`](#variables) section. +### Fallback cache key + +> [Introduced](https://gitlab.com/gitlab-org/gitlab-runner/-/merge_requests/1534) in GitLab Runner 13.4. + +You can use the `$CI_COMMIT_REF_SLUG` variable to specify your [`cache:key`](#cachekey). +For example, if your `$CI_COMMIT_REF_SLUG` is `test` you can set a job +to download cache that's tagged with `test`. + +If a cache with this tag is not found, you can use `CACHE_FALLBACK_KEY` to +specify a cache to use when none exists. + +For example: + +```yaml +variables: + CACHE_FALLBACK_KEY: fallback-key + +cache: + key: "$CI_COMMIT_REF_SLUG" + paths: + - binaries/ +``` + +In this example, if the `$CI_COMMIT_REF_SLUG` is not found, the job uses the key defined +by the `CACHE_FALLBACK_KEY` variable. + ### Shallow cloning > Introduced in GitLab 8.9 as an experimental feature. -NOTE: **Note:** -As of GitLab 12.0, newly created projects will automatically have a [default `git depth` value of `50`](../pipelines/settings.md#git-shallow-clone). - -You can specify the depth of fetching and cloning using `GIT_DEPTH`. This does a -shallow clone of the repository and can significantly speed up cloning for -repositories with a large number of commits or old, large binaries. The value is +You can specify the depth of fetching and cloning using `GIT_DEPTH`. +`GIT_DEPTH` does a shallow clone of the repository and can significantly speed up cloning. +It can be helpful for repositories with a large number of commits or old, large binaries. The value is passed to `git fetch` and `git clone`. -NOTE: **Note:** -If you use a depth of 1 and have a queue of jobs or retry +In GitLab 12.0 and later, newly-created projects automatically have a +[default `git depth` value of `50`](../pipelines/settings.md#git-shallow-clone). + +If you use a depth of `1` and have a queue of jobs or retry jobs, jobs may fail. -Since Git fetching and cloning is based on a ref, such as a branch name, runners -can't clone a specific commit SHA. If there are multiple jobs in the queue, or -you're retrying an old job, the commit to be tested needs to be within the +Git fetching and cloning is based on a ref, such as a branch name, so runners +can't clone a specific commit SHA. If multiple jobs are in the queue, or +you're retrying an old job, the commit to be tested must be within the Git history that is cloned. Setting too small a value for `GIT_DEPTH` can make -it impossible to run these old commits. You will see `unresolved reference` in +it impossible to run these old commits and `unresolved reference` is displayed in job logs. You should then reconsider changing `GIT_DEPTH` to a higher value. Jobs that rely on `git describe` may not work correctly when `GIT_DEPTH` is @@ -4407,11 +4465,6 @@ You can set it globally or per-job in the [`variables`](#variables) section. > [Introduced](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/2211) in GitLab Runner 11.10. -NOTE: **Note:** -This can only be used when `custom_build_dir` is enabled in the [runner's -configuration](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-runnerscustom_build_dir-section). -This is the default configuration for `docker` and `kubernetes` executor. - By default, GitLab Runner clones the repository in a unique subpath of the `$CI_BUILDS_DIR` directory. However, your project might require the code in a specific directory (Go projects, for example). In that case, you can specify @@ -4431,12 +4484,17 @@ The `GIT_CLONE_PATH` has to always be within `$CI_BUILDS_DIR`. The directory set is dependent on executor and configuration of [runners.builds_dir](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-runners-section) setting. +This can only be used when `custom_build_dir` is enabled in the +[runner's configuration](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-runnerscustom_build_dir-section). +This is the default configuration for the `docker` and `kubernetes` executors. + #### Handling concurrency -An executor using a concurrency greater than `1` might lead -to failures because multiple jobs might be working on the same directory if the `builds_dir` +An executor that uses a concurrency greater than `1` might lead +to failures. Multiple jobs might be working on the same directory if the `builds_dir` is shared between jobs. -GitLab Runner does not try to prevent this situation. It's up to the administrator + +The runner does not try to prevent this situation. It's up to the administrator and developers to comply with the requirements of runner configuration. To avoid this scenario, you can use a unique path within `$CI_BUILDS_DIR`, because runner @@ -4503,15 +4561,20 @@ need to be used to merge arrays. > Introduced in GitLab 8.6 and GitLab Runner v1.1.1. -YAML has a handy feature called 'anchors', which lets you easily duplicate -content across your document. Anchors can be used to duplicate/inherit -properties, and is a perfect example to be used with [hidden jobs](#hide-jobs) -to provide templates for your jobs. When there is duplicate keys, GitLab will -perform a reverse deep merge based on the keys. +YAML has a feature called 'anchors' that you can use to duplicate +content across your document. + +Use anchors to duplicate or inherit properties. Use anchors with [hidden jobs](#hide-jobs) +to provide templates for your jobs. When there are duplicate keys, GitLab +performs a reverse deep merge based on the keys. + +You can't use YAML anchors across multiple files when leveraging the [`include`](#include) +feature. Anchors are only valid within the file they were defined in. Instead +of using YAML anchors, you can use the [`extends` keyword](#extends). -The following example uses anchors and map merging. It will create two jobs, -`test1` and `test2`, that will inherit the parameters of `.job_template`, each -having their own custom `script` defined: +The following example uses anchors and map merging. It creates two jobs, +`test1` and `test2`, that inherit the parameters of `.job_template`, each +with their own custom `script` defined: ```yaml .job_template: &job_definition # Hidden key that defines an anchor named 'job_definition' @@ -4559,9 +4622,9 @@ test2: - test2 project ``` -Let's see another one example. This time we will use anchors to define two sets -of services. This will create two jobs, `test:postgres` and `test:mysql`, that -will share the `script` directive defined in `.job_template`, and the `services` +Let's see another example. This time we use anchors to define two sets +of services. This configuration creates two jobs, `test:postgres` and `test:mysql`, that +share the `script` directive defined in `.job_template`, and the `services` directive defined in `.postgres_services` and `.mysql_services` respectively: ```yaml @@ -4630,15 +4693,8 @@ test:mysql: - dev ``` -You can see that the hidden jobs are conveniently used as templates. - -NOTE: **Note:** -Note that `tags: [dev]` has been overwritten by `tags: [postgres]`. - -NOTE: **Note:** -You can't use YAML anchors across multiple files when leveraging the [`include`](#include) -feature. Anchors are only valid within the file they were defined in. Instead -of using YAML anchors, you can use the [`extends` keyword](#extends). +You can see that the hidden jobs are conveniently used as templates, and +`tags: [dev]` has been overwritten by `tags: [postgres]`. #### YAML anchors for `before_script` and `after_script` @@ -4692,7 +4748,7 @@ job_name: of variables across multiple jobs. It can also enable more flexibility when a job requires a specific `variables` block that would otherwise override the global variables. -In the example below, we will override the `GIT_STRATEGY` variable without affecting +In the example below, we override the `GIT_STRATEGY` variable without affecting the use of the `SAMPLE_VARIABLE` variable: ```yaml @@ -4701,7 +4757,7 @@ variables: &global-variables SAMPLE_VARIABLE: sample_variable_value ANOTHER_SAMPLE_VARIABLE: another_sample_variable_value -# a job that needs to set the GIT_STRATEGY variable, yet depend on global variables +# a job that must set the GIT_STRATEGY variable, yet depend on global variables job_no_git_strategy: stage: cleanup variables: @@ -4723,8 +4779,8 @@ lines where the job is defined: # - run test ``` -You can instead start its name with a dot (`.`) and it won't be processed by -GitLab CI/CD. In the following example, `.hidden_job` will be ignored: +Instead, you can start its name with a dot (`.`) and it is not processed by +GitLab CI/CD. In the following example, `.hidden_job` is ignored: ```yaml .hidden_job: @@ -4739,18 +4795,18 @@ into templates. ## Skip Pipeline If your commit message contains `[ci skip]` or `[skip ci]`, using any -capitalization, the commit will be created but the pipeline will be skipped. +capitalization, the commit is created but the pipeline is skipped. Alternatively, one can pass the `ci.skip` [Git push option](../../user/project/push_options.md#push-options-for-gitlab-cicd) if using Git 2.10 or newer. ## Processing Git pushes -GitLab will create at most 4 branch and tag pipelines when -pushing multiple changes in single `git push` invocation. +GitLab creates at most four branch and tag pipelines when +pushing multiple changes in a single `git push` invocation. -This limitation does not affect any of the updated Merge Request pipelines. -All updated Merge Requests will have a pipeline created when using +This limitation does not affect any of the updated merge request pipelines. +All updated merge requests have a pipeline created when using [pipelines for merge requests](../merge_request_pipelines/index.md). ## Deprecated parameters diff --git a/doc/ci/yaml/img/ci_config_visualization_hover_v13_5.png b/doc/ci/yaml/img/ci_config_visualization_hover_v13_5.png Binary files differnew file mode 100644 index 00000000000..e6c85bd39e4 --- /dev/null +++ b/doc/ci/yaml/img/ci_config_visualization_hover_v13_5.png diff --git a/doc/ci/yaml/img/ci_config_visualization_v13_5.png b/doc/ci/yaml/img/ci_config_visualization_v13_5.png Binary files differnew file mode 100644 index 00000000000..0aee5cff7be --- /dev/null +++ b/doc/ci/yaml/img/ci_config_visualization_v13_5.png diff --git a/doc/ci/yaml/includes.md b/doc/ci/yaml/includes.md index f7ed7248dc4..d7945617dc9 100644 --- a/doc/ci/yaml/includes.md +++ b/doc/ci/yaml/includes.md @@ -73,10 +73,11 @@ automatically fetched and evaluated along with the content of `.gitlab-ci.yml`. Content of `https://gitlab.com/awesome-project/raw/master/.before-script-template.yml`: ```yaml -before_script: - - apt-get update -qq && apt-get install -y -qq sqlite3 libsqlite3-dev nodejs - - gem install bundler --no-document - - bundle install --jobs $(nproc) "${FLAGS[@]}" +default: + before_script: + - apt-get update -qq && apt-get install -y -qq sqlite3 libsqlite3-dev nodejs + - gem install bundler --no-document + - bundle install --jobs $(nproc) "${FLAGS[@]}" ``` Content of `.gitlab-ci.yml`: diff --git a/doc/ci/yaml/visualization.md b/doc/ci/yaml/visualization.md new file mode 100644 index 00000000000..ac5f5c8fd14 --- /dev/null +++ b/doc/ci/yaml/visualization.md @@ -0,0 +1,46 @@ +# Visualize your CI/CD configuration + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/241722) in GitLab 13.5. +> - It's [deployed behind a feature flag](../../user/feature_flags.md), disabled by default. +> - It's disabled on GitLab.com. +> - It's not recommended for production use. +> - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enable-or-disable-cicd-configuration-visualization). **(CORE ONLY)** + +CAUTION: **Warning:** +This feature might not be available to you. Check the **version history** note above for details. + +To see a visualization of your `gitlab-ci.yml` configuration, navigate to any CI/CD +configuration file and click on the `Visualization` tab. The visualization shows +all stages and jobs. [`needs`](README.md#needs) relationships are displayed as lines +connecting jobs together, showing the hierarchy of execution: + + + +Hovering on a job highlights its `needs` relationships: + + + +If the configuration does not have any `needs` relationships, then no lines are drawn because +each job depends only on the previous stage being completed successfully. + +You can only preview one `gitlab-ci.yml` file at a time. Configuration imported with +[`includes`](README.md#include) is ignored and not included in the visualization. + +## Enable or disable CI/CD configuration visualization **(CORE ONLY)** + +CI/CD configuration visualization is under development and not ready for production use. It is +deployed behind a feature flag that is **disabled by default**. +[GitLab administrators with access to the GitLab Rails console](../../administration/feature_flags.md) +can enable it. + +To enable it: + +```ruby +Feature.enable(:gitlab_ci_yml_preview) +``` + +To disable it: + +```ruby +Feature.disable(:gitlab_ci_yml_preview) +``` diff --git a/doc/container_registry/README.md b/doc/container_registry/README.md deleted file mode 100644 index b98d1b51999..00000000000 --- a/doc/container_registry/README.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../user/packages/container_registry/index.md' ---- - -This document was moved to [another location](../user/packages/container_registry/index.md). diff --git a/doc/container_registry/troubleshooting.md b/doc/container_registry/troubleshooting.md deleted file mode 100644 index 092d7831e35..00000000000 --- a/doc/container_registry/troubleshooting.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../user/packages/container_registry/index.md#troubleshooting-the-gitlab-container-registry' ---- - -This document was moved to [user/project/container_registry](../user/packages/container_registry/index.md#troubleshooting-the-gitlab-container-registry). diff --git a/doc/customization/welcome_message.md b/doc/customization/welcome_message.md index 45f1fed355e..9667e5e380a 100644 --- a/doc/customization/welcome_message.md +++ b/doc/customization/welcome_message.md @@ -1,5 +1,5 @@ --- -redirect_to: 'branded_login_page.md' +redirect_to: '../user/admin_area/appearance.md#sign-in--sign-up-pages' --- -This document was moved to [another location](branded_login_page.md). +This document was moved to [another location](../user/admin_area/appearance.md#sign-in--sign-up-pages). diff --git a/doc/development/README.md b/doc/development/README.md index abdd5c662f3..7b8a5cd5f75 100644 --- a/doc/development/README.md +++ b/doc/development/README.md @@ -43,6 +43,7 @@ For information on how to install, configure, update, and upgrade your own GitLa **Must-reads:** +- [Guide on adapting existing and introducing new components](architecture.md#adapting-existing-and-introducing-new-components) - [Code review guidelines](code_review.md) for reviewing code and having code reviewed - [Database review guidelines](database_review.md) for reviewing database-related changes and complex SQL queries, and having them reviewed - [Secure coding guidelines](secure_coding_guidelines.md) @@ -115,6 +116,7 @@ Complementary reads: - [Code Intelligence](code_intelligence/index.md) - [Approval Rules](approval_rules.md) - [Feature categorization](feature_categorization/index.md) +- [Wikis development guide](wikis.md) ## Performance guides @@ -163,11 +165,11 @@ See [database guidelines](database/index.md). - [Externalization](i18n/externalization.md) - [Translation](i18n/translation.md) -## Telemetry guides +## Product Analytics guides -- [Telemetry guide](telemetry/index.md) -- [Usage Ping guide](telemetry/usage_ping.md) -- [Snowplow guide](telemetry/snowplow.md) +- [Product Analytics guide](product_analytics/index.md) +- [Usage Ping guide](product_analytics/usage_ping.md) +- [Snowplow guide](product_analytics/snowplow.md) ## Experiment guide diff --git a/doc/development/adding_database_indexes.md b/doc/development/adding_database_indexes.md index 03557491e68..d2526d6c4f2 100644 --- a/doc/development/adding_database_indexes.md +++ b/doc/development/adding_database_indexes.md @@ -1,3 +1,9 @@ +--- +stage: Enablement +group: Database +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Adding Database Indexes Indexes can be used to speed up database queries, but when should you add a new diff --git a/doc/development/adding_service_component.md b/doc/development/adding_service_component.md index 2801e27145d..bb5af286c1d 100644 --- a/doc/development/adding_service_component.md +++ b/doc/development/adding_service_component.md @@ -46,7 +46,7 @@ TIP: **Tip:** **For services that depend on the existing GitLab codebase:** -The first iteration should be opt-in, either through the `gitlab.yml` configuration or through [feature flags](feature_flags.md). For these types of services it is often necessary to [bundle the service and its dependencies with GitLab](#bundling-a-service-with-gitlab) as part of the initial integration. +The first iteration should be opt-in, either through the `gitlab.yml` configuration or through [feature flags](feature_flags/index.md). For these types of services it is often necessary to [bundle the service and its dependencies with GitLab](#bundling-a-service-with-gitlab) as part of the initial integration. TIP: **Tip:** [ActionCable](https://docs.gitlab.com/omnibus/settings/actioncable.html) is an example of a service that has been added this way. diff --git a/doc/development/api_graphql_styleguide.md b/doc/development/api_graphql_styleguide.md index 18fc0fb7d33..3d4c033e676 100644 --- a/doc/development/api_graphql_styleguide.md +++ b/doc/development/api_graphql_styleguide.md @@ -49,6 +49,20 @@ See also: - [Exposing Global IDs](#exposing-global-ids). - [Mutation arguments](#object-identifier-arguments). +We have a custom scalar type (`Types::GlobalIDType`) which should be used as the +type of input and output arguments when the value is a `GlobalID`. The benefits +of using this type instead of `ID` are: + +- it validates that the value is a `GlobalID` +- it parses it into a `GlobalID` before passing it to user code +- it can be parameterized on the type of the object (e.g. + `GlobalIDType[Project]`) which offers even better validation and security. + +Consider using this type for all new arguments and result types. Remember that +it is perfectly possible to parameterize this type with a concern or a +supertype, if you want to accept a wider range of objects (e.g. +`GlobalIDType[Issuable]` vs `GlobalIDType[Issue]`). + ## Types We use a code-first schema, and we declare what type everything is in Ruby. @@ -371,8 +385,8 @@ end GitLab's GraphQL API is versionless, which means we maintain backwards compatibility with older versions of the API with every change. Rather than removing a field or [enum value](#enums), we need to _deprecate_ it instead. -In future, GitLab -[may remove deprecated parts of the schema](https://gitlab.com/gitlab-org/gitlab/-/issues/32292). +The deprecated parts of the schema can then be removed in a future release +in accordance with [GitLab's deprecation process](../api/graphql/index.md#deprecation-process). Fields and enum values are deprecated using the `deprecated` property. The value of the property is a `Hash` of: @@ -472,6 +486,28 @@ end Enum values can be deprecated using the [`deprecated` keyword](#deprecating-fields-and-enum-values). +### Defining GraphQL enums dynamically from Rails enums + +If your GraphQL enum is backed by a [Rails enum](creating_enums.md), then consider +using the Rails enum to dynamically define the GraphQL enum values. Doing so +binds the GraphQL enum values to the Rails enum definition, so if values are +ever added to the Rails enum then the GraphQL enum automatically reflects the change. + +Example: + +```ruby +module Types + class IssuableSeverityEnum < BaseEnum + graphql_name 'IssuableSeverity' + description 'Incident severity' + + ::IssuableSeverity.severities.keys.each do |severity| + value severity.upcase, value: severity, description: "#{severity.titleize} severity" + end + end +end +``` + ## JSON When data to be returned by GraphQL is stored as @@ -780,6 +816,25 @@ to advertise the need for lookahead: For an example of real world use, please see [`ResolvesMergeRequests`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/graphql/resolvers/concerns/resolves_merge_requests.rb). +### Negated arguments + +Negated filters can filter some resources (for example, find all issues that +have the `bug` label, but don't have the `bug2` label assigned). The `not` +argument is the preferred syntax to pass negated arguments: + +```graphql +issues(labelName: "bug", not: {labelName: "bug2"}) { + nodes { + id + title + } +} +``` + +To avoid duplicated argument definitions, you can place these arguments in a reusable module (or +class, if the arguments are nested). Alternatively, you can consider to add a +[helper resolver method](https://gitlab.com/gitlab-org/gitlab/-/issues/258969). + ## Pass a parent object into a child Presenter Sometimes you need to access the resolved query parent in a child context to compute fields. Usually the parent is only @@ -827,10 +882,39 @@ mutation. ### Building Mutations -Mutations live in `app/graphql/mutations` ideally grouped per +Mutations are stored in `app/graphql/mutations`, ideally grouped per resources they are mutating, similar to our services. They should inherit `Mutations::BaseMutation`. The fields defined on the mutation -will be returned as the result of the mutation. +are returned as the result of the mutation. + +#### Update mutation granularity + +GitLab's service-oriented architecture means that most mutations call a Create, Delete, or Update +service, for example `UpdateMergeRequestService`. +For Update mutations, a you might want to only update one aspect of an object, and thus only need a +_fine-grained_ mutation, for example `MergeRequest::SetWip`. + +It's acceptable to have both fine-grained mutations and coarse-grained mutations, but be aware +that too many fine-grained mutations can lead to organizational challenges in maintainability, code +comprehensibility, and testing. +Each mutation requires a new class, which can lead to technical debt. +It also means the schema becomes very big, and we want users to easily navigate our schema. +As each new mutation also needs tests (including slower request integration tests), adding mutations +slows down the test suite. + +To minimize changes: + +- Use existing mutations, such as `MergeRequest::Update`, when available. +- Expose existing services as a coarse-grained mutation. + +When a fine-grained mutation might be more appropriate: + +- Modifying a property that requires specific permissions or other specialized logic. +- Exposing a state-machine-like transition (locking issues, merging MRs, closing epics, etc). +- Accepting nested properties (where we accept properties for a child object). +- The semantics of the mutation can be expressed clearly and concisely. + +See [issue #233063](https://gitlab.com/gitlab-org/gitlab/-/issues/233063) for further context. ### Naming conventions @@ -1361,5 +1445,4 @@ For information on generating GraphQL documentation and schema files, see [updating the schema documentation](rake_tasks.md#update-graphql-documentation-and-schema-definitions). To help our readers, you should also add a new page to our [GraphQL API](../api/graphql/index.md) documentation. -For guidance, see the [GraphQL API](documentation/styleguide.md#graphql-api) section -of our documentation style guide. +For guidance, see the [GraphQL API](documentation/graphql_styleguide.md) page. diff --git a/doc/development/api_styleguide.md b/doc/development/api_styleguide.md index 400752c69e9..47c06377d88 100644 --- a/doc/development/api_styleguide.md +++ b/doc/development/api_styleguide.md @@ -17,7 +17,7 @@ Each new or updated API endpoint must come with documentation, unless it is inte The docs should be in the same merge request, or, if strictly necessary, in a follow-up with the same milestone as the original merge request. -See the [Documentation Style Guide RESTful API section](documentation/styleguide.md#restful-api) for details on documenting API resources in Markdown as well as in OpenAPI definition files. +See the [Documentation Style Guide RESTful API page](documentation/restful_api_styleguide.md) for details on documenting API resources in Markdown as well as in OpenAPI definition files. ## Methods and parameters description diff --git a/doc/development/architecture.md b/doc/development/architecture.md index f6b1c8cd914..513af491576 100644 --- a/doc/development/architecture.md +++ b/doc/development/architecture.md @@ -1,32 +1,91 @@ -# GitLab Architecture Overview +# GitLab architecture overview ## Software delivery -There are two software distributions of GitLab: the open source [Community Edition](https://gitlab.com/gitlab-org/gitlab-foss/) (CE), and the open core [Enterprise Edition](https://gitlab.com/gitlab-org/gitlab/) (EE). GitLab is available under [different subscriptions](https://about.gitlab.com/pricing/). +There are two software distributions of GitLab: -New versions of GitLab are released in stable branches and the master branch is for bleeding edge development. +- The open source [Community Edition](https://gitlab.com/gitlab-org/gitlab-foss/) (CE). +- The open core [Enterprise Edition](https://gitlab.com/gitlab-org/gitlab/) (EE). -For information, see the [GitLab Release Process](https://gitlab.com/gitlab-org/release/docs/-/tree/master#gitlab-release-process). +GitLab is available under [different subscriptions](https://about.gitlab.com/pricing/). -Both EE and CE require some add-on components called GitLab Shell and Gitaly. These components are available from the [GitLab Shell](https://gitlab.com/gitlab-org/gitlab-shell/-/tree/master) and [Gitaly](https://gitlab.com/gitlab-org/gitaly/-/tree/master) repositories respectively. New versions are usually tags but staying on the master branch will give you the latest stable version. New releases are generally around the same time as GitLab CE releases with the exception of informal security updates deemed critical. +New versions of GitLab are released from stable branches, and the `master` branch is used for +bleeding-edge development. + +For more information, visit the [GitLab Release Process](https://about.gitlab.com/handbook/engineering/releases/). + +Both distributions require additional components. These components are described in the +[Component details](#components) section, and all have their own repositories. +New versions of each dependent component are usually tags, but staying on the `master` branch of the +GitLab codebase gives you the latest stable version of those components. New versions are +generally released around the same time as GitLab releases, with the exception of informal security +updates deemed critical. ## Components -A typical install of GitLab will be on GNU/Linux. It uses NGINX or Apache as a web front end to proxypass the Unicorn web server. By default, communication between Unicorn and the front end is via a Unix domain socket but forwarding requests via TCP is also supported. The web front end accesses `/home/git/gitlab/public` bypassing the Unicorn server to serve static pages, uploads (e.g. avatar images or attachments), and pre-compiled assets. GitLab serves web pages and the [GitLab API](../api/README.md) using the Unicorn web server. It uses Sidekiq as a job queue which, in turn, uses Redis as a non-persistent database backend for job information, meta data, and incoming jobs. +A typical install of GitLab is on GNU/Linux, but growing number of deployments also use the +Kubernetes platform. The largest known GitLab instance is on GitLab.com, which is deployed using our +[official GitLab Helm chart](https://docs.gitlab.com/charts/) and the [official Linux package](https://about.gitlab.com/install/). -We also support deploying GitLab on Kubernetes using our [GitLab Helm chart](https://docs.gitlab.com/charts/). +A typical installation uses NGINX or Apache as a web server to proxy through +[GitLab Workhorse](https://gitlab.com/gitlab-org/gitlab-workhorse) and into the [Puma](https://puma.io) +application server. GitLab serves web pages and the [GitLab API](../api/README.md) using the Puma +application server. It uses Sidekiq as a job queue which, in turn, uses Redis as a non-persistent +database backend for job information, metadata, and incoming jobs. -The GitLab web app uses PostgreSQL for persistent database information (e.g. users, permissions, issues, other meta data). GitLab stores the bare Git repositories it serves in `/home/git/repositories` by default. It also keeps default branch and hook information with the bare repository. +By default, communication between Puma and Workhorse is via a Unix domain socket, but forwarding +requests via TCP is also supported. Workhorse accesses the `gitlab/public` directory, bypassing the +Puma application server to serve static pages, uploads (for example, avatar images or attachments), +and pre-compiled assets. -When serving repositories over HTTP/HTTPS GitLab utilizes the GitLab API to resolve authorization and access as well as serving Git objects. +The GitLab application uses PostgreSQL for persistent database information (for example, users, +permissions, issues, or other metadata). GitLab stores the bare Git repositories in the location +defined in [the configuration file, `repositories:` section](https://gitlab.com/gitlab-org/gitlab/blob/master/config/gitlab.yml.example). +It also keeps default branch and hook information with the bare repository. -The add-on component GitLab Shell serves repositories over SSH. It manages the SSH keys within `/home/git/.ssh/authorized_keys` which should not be manually edited. GitLab Shell accesses the bare repositories through Gitaly to serve Git objects and communicates with Redis to submit jobs to Sidekiq for GitLab to process. GitLab Shell queries the GitLab API to determine authorization and access. +When serving repositories over HTTP/HTTPS GitLab uses the GitLab API to resolve authorization and +access and to serve Git objects. -Gitaly executes Git operations from GitLab Shell and the GitLab web app, and provides an API to the GitLab web app to get attributes from Git (e.g. title, branches, tags, other meta data), and to get blobs (e.g. diffs, commits, files). +The add-on component GitLab Shell serves repositories over SSH. It manages the SSH keys within the +location defined in [the configuration file, `GitLab Shell` section](https://gitlab.com/gitlab-org/gitlab/blob/master/config/gitlab.yml.example). +The file in that location should never be manually edited. GitLab Shell accesses the bare +repositories through Gitaly to serve Git objects, and communicates with Redis to submit jobs to +Sidekiq for GitLab to process. GitLab Shell queries the GitLab API to determine authorization and access. + +Gitaly executes Git operations from GitLab Shell and the GitLab web app, and provides an API to the +GitLab web app to get attributes from Git (for example, title, branches, tags, or other metadata), +and to get blobs (for example, diffs, commits, or files). You may also be interested in the [production architecture of GitLab.com](https://about.gitlab.com/handbook/engineering/infrastructure/production/architecture/). -### Simplified Component Overview +## Adapting existing and introducing new components + +There are fundamental differences in how the application behaves when it is installed on a +traditional Linux machine compared to a containerized platform, such as Kubernetes. + +Compared to [our official installation methods](https://about.gitlab.com/install/), some of the +notable differences are: + +- Official Linux packages can access files on the same file system with different services. + [Shared files](shared_files.md) are not an option for the application running on the Kubernetes + platform. +- Official Linux packages by default have services that have access to the shared configuration and + network. This is not the case for services running in Kubernetes, where services might be running + in complete isolation, or only accessible through specific ports. + +In other words, the shared state between services needs to be carefully considered when +architecting new features and adding new components. Services that need to have access to the same +files, need to be able to exchange information through the appropriate APIs. Whenever possible, +this should not be done with files. + +Since components written with the API-first philosophy in mind are compatible with both methods, all +new features and services must be written to consider Kubernetes compatibility **first**. + +The simplest way to ensure this, is to add support for your feature or service to +[the official GitLab Helm chart](https://docs.gitlab.com/charts/) or reach out to +[the Distribution team](https://about.gitlab.com/handbook/engineering/development/enablement/distribution/#how-to-work-with-distribution). + +### Simplified component overview This is a simplified architecture diagram that can be used to understand GitLab's architecture. @@ -51,23 +110,25 @@ SSH -- TCP 22 --> GitLabShell[GitLab Shell] SMTP[SMTP Gateway] Geo[GitLab Geo Node] -- TCP 22, 80, 443 --> NGINX -GitLabShell --TCP 8080 -->Unicorn["Unicorn (GitLab Rails)"] +GitLabShell --TCP 8080 -->Puma["Puma (GitLab Rails)"] GitLabShell --> Praefect -Unicorn --> PgBouncer[PgBouncer] -Unicorn --> Redis -Unicorn --> Praefect +Puma --> PgBouncer[PgBouncer] +Puma --> Redis +Puma --> Praefect Sidekiq --> Redis Sidekiq --> PgBouncer Sidekiq --> Praefect -GitLabWorkhorse[GitLab Workhorse] --> Unicorn +GitLabWorkhorse[GitLab Workhorse] --> Puma GitLabWorkhorse --> Redis GitLabWorkhorse --> Praefect Praefect --> Gitaly NGINX --> GitLabWorkhorse NGINX -- TCP 8090 --> GitLabPages[GitLab Pages] NGINX --> Grafana[Grafana] +NGINX -- TCP 8150 --> GitLabKas[GitLab Kubernetes Agent Server] +GitLabKas --> Praefect Grafana -- TCP 9090 --> Prometheus[Prometheus] -Prometheus -- TCP 80, 443 --> Unicorn +Prometheus -- TCP 80, 443 --> Puma RedisExporter[Redis Exporter] --> Redis Prometheus -- TCP 9121 --> RedisExporter PostgreSQLExporter[PostgreSQL Exporter] --> PostgreSQL @@ -83,27 +144,27 @@ PgBouncer --> Consul PostgreSQL --> Consul PgBouncer --> PostgreSQL NGINX --> Registry -Unicorn --> Registry +Puma --> Registry NGINX --> Mattermost -Mattermost --- Unicorn +Mattermost --- Puma Prometheus --> Alertmanager Migrations --> PostgreSQL Runner -- TCP 443 --> NGINX -Unicorn -- TCP 9200 --> Elasticsearch +Puma -- TCP 9200 --> Elasticsearch Sidekiq -- TCP 9200 --> Elasticsearch Sidekiq -- TCP 80, 443 --> Sentry -Unicorn -- TCP 80, 443 --> Sentry +Puma -- TCP 80, 443 --> Sentry Sidekiq -- UDP 6831 --> Jaeger -Unicorn -- UDP 6831 --> Jaeger +Puma -- UDP 6831 --> Jaeger Gitaly -- UDP 6831 --> Jaeger GitLabShell -- UDP 6831 --> Jaeger GitLabWorkhorse -- UDP 6831 --> Jaeger Alertmanager -- TCP 25 --> SMTP Sidekiq -- TCP 25 --> SMTP -Unicorn -- TCP 25 --> SMTP -Unicorn -- TCP 369 --> LDAP +Puma -- TCP 25 --> SMTP +Puma -- TCP 369 --> LDAP Sidekiq -- TCP 369 --> LDAP -Unicorn -- TCP 443 --> ObjectStorage["Object Storage"] +Puma -- TCP 443 --> ObjectStorage["Object Storage"] Sidekiq -- TCP 443 --> ObjectStorage GitLabWorkhorse -- TCP 443 --> ObjectStorage Registry -- TCP 443 --> ObjectStorage @@ -121,7 +182,7 @@ click Gitaly "./architecture.html#gitaly" click Jaeger "./architecture.html#jaeger" click GitLabWorkhorse "./architecture.html#gitlab-workhorse" click LDAP "./architecture.html#ldap-authentication" -click Unicorn "./architecture.html#unicorn" +click Puma "./architecture.html#puma" click GitLabShell "./architecture.html#gitlab-shell" click SSH "./architecture.html#ssh-request-22" click Sidekiq "./architecture.html#sidekiq" @@ -175,6 +236,7 @@ Table description links: | [GitLab Geo Node](#gitlab-geo) | Geographically distributed GitLab nodes | ⚙ | ⚙ | ❌ | ✅ | ❌ | ⚙ | EE Only | | [GitLab Managed Apps](#gitlab-managed-apps) | Deploy Helm, Ingress, Cert-Manager, Prometheus, GitLab Runner, JupyterHub, or Knative to a cluster | ⤓ | ⤓ | ⤓ | ⤓ | ⤓ | ⤓ | CE & EE | | [GitLab Pages](#gitlab-pages) | Hosts static websites | ⚙ | ❌ | ❌ | ✅ | ⚙ | ⚙ | CE & EE | +| [GitLab Kubernetes Agent](#gitlab-kubernetes-agent) | Integrate Kubernetes clusters in a cloud-native way | ⚙ | ⚙ | ❌ | ❌ | ⤓ | ⚙ | EE Only | | [GitLab self-monitoring: Alertmanager](#alertmanager) | Deduplicates, groups, and routes alerts from Prometheus | ⚙ | ✅ | ⚙ | ✅ | ❌ | ❌ | CE & EE | | [GitLab self-monitoring: Grafana](#grafana) | Metrics dashboard | ✅ | ⚙ | ⤓ | ✅ | ❌ | ❌ | CE & EE | | [GitLab self-monitoring: Jaeger](#jaeger) | View traces generated by the GitLab instance | ❌ | ⚙ | ❌ | ❌ | ⤓ | ⚙ | CE & EE | @@ -201,7 +263,7 @@ Table description links: | [Runner](#gitlab-runner) | Executes GitLab CI/CD jobs | ⤓ | ✅ | ⚙ | ✅ | ⚙ | ⚙ | CE & EE | | [Sentry integration](#sentry) | Error tracking for deployed apps | ⤓ | ⤓ | ⤓ | ⤓ | ⤓ | ⤓ | CE & EE | | [Sidekiq](#sidekiq) | Background jobs processor | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | CE & EE | -| [Unicorn (GitLab Rails)](#unicorn) | Handles requests for the web interface and API | ✅ | ✅ | ✅ | ✅ | ⚙ | ✅ | CE & EE | +| [Puma (GitLab Rails)](#puma) | Handles requests for the web interface and API | ✅ | ✅ | ✅ | ✅ | ⚙ | ✅ | CE & EE | ### Component details @@ -271,7 +333,7 @@ Consul is a tool for service discovery and configuration. Consul is distributed, - [Source](../integration/elasticsearch.md) - [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit/blob/master/doc/howto/elasticsearch.md) - Layer: Core Service (Data) -- GitLab.com: [Get Advanced Global Search working on GitLab.com](https://gitlab.com/groups/gitlab-org/-/epics/153) epic. +- GitLab.com: [Get Advanced Search working on GitLab.com](https://gitlab.com/groups/gitlab-org/-/epics/153) epic. Elasticsearch is a distributed RESTful search engine built for the cloud. @@ -321,6 +383,19 @@ repository updates to secondary nodes. GitLab Exporter is a process designed in house that allows us to export metrics about GitLab application internals to Prometheus. You can read more [in the project's README](https://gitlab.com/gitlab-org/gitlab-exporter). +#### GitLab Kubernetes Agent + +- [Project page](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent) +- Configuration: + - [Omnibus](https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-config-template/gitlab.rb.template) + - [Charts](https://docs.gitlab.com/charts/charts/gitlab/kas/index.html) + +[GitLab Kubernetes Agent](../user/clusters/agent/index.md) is an active in-cluster +component for solving GitLab and Kubernetes integration tasks in a secure and +cloud-native way. + +You can use it to sync deployments onto your Kubernetes cluster. + #### GitLab Pages - Configuration: @@ -368,13 +443,13 @@ GitLab CI/CD is the open-source continuous integration service included with Git - [Project page](https://gitlab.com/gitlab-org/gitlab-workhorse/blob/master/README.md) - Configuration: - [Omnibus](https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-config-template/gitlab.rb.template) - - [Charts](https://docs.gitlab.com/charts/charts/gitlab/unicorn/) + - [Charts](https://docs.gitlab.com/charts/charts/gitlab/webservice/) - [Source](../install/installation.md#install-gitlab-workhorse) - Layer: Core Service (Processor) - Process: `gitlab-workhorse` - GitLab.com: [Service Architecture](https://about.gitlab.com/handbook/engineering/infrastructure/production/architecture/#service-architecture) -[GitLab Workhorse](https://gitlab.com/gitlab-org/gitlab-workhorse) is a program designed at GitLab to help alleviate pressure from Unicorn. You can read more about the [historical reasons for developing](https://about.gitlab.com/blog/2016/04/12/a-brief-history-of-gitlab-workhorse/). It's designed to act as a smart reverse proxy to help speed up GitLab as a whole. +[GitLab Workhorse](https://gitlab.com/gitlab-org/gitlab-workhorse) is a program designed at GitLab to help alleviate pressure from Puma. You can read more about the [historical reasons for developing](https://about.gitlab.com/blog/2016/04/12/a-brief-history-of-gitlab-workhorse/). It's designed to act as a smart reverse proxy to help speed up GitLab as a whole. #### Grafana @@ -411,7 +486,8 @@ For monitoring deployed apps, see [Jaeger tracing documentation](../operations/t - Layer: Core Service - Process: `logrotate` -GitLab is comprised of a large number of services that all log. We started bundling our own logrotate as of 7.4 to make sure we were logging responsibly. This is just a packaged version of the common open source offering. +GitLab is comprised of a large number of services that all log. We started bundling our own Logrotate +as of GitLab 7.4 to make sure we were logging responsibly. This is just a packaged version of the common open source offering. #### Mattermost @@ -560,7 +636,7 @@ Redis is packaged to provide a place to store: - [Source](../administration/packages/container_registry.md#enable-the-container-registry) - [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit/blob/master/doc/howto/registry.md) - Layer: Core Service (Processor) -- GitLab.com: [GitLab Container Registry](../user/packages/container_registry/index.md#build-and-push-images-using-gitlab-cicd) +- GitLab.com: [GitLab Container Registry](../user/packages/container_registry/index.md#build-and-push-by-using-gitlab-cicd) The registry is what users use to store their own Docker images. The bundled registry uses NGINX as a load balancer and GitLab as an authentication manager. @@ -603,8 +679,30 @@ For monitoring deployed apps, see the [Sentry integration docs](../operations/er Sidekiq is a Ruby background job processor that pulls jobs from the Redis queue and processes them. Background jobs allow GitLab to provide a faster request/response cycle by moving work into the background. +#### Puma + +NOTE: **Note:** +Starting with GitLab 13.0, Puma is the default web server and Unicorn has been +disabled by default. + +- [Project page](https://gitlab.com/gitlab-org/gitlab/blob/master/README.md) +- Configuration: + - [Omnibus](https://docs.gitlab.com/omnibus/settings/puma.html) + - [Charts](https://docs.gitlab.com/charts/charts/gitlab/webservice/) + - [Source](../install/installation.md#configure-it) + - [GDK](https://gitlab.com/gitlab-org/gitlab/blob/master/config/gitlab.yml.example) +- Layer: Core Service (Processor) +- Process: `puma` +- GitLab.com: [Puma](../user/gitlab_com/index.md#puma) + +[Puma](https://puma.io/) is a Ruby application server that is used to run the core Rails Application that provides the user facing features in GitLab. Often process output you will see this as `bundle` or `config.ru` depending on the GitLab version. + #### Unicorn +NOTE: **Note:** +Starting with GitLab 13.0, Puma is the default web server and Unicorn has been +disabled by default. + - [Project page](https://gitlab.com/gitlab-org/gitlab/blob/master/README.md) - Configuration: - [Omnibus](https://docs.gitlab.com/omnibus/settings/unicorn.html) @@ -669,7 +767,7 @@ You can install them after you create a cluster. This includes: - [JupyterHub](https://jupyter.org) - [Knative](https://cloud.google.com/knative/) -## GitLab by Request Type +## GitLab by request type GitLab provides two "interfaces" for end users to access the service: @@ -678,20 +776,20 @@ GitLab provides two "interfaces" for end users to access the service: It's important to understand the distinction as some processes are used in both and others are exclusive to a specific request type. -### GitLab Web HTTP Request Cycle +### GitLab Web HTTP request cycle When making a request to an HTTP Endpoint (think `/users/sign_in`) the request will take the following path through the GitLab Service: - NGINX - Acts as our first line reverse proxy. -- GitLab Workhorse - This determines if it needs to go to the Rails application or somewhere else to reduce load on Unicorn. -- Unicorn - Since this is a web request, and it needs to access the application it will go to Unicorn. +- GitLab Workhorse - This determines if it needs to go to the Rails application or somewhere else to reduce load on Puma. +- Puma - Since this is a web request, and it needs to access the application it will go to Puma. - PostgreSQL/Gitaly/Redis - Depending on the type of request, it may hit these services to store or retrieve data. -### GitLab Git Request Cycle +### GitLab Git request cycle Below we describe the different paths that HTTP vs. SSH Git requests will take. There is some overlap with the Web Request Cycle but also some differences. -### Web Request (80/443) +### Web request (80/443) Git operations over HTTP use the stateless "smart" protocol described in the [Git documentation](https://git-scm.com/docs/http-protocol), but responsibility @@ -736,7 +834,7 @@ sequenceDiagram The sequence is similar for `git push`, except `git-receive-pack` is used instead of `git-upload-pack`. -### SSH Request (22) +### SSH request (22) Git operations over SSH can use the stateful protocol described in the [Git documentation](https://git-scm.com/docs/pack-protocol#_ssh_transport), but @@ -801,7 +899,7 @@ except there is no round-trip into Gitaly - Rails performs the action as part of the [internal API](internal_api.md) call, and GitLab Shell streams the response back to the user directly. -## System Layout +## System layout When referring to `~git` in the pictures it means the home directory of the Git user which is typically `/home/git`. @@ -811,9 +909,9 @@ The bare repositories are located in `/home/git/repositories`. GitLab is a Ruby To serve repositories over SSH there's an add-on application called GitLab Shell which is installed in `/home/git/gitlab-shell`. -### Installation Folder Summary +### Installation folder summary -To summarize here's the [directory structure of the `git` user home directory](../install/structure.md). +To summarize here's the [directory structure of the `git` user home directory](../install/installation.md#gitlab-directory-structure). ### Processes @@ -823,12 +921,12 @@ ps aux | grep '^git' GitLab has several components to operate. It requires a persistent database (PostgreSQL) and Redis database, and uses Apache `httpd` or NGINX to proxypass -Unicorn. All these components should run as different system users to GitLab -(e.g., `postgres`, `redis` and `www-data`, instead of `git`). +Puma. All these components should run as different system users to GitLab +(for example, `postgres`, `redis`, and `www-data`, instead of `git`). -As the `git` user it starts Sidekiq and Unicorn (a simple Ruby HTTP server +As the `git` user it starts Sidekiq and Puma (a simple Ruby HTTP server running on port `8080` by default). Under the GitLab user there are normally 4 -processes: `unicorn_rails master` (1 process), `unicorn_rails worker` +processes: `puma master` (1 process), `puma cluster worker` (2 processes), `sidekiq` (1 process). ### Repository access @@ -841,7 +939,7 @@ See the README for more information. ### Init scripts of the services -The GitLab init script starts and stops Unicorn and Sidekiq: +The GitLab init script starts and stops Puma and Sidekiq: ```plaintext /etc/init.d/gitlab @@ -881,9 +979,9 @@ Usage: /etc/init.d/postgresql {start|stop|restart|reload|force-reload|status} [v ### Log locations of the services -GitLab (includes Unicorn and Sidekiq logs): +GitLab (includes Puma and Sidekiq logs): -- `/home/git/gitlab/log/` contains `application.log`, `production.log`, `sidekiq.log`, `unicorn.stdout.log`, `git_json.log` and `unicorn.stderr.log` normally. +- `/home/git/gitlab/log/` contains `application.log`, `production.log`, `sidekiq.log`, `puma.stdout.log`, `git_json.log` and `puma.stderr.log` normally. GitLab Shell: @@ -914,17 +1012,18 @@ PostgreSQL: ### GitLab specific configuration files -GitLab has configuration files located in `/home/git/gitlab/config/*`. Commonly referenced config files include: +GitLab has configuration files located in `/home/git/gitlab/config/*`. Commonly referenced +configuration files include: -- `gitlab.yml` - GitLab configuration. -- `unicorn.rb` - Unicorn web server settings. -- `database.yml` - Database connection settings. +- `gitlab.yml` - GitLab configuration +- `puma.rb` - Puma web server settings +- `database.yml` - Database connection settings GitLab Shell has a configuration file at `/home/git/gitlab-shell/config.yml`. -### Maintenance Tasks +### Maintenance tasks -[GitLab](https://gitlab.com/gitlab-org/gitlab/tree/master) provides Rake tasks with which you see version information and run a quick check on your configuration to ensure it is configured properly within the application. See [maintenance Rake tasks](../raketasks/maintenance.md). +[GitLab](https://gitlab.com/gitlab-org/gitlab/tree/master) provides Rake tasks with which you see version information and run a quick check on your configuration to ensure it is configured properly within the application. See [maintenance Rake tasks](../administration/raketasks/maintenance.md). In a nutshell, do the following: ```shell @@ -934,7 +1033,8 @@ bundle exec rake gitlab:env:info RAILS_ENV=production bundle exec rake gitlab:check RAILS_ENV=production ``` -Note: It is recommended to log into the `git` user using `sudo -i -u git` or `sudo su - git`. While the sudo commands provided by GitLab work in Ubuntu they do not always work in RHEL. +Note: It is recommended to log into the `git` user using `sudo -i -u git` or `sudo su - git`. While +the `sudo` commands provided by GitLab work in Ubuntu they do not always work in RHEL. ## GitLab.com diff --git a/doc/development/background_migrations.md b/doc/development/background_migrations.md index 4b58758b5c7..e5cc2ae4d1d 100644 --- a/doc/development/background_migrations.md +++ b/doc/development/background_migrations.md @@ -1,4 +1,11 @@ -# Background Migrations +--- +type: reference, dev +stage: none +group: Development +info: "See the Technical Writers assigned to Development Guidelines: https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments-to-development-guidelines" +--- + +# Background migrations Background migrations can be used to perform data migrations that would otherwise take a very long time (hours, days, years, etc) to complete. For @@ -92,6 +99,20 @@ bulk_migrate_async( ) ``` +Note that this will queue a Sidekiq job immediately: if you have a large number +of records, this may not be what you want. You can use the function +`queue_background_migration_jobs_by_range_at_intervals` to split the job into +batches: + +```ruby +queue_background_migration_jobs_by_range_at_intervals( + ClassName, + BackgroundMigrationClassName, + 2.minutes, + batch_size: 10_000 + ) +``` + You'll also need to make sure that newly created data is either migrated, or saved in both the old and new version upon creation. For complex and time consuming migrations it's best to schedule a background job using an diff --git a/doc/development/changelog.md b/doc/development/changelog.md index f57e666540c..922c4814d91 100644 --- a/doc/development/changelog.md +++ b/doc/development/changelog.md @@ -31,14 +31,16 @@ the `author` field. GitLab team members **should not**. - Any change that introduces a database migration, whether it's regular, post, or data migration, **must** have a changelog entry, even if it is behind a - disabled feature flag. + disabled feature flag. Since the migration is executed on [GitLab FOSS](https://gitlab.com/gitlab-org/gitlab-foss/), + the changelog for database schema changes should be written to the + `changelogs/unreleased/` directory, even when other elements of that change affect only GitLab EE. + - [Security fixes](https://gitlab.com/gitlab-org/release/docs/blob/master/general/security/developer.md) **must** have a changelog entry, without `merge_request` value and with `type` set to `security`. -- Any user-facing change **should** have a changelog entry. Example: "GitLab now - uses system fonts for all text." +- Any user-facing change **should** have a changelog entry. This includes both visual changes (regardless of how minor), and changes to the rendered DOM which impact how a screen reader may announce the content. - Performance improvements **should** have a changelog entry. -- Changes that need to be documented in the Telemetry [Event Dictionary](telemetry/event_dictionary.md) +- Changes that need to be documented in the Product Analytics [Event Dictionary](product_analytics/event_dictionary.md) also require a changelog entry. - _Any_ contribution from a community member, no matter how small, **may** have a changelog entry regardless of these guidelines if the contributor wants one. @@ -46,6 +48,7 @@ the `author` field. GitLab team members **should not**. - Any docs-only changes **should not** have a changelog entry. - Any change behind a disabled feature flag **should not** have a changelog entry. - Any change behind an enabled feature flag **should** have a changelog entry. +- Any change that adds new usage data metrics and changes that needs to be documented in Product Analytics [Event Dictionary](telemetry/event_dictionary.md) **should** have a changelog entry. - A change that [removes a feature flag](feature_flags/development.md) **should** have a changelog entry - only if the feature flag did not default to true already. - A fix for a regression introduced and then fixed in the same release (i.e., diff --git a/doc/development/chatops_on_gitlabcom.md b/doc/development/chatops_on_gitlabcom.md index 3c1c7750842..c64766af589 100644 --- a/doc/development/chatops_on_gitlabcom.md +++ b/doc/development/chatops_on_gitlabcom.md @@ -22,9 +22,7 @@ To request access to Chatops on GitLab.com: 1. Log into <https://ops.gitlab.net/users/sign_in> **using the same username** as for GitLab.com (you may have to rename it). 1. You could also use the "Sign in with" Google button to sign in, with your GitLab.com email address. 1. Ask one of your team members to add you to the `chatops` project in Ops. They can do it by running `/chatops run member add <username> gitlab-com/chatops --ops` command in the `#chat-ops-test` Slack channel. - -NOTE: **Note:** -If you had to change your username for GitLab.com on the first step, make sure [to reflect this information](https://gitlab.com/gitlab-com/www-gitlab-com#adding-yourself-to-the-team-page) on [the team page](https://about.gitlab.com/company/team/). +1. If you had to change your username for GitLab.com on the first step, make sure [to reflect this information](https://gitlab.com/gitlab-com/www-gitlab-com#adding-yourself-to-the-team-page) on [the team page](https://about.gitlab.com/company/team/). ## See also diff --git a/doc/development/cicd/templates.md b/doc/development/cicd/templates.md index 77cedc9814e..7d522a55559 100644 --- a/doc/development/cicd/templates.md +++ b/doc/development/cicd/templates.md @@ -13,15 +13,16 @@ This document explains how to develop [GitLab CI/CD templates](../../ci/examples All template files reside in the `lib/gitlab/ci/templates` directory, and are categorized by the following sub-directories: -| Sub-directory | Content | [Selectable in UI](#make-sure-the-new-template-can-be-selected-in-ui) | -|----------------|--------------------------------------------------------------|-----------------------------------------------------------------------| -| `/AWS/*` | Cloud Deployment (AWS) related jobs | No | -| `/Jobs/*` | Auto DevOps related jobs | No | -| `/Pages/*` | Static site generators for GitLab Pages (for example Jekyll) | Yes | -| `/Security/*` | Security related jobs | Yes | -| `/Verify/*` | Verify/testing related jobs | Yes | -| `/Workflows/*` | Common uses of the `workflow:` keyword | No | -| `/*` (root) | General templates | Yes | +| Sub-directory | Content | [Selectable in UI](#make-sure-the-new-template-can-be-selected-in-ui) | +|----------------|----------------------------------------------------|-----------------------------------------------------------------------| +| `/AWS/*` | Cloud Deployment (AWS) related jobs | No | +| `/Jobs/*` | Auto DevOps related jobs | No | +| `/Pages/*` | Static site generators for GitLab Pages (for example Jekyll) | Yes | +| `/Security/*` | Security related jobs | Yes | +| `/Terraform/*` | Infrastructure as Code related templates | No | +| `/Verify/*` | Verify/testing related jobs | Yes | +| `/Workflows/*` | Common uses of the `workflow:` keyword | No | +| `/*` (root) | General templates | Yes | ## Criteria @@ -110,7 +111,7 @@ to include older template versions. If other templates are included with `includ they can be combined with the `include: remote`: ```yaml -# To use the v13 stable template, which is not included in v14, fetch the specifc +# To use the v13 stable template, which is not included in v14, fetch the specific # template from the remote template repository with the `include:remote:` keyword. # If you fetch from the GitLab canonical project, use the following URL format: # https://gitlab.com/gitlab-org/gitlab/-/raw/<version>/lib/gitlab/ci/templates/<template-name> diff --git a/doc/development/code_review.md b/doc/development/code_review.md index 2e319efa5f3..3ff802d3b23 100644 --- a/doc/development/code_review.md +++ b/doc/development/code_review.md @@ -67,6 +67,10 @@ page, with these behaviors: contains the string 'OOO', or the emoji is `:palm_tree:` or `:beach:`. 1. [Trainee maintainers](https://about.gitlab.com/handbook/engineering/workflow/code-review/#trainee-maintainer) are three times as likely to be picked as other reviewers. +1. People whose [GitLab status](../user/profile/index.md#current-status) emoji + is `:large_blue_circle:` are more likely to be picked. This applies to both reviewers and trainee maintainers. + - Reviewers with `:large_blue_circle:` are two times as likely to be picked as other reviewers. + - Trainee maintainers with `:large_blue_circle:` are four times as likely to be picked as other reviewers. 1. It always picks the same reviewers and maintainers for the same branch name (unless their OOO status changes, as in point 1). It removes leading `ce-` and `ee-`, and trailing `-ce` and `-ee`, so @@ -100,6 +104,7 @@ with [domain expertise](#domain-experts). by a [Software Engineer in Test](https://about.gitlab.com/handbook/engineering/quality/#individual-contributors)**. 1. If your merge request only includes end-to-end changes (*3*) **or** if the MR author is a [Software Engineer in Test](https://about.gitlab.com/handbook/engineering/quality/#individual-contributors), it must be **approved by a [Quality maintainer](https://about.gitlab.com/handbook/engineering/projects/#gitlab_maintainers_qa)** 1. If your merge request includes a new or updated [application limit](https://about.gitlab.com/handbook/product/product-processes/#introducing-application-limits), it must be **approved by a [product manager](https://about.gitlab.com/company/team/)**. +1. If your merge request includes Product Analytics (telemetry) changes, it should be reviewed and approved by a [Product analytics engineer](https://gitlab.com/gitlab-org/growth/product-analytics/engineers). - (*1*): Please note that specs other than JavaScript specs are considered backend code. - (*2*): We encourage you to seek guidance from a database maintainer if your merge diff --git a/doc/development/contributing/community_roles.md b/doc/development/contributing/community_roles.md index 7d2d1b77a0e..d880361e3aa 100644 --- a/doc/development/contributing/community_roles.md +++ b/doc/development/contributing/community_roles.md @@ -1,3 +1,9 @@ +--- +stage: none +group: Development +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Community members & roles GitLab community members and their privileges/responsibilities. diff --git a/doc/development/contributing/design.md b/doc/development/contributing/design.md index 352392931c0..891c764f07f 100644 --- a/doc/development/contributing/design.md +++ b/doc/development/contributing/design.md @@ -1,3 +1,10 @@ +--- +type: reference, dev +stage: none +group: Development +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Implement design & UI elements For guidance on UX implementation at GitLab, please refer to our [Design System](https://design.gitlab.com/). diff --git a/doc/development/contributing/index.md b/doc/development/contributing/index.md index 7550fe69546..6cbe57bf926 100644 --- a/doc/development/contributing/index.md +++ b/doc/development/contributing/index.md @@ -1,3 +1,10 @@ +--- +type: reference, dev +stage: none +group: Development +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Contribute to GitLab Thank you for your interest in contributing to GitLab. This guide details how diff --git a/doc/development/contributing/issue_workflow.md b/doc/development/contributing/issue_workflow.md index bb7b4713a5e..b7c05a369f0 100644 --- a/doc/development/contributing/issue_workflow.md +++ b/doc/development/contributing/issue_workflow.md @@ -1,3 +1,10 @@ +--- +type: reference, dev +stage: none +group: Development +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Issues workflow ## Issue tracker guidelines @@ -320,6 +327,11 @@ look for issues labeled `~"Accepting merge requests"` with a [weight of 1](https More experienced contributors are very welcome to tackle [any of them](https://gitlab.com/groups/gitlab-org/-/issues?state=opened&label_name[]=Accepting+merge+requests&assignee_id=None). +For more complex features that have a weight of 2 or more and clear scope, we recommend looking at issues +with the [label `~"Community Challenge"`](https://gitlab.com/gitlab-org/gitlab/-/issues?scope=all&utf8=%E2%9C%93&state=opened&label_name[]=Accepting%20merge%20requests&label_name[]=Community%20challenge). +If your MR for the `~"Community Challenge"` issue gets merged, you will also have a chance to win a custom +GitLab merchandise. + If you've decided that you would like to work on an issue, please @-mention the [appropriate product manager](https://about.gitlab.com/handbook/product/#who-to-talk-to-for-what) as soon as possible. The product manager will then pull in appropriate GitLab team diff --git a/doc/development/contributing/merge_request_workflow.md b/doc/development/contributing/merge_request_workflow.md index d88b159b666..31f59ad875c 100644 --- a/doc/development/contributing/merge_request_workflow.md +++ b/doc/development/contributing/merge_request_workflow.md @@ -1,3 +1,10 @@ +--- +type: reference, dev +stage: none +group: Development +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Merge requests workflow We welcome merge requests from everyone, with fixes and improvements @@ -218,7 +225,7 @@ requirements. 1. [Secure coding guidelines](https://gitlab.com/gitlab-com/gl-security/security-guidelines) have been followed. 1. [Documented](../documentation/index.md) in the `/doc` directory. 1. [Changelog entry added](../changelog.md), if necessary. -1. Reviewed by relevant (UX/FE/BE/tech writing) reviewers and all concerns are addressed. +1. Reviewed by relevant reviewers and all concerns are addressed for Availability, Regressions, Security. Documentation reviews should take place as soon as possible, but they should not block a merge request. 1. Merged by a project maintainer. 1. Create an issue in the [infrastructure issue tracker](https://gitlab.com/gitlab-com/gl-infra/infrastructure/-/issues) to inform the Infrastructure department when your contribution is changing default settings or introduces a new setting, if relevant. 1. Confirmed to be working in the [Canary stage](https://about.gitlab.com/handbook/engineering/#canary-testing) or on GitLab.com once the contribution is deployed. diff --git a/doc/development/contributing/style_guides.md b/doc/development/contributing/style_guides.md index f6e64c1f1e6..773c1a771cd 100644 --- a/doc/development/contributing/style_guides.md +++ b/doc/development/contributing/style_guides.md @@ -1,3 +1,10 @@ +--- +type: reference, dev +stage: none +group: Development +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Style guides ## Editor/IDE styling standardization diff --git a/doc/development/database/add_foreign_key_to_existing_column.md b/doc/development/database/add_foreign_key_to_existing_column.md index 1b41a52c95e..85411ff9aa7 100644 --- a/doc/development/database/add_foreign_key_to_existing_column.md +++ b/doc/development/database/add_foreign_key_to_existing_column.md @@ -1,3 +1,9 @@ +--- +stage: Enablement +group: Database +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Adding foreign key constraint to an existing column Foreign keys help ensure consistency between related database tables. The current database review process **always** encourages you to add [foreign keys](../foreign_keys.md) when creating tables that reference records from other tables. @@ -103,7 +109,7 @@ class RemoveRecordsWithoutUserFromEmailsTable < ActiveRecord::Migration[5.2] end def down - # Can be a no-op when data inconsistency is not affecting the pre and post deploymnet version of the application. + # Can be a no-op when data inconsistency is not affecting the pre and post deployment version of the application. # In this case we might have records in the `emails` table where the associated record in the `users` table is not there anymore. end end diff --git a/doc/development/database/client_side_connection_pool.md b/doc/development/database/client_side_connection_pool.md new file mode 100644 index 00000000000..1a30d2d73a3 --- /dev/null +++ b/doc/development/database/client_side_connection_pool.md @@ -0,0 +1,63 @@ +--- +type: dev, reference +stage: none +group: Development +info: "See the Technical Writers assigned to Development Guidelines: https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments-to-development-guidelines" +--- + +# Client-side connection-pool + +Ruby processes accessing the database through +ActiveRecord, automatically calculate the connection-pool size for the +process based on the concurrency. + +Because of the way [Ruby on Rails manages database +connections](#connection-lifecycle), it is important that we have at +least as many connections as we have threads. While there is a 'pool' +setting in [`database.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/database.yml.postgresql), it is not very practical because you need to +maintain it in tandem with the number of application threads. For this +reason, we override the number of allowed connections in the database +connection-pool based on the configured number of application threads. + +`Gitlab::Runtime.max_threads` is the number of user-facing +application threads the process has been configured with. We also have +auxiliary threads that use database connections. As it isn't +straightforward to keep an accurate count of the number of auxiliary threads as +the application evolves over time, we just add a fixed headroom to the +number of user-facing threads. It is OK if this number is too large +because connections are instantiated lazily. + +## Troubleshooting connection-pool issues + +The connection-pool usage can be seen per environment in the [connection-pool +saturation +dashboard](https://dashboards.gitlab.net/d/alerts-sat_rails_db_connection_pool/alerts-rails_db_connection_pool-saturation-detail?orgId=1). + +If the connection-pool is too small, this would manifest in +`ActiveRecord::ConnectionTimeoutError`s from the application. Because we alert +when almost all connections are used, we should know this before +timeouts occur. If this happens we can remediate by setting the +`DB_POOL_HEADROOM` environment variable to something bigger than the +hardcoded value (10). + +At this point, we need to investigate what is using more connections +than we anticipated. To do that, we can use the +`gitlab_ruby_threads_running_threads` metric. For example, [this +graph](https://thanos-query.ops.gitlab.net/graph?g0.range_input=1h&g0.max_source_resolution=0s&g0.expr=sum%20by%20(thread_name)%20(%20gitlab_ruby_threads_running_threads%7Buses_db_connection%3D%22yes%22%7D%20)&g0.tab=0) +shows all running threads that connect to the database by their +name. Threads labeled `puma worker` or `sidekiq_worker_thread` are +the threads that define `Gitlab::Runtime.max_threads` so those are +accounted for. If there's more than 10 other threads running, we could +consider raising the default headroom. + +## Connection lifecycle + +For web requests, a connection is obtained from the pool at the first +time a database query is made. The connection is returned to the pool +after the request completes. + +For background jobs, the behavior is very similar. The thread obtains +a connection for the first query, and returns it after the job is +finished. + +This is managed by Rails internally. diff --git a/doc/development/database/database_reviewer_guidelines.md b/doc/development/database/database_reviewer_guidelines.md index 6cb061f9959..3345df8b46b 100644 --- a/doc/development/database/database_reviewer_guidelines.md +++ b/doc/development/database/database_reviewer_guidelines.md @@ -1,3 +1,9 @@ +--- +stage: Enablement +group: Database +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Database Reviewer Guidelines This page includes introductory material for new database reviewers. diff --git a/doc/development/database/index.md b/doc/development/database/index.md index 9ea5b6fcaac..4bcefefe7a7 100644 --- a/doc/development/database/index.md +++ b/doc/development/database/index.md @@ -1,3 +1,9 @@ +--- +stage: Enablement +group: Database +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Database guides ## Database Reviews @@ -24,6 +30,7 @@ - [Background migrations](../background_migrations.md) - [Swapping tables](../swapping_tables.md) - [Deleting migrations](../deleting_migrations.md) +- [Partitioning tables](table_partitioning.md) ## Debugging @@ -49,6 +56,8 @@ - [Database Debugging and Troubleshooting](../database_debugging.md) - [Query Count Limits](../query_count_limits.md) - [Creating enums](../creating_enums.md) +- [Client-side connection-pool](client_side_connection_pool.md) +- [Updating multiple values](./setting_multiple_values.md) ## Case studies diff --git a/doc/development/database/not_null_constraints.md b/doc/development/database/not_null_constraints.md index e4dec2afa10..96271863d94 100644 --- a/doc/development/database/not_null_constraints.md +++ b/doc/development/database/not_null_constraints.md @@ -1,3 +1,9 @@ +--- +stage: Enablement +group: Database +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # `NOT NULL` constraints > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/38358) in GitLab 13.0. @@ -33,7 +39,7 @@ end ## Add a `NOT NULL` column to an existing table -With PostgreSQL 11 being the minimum version since GitLab 13.0, adding columns with `NULL` and/or +With PostgreSQL 11 being the minimum version in GitLab 13.0 and later, adding columns with `NULL` and/or default values has become much easier and the standard `add_column` helper should be used in all cases. For example, consider a migration that adds a new `NOT NULL` column `active` to table `db_guides`, diff --git a/doc/development/database/setting_multiple_values.md b/doc/development/database/setting_multiple_values.md new file mode 100644 index 00000000000..5569a0e10b7 --- /dev/null +++ b/doc/development/database/setting_multiple_values.md @@ -0,0 +1,103 @@ +# Setting Multiple Values + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/32921) in GitLab 13.5. + +Frequently, we will want to update multiple objects with new values for one +or more columns. The obvious way to do this is using `Relation#update_all`: + +```ruby +user.issues.open.update_all(due_date: 7.days.from_now) # (1) +user.issues.update_all('relative_position = relative_position + 1') # (2) +``` + +But what do you do if you cannot express the update as either a static value (1) +or as a calculation (2)? + +Thankfully we can use `UPDATE FROM` to express the need to update multiple rows +with distinct values in a single query. One can either use a temporary table, or +a Common Table Expression (CTE), and then use that as the source of the updates: + +```sql +with updates(obj_id, new_title, new_weight) as ( + values (1 :: integer, 'Very difficult issue' :: text, 8 :: integer), + (2, 'Very easy issue', 1) +) +update issues + set title = new_title, weight = new_weight + from updates + where id = obj_id +``` + +The bad news: There is no way to express this in ActiveRecord or even dropping +down to ARel - the `UpdateManager` just does not support `update from`, so this +is not expressible. + +The good news: We supply an abstraction to help you generate these kinds of +updates, called `Gitlab::Database::BulkUpdate`. This constructs queries such as the +above, and uses binding parameters to avoid SQL injection. + +## Usage + +To use this, we need: + +- the list of columns to update +- a mapping from object/ID to the new values to set for that object +- a way to determine the table for each object + +So for example, we can express the query above as: + +```ruby +issue_a = Issue.find(..) +issue_b = Issue.find(..) + +# Issues a single query: +::Gitlab::Database::BulkUpdate.execute(%i[title weight], { + issue_a => { title: 'Very difficult issue', weight: 8 }, + issue_b => { title: 'Very easy issue', weight: 1 } +}) +``` + +Here the table can be determined automatically, from calling +`object.class.table_name`, so we don't need to provide anything. + +We can even pass heterogeneous sets of objects, if the updates all make sense +for them: + +```ruby +issue_a = Issue.find(..) +issue_b = Issue.find(..) +merge_request = MergeRequest.find(..) + +# Issues two queries +::Gitlab::Database::BulkUpdate.execute(%i[title], { + issue_a => { title: 'A' }, + issue_b => { title: 'B' }, + merge_request => { title: 'B' } +}) +``` + +If your objects do not return the correct model class (perhaps because they are +part of a union), then we need to specify this explicitly in a block: + +```ruby +bazzes = params +objects = Foo.from_union([ + Foo.select("id, 'foo' as object_type").where(quux: true), + Bar.select("id, 'bar' as object_type").where(wibble: true) + ]) +# At this point, all the objects are instances of Foo, even the ones from the +# Bar table +mapping = objects.to_h { |obj| [obj, bazzes[obj.id] } + +# Issues at most 2 queries +::Gitlab::Database::BulkUpdate.execute(%i[baz], mapping) do |obj| + obj.object_type.constantize +end +``` + +## Caveats + +Note that this is a **very low level** tool, and operates on the raw column +values. Enumerations and state fields must be translated into their underlying +representations, for example, and nested associations are not supported. No +validations or hooks will be called. diff --git a/doc/development/database/strings_and_the_text_data_type.md b/doc/development/database/strings_and_the_text_data_type.md index b73dfa859fb..fe8cfa5cd22 100644 --- a/doc/development/database/strings_and_the_text_data_type.md +++ b/doc/development/database/strings_and_the_text_data_type.md @@ -1,3 +1,9 @@ +--- +stage: Enablement +group: Database +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Strings and the Text data type > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/30453) in GitLab 13.0. diff --git a/doc/development/database/table_partitioning.md b/doc/development/database/table_partitioning.md new file mode 100644 index 00000000000..30d0b0a2f5b --- /dev/null +++ b/doc/development/database/table_partitioning.md @@ -0,0 +1,259 @@ +--- +stage: Enablement +group: Database +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Database table partitioning + +Table partitioning is a powerful database feature that allows a table's +data to be split into smaller physical tables that act as a single large +table. If the application is designed to work with partitioning in mind, +there can be multiple benefits, such as: + +- Query performance can be improved greatly, because the database can +cheaply eliminate much of the data from the search space, while still +providing full SQL capabilities. + +- Bulk deletes can be achieved with minimal impact on the database by +dropping entire partitions. This is a natural fit for features that need +to periodically delete data that falls outside the retention window. + +- Administrative tasks like `VACUUM` and index rebuilds can operate on +individual partitions, rather than across a single massive table. + +Unfortunately, not all models fit a partitioning scheme, and there are +significant drawbacks if implemented incorrectly. Additionally, tables +can only be partitioned at their creation, making it nontrivial to apply +partitioning to a busy database. A suite of migration tools are available +to enable backend developers to partition existing tables, but the +migration process is rather heavy, taking multiple steps split across +several releases. Due to the limitations of partitioning and the related +migrations, you should understand how partitioning fits your use case +before attempting to leverage this feature. + +## Determining when to use partitioning + +While partitioning can be very useful when properly applied, it's +imperative to identify if the data and workload of a table naturally fit a +partitioning scheme. There are a few details you'll have to understand +in order to decide if partitioning is a good fit for your particular +problem. + +First, a table is partitioned on a partition key, which is a column or +set of columns which determine how the data will be split across the +partitions. The partition key is used by the database when reading or +writing data, to decide which partition(s) need to be accessed. The +partition key should be a column that would be included in a `WHERE` +clause on almost all queries accessing that table. + +Second, it's necessary to understand the strategy the database will +use to split the data across the partitions. The scheme supported by the +GitLab migration helpers is date-range partitioning, where each partition +in the table contains data for a single month. In this case, the partitioning +key would need to be a timestamp or date column. In order for this type of +partitioning to work well, most queries would need to access data within a +certain date range. + +For a more concrete example, the `audit_events` table can be used, which +was the first table to be partitioned in the application database +(scheduled for deployment with the GitLab 13.5 release). This +table tracks audit entries of security events that happen in the +application. In almost all cases, users want to see audit activity that +occurs in a certain timeframe. As a result, date-range partitioning +was a natural fit for how the data would be accessed. + +To look at this in more detail, imagine a simplified `audit_events` schema: + +```sql +CREATE TABLE audit_events ( + id SERIAL NOT NULL PRIMARY KEY, + author_id INT NOT NULL, + details jsonb NOT NULL, + created_at timestamptz NOT NULL); +``` + +Now imagine typical queries in the UI would display the data within a +certain date range, like a single week: + +```sql +SELECT * +FROM audit_events +WHERE created_at >= '2020-01-01 00:00:00' + AND created_at < '2020-01-08 00:00:00' +ORDER BY created_at DESC +LIMIT 100 +``` + +If the table is partitioned on the `created_at` column the base table would +look like: + +```sql +CREATE TABLE audit_events ( + id SERIAL NOT NULL, + author_id INT NOT NULL, + details jsonb NOT NULL, + created_at timestamptz NOT NULL, + PRIMARY KEY (id, created_at)) +PARTITION BY RANGE(created_at); +``` + +NOTE: **Note:** +The primary key of a partitioned table must include the partition key as +part of the primary key definition. + +And we might have a list of partitions for the table, such as: + +```sql +audit_events_202001 FOR VALUES FROM ('2020-01-01') TO ('2020-02-01') +audit_events_202002 FOR VALUES FROM ('2020-02-01') TO ('2020-03-01') +audit_events_202003 FOR VALUES FROM ('2020-03-01') TO ('2020-04-01') +``` + +Each partition is a separate physical table, with the same structure as +the base `audit_events` table, but contains only data for rows where the +partition key falls in the specified range. For example, the partition +`audit_events_202001` contains rows where the `created_at` column is +greater than or equal to `2020-01-01` and less than `2020-02-01`. + +Now, if we look at the previous example query again, the database can +use the `WHERE` to recognize that all matching rows will be in the +`audit_events_202001` partition. Rather than searching all of the data +in all of the partitions, it can search only the single month's worth +of data in the appropriate partition. In a large table, this can +dramatically reduce the amount of data the database needs to access. +However, imagine a query that does not filter based on the partitioning +key, such as: + +```sql +SELECT * +FROM audit_events +WHERE author_id = 123 +ORDER BY created_at DESC +LIMIT 100 +``` + +In this example, the database can't prune any partitions from the search, +because matching data could exist in any of them. As a result, it has to +query each partition individually, and aggregate the rows into a single result +set. Since `author_id` would be indexed, the performance impact could +likely be acceptable, but on more complex queries the overhead can be +substantial. Partitioning should only be leveraged if the access patterns +of the data support the partitioning strategy, otherwise performance will +suffer. + +## Partitioning a table + +Unfortunately, tables can only be partitioned at their creation, making +it nontrivial to apply to a busy database. A suite of migration +tools have been developed to enable backend developers to partition +existing tables. This migration process takes multiple steps which must +be split across several releases. + +### Caveats + +The partitioning migration helpers work by creating a partitioned duplicate +of the original table and using a combination of a trigger and a background +migration to copy data into the new table. Changes to the original table +schema can be made in parallel with the partitioning migration, but they +must take care to not break the underlying mechanism that makes the migration +work. For example, if a column is added to the table that is being +partitioned, both the partitioned table and the trigger definition need to +be updated to match. + +### Step 1: Creating the partitioned copy (Release N) + +The first step is to add a migration to create the partitioned copy of +the original table. This migration will also create the appropriate +partitions based on the data in the original table, and install a +trigger that will sync writes from the original table into the +partitioned copy. + +An example migration of partitioning the `audit_events` table by its +`created_at` column would look like: + +```ruby +class PartitionAuditEvents < ActiveRecord::Migration[6.0] + include Gitlab::Database::PartitioningMigrationHelpers + + def up + partition_table_by_date :audit_events, :created_at + end + + def down + drop_partitioned_table_for :audit_events + end +end +``` + +Once this has executed, any inserts, updates or deletes in the +original table will also be duplicated in the new table. For updates and +deletes, the operation will only have an effect if the corresponding row +exists in the partitioned table. + +### Step 2: Backfill the partitioned copy (Release N) + +The second step is to add a post-deployment migration that will schedule +the background jobs that will backfill existing data from the original table +into the partitioned copy. + +Continuing the above example, the migration would look like: + +```ruby +class BackfillPartitionAuditEvents < ActiveRecord::Migration[6.0] + include Gitlab::Database::PartitioningMigrationHelpers + + def up + enqueue_partitioning_data_migration :audit_events + end + + def down + cleanup_partitioning_data_migration :audit_events + end +end +``` + +This step uses the same mechanism as any background migration, so you +may want to read the [Background Migration](../background_migrations.md) +guide for details on that process. Background jobs are scheduled every +2 minutes and copy `50_000` records at a time, which can be used to +estimate the timing of the background migration portion of the +partitioning migration. + +### Step 3: Post-backfill cleanup (Release N+1) + +The third step must occur at least one release after the release that +includes the background migration. This gives time for the background +migration to execute properly in self-managed installations. In this step, +add another post-deployment migration that will cleanup after the +background migration. This includes forcing any remaining jobs to +execute, and copying data that may have been missed, due to dropped or +failed jobs. + +Once again, continuing the example, this migration would look like: + +```ruby +class CleanupPartitionedAuditEventsBackfill < ActiveRecord::Migration[6.0] + include Gitlab::Database::PartitioningMigrationHelpers + + def up + finalize_backfilling_partitioned_table :audit_events + end + + def down + # no op + end +end +``` + +After this migration has completed, the original table and partitioned +table should contain identical data. The trigger installed on the +original table guarantees that the data will remain in sync going +forward. + +### Step 4: Swap the partitioned and non-partitioned tables (Release N+1) + +The final step of the migration will make the partitioned table ready +for use by the application. This section will be updated when the +migration helper is ready, for now development can be followed in the +[Tracking Issue](https://gitlab.com/gitlab-org/gitlab/-/issues/241267). diff --git a/doc/development/database_debugging.md b/doc/development/database_debugging.md index 61e8ac60bfe..ebd57df498e 100644 --- a/doc/development/database_debugging.md +++ b/doc/development/database_debugging.md @@ -1,3 +1,9 @@ +--- +stage: Enablement +group: Database +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Troubleshooting and Debugging Database This section is to help give some copy-pasta you can use as a reference when you @@ -19,7 +25,7 @@ If you just want to delete everything and start over with an empty DB (approxima bundle exec rake db:reset RAILS_ENV=development ``` -If you just want to delete everything and start over with dummy data (approximately 4 minutes). This +If you just want to delete everything and start over with sample data (approximately 4 minutes). This also does `db:reset` and runs DB-specific migrations: ```shell diff --git a/doc/development/database_query_comments.md b/doc/development/database_query_comments.md index 77943f2b261..ccaaadef4f4 100644 --- a/doc/development/database_query_comments.md +++ b/doc/development/database_query_comments.md @@ -1,3 +1,9 @@ +--- +stage: Enablement +group: Database +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Database query comments with Marginalia The [Marginalia gem](https://github.com/basecamp/marginalia) is used to add diff --git a/doc/development/database_review.md b/doc/development/database_review.md index f56ffdbad21..3f5f36b0b6e 100644 --- a/doc/development/database_review.md +++ b/doc/development/database_review.md @@ -1,3 +1,9 @@ +--- +stage: Enablement +group: Database +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Database Review Guidelines This page is specific to database reviews. Please refer to our @@ -21,7 +27,7 @@ A database review is required for: database review. - Changes in usage data metrics that use `count` and `distinct_count`. These metrics could have complex queries over large tables. - See the [Telemetry Guide](telemetry/usage_ping.md#implementing-usage-ping) + See the [Product Analytics Guide](product_analytics/usage_ping.md#implementing-usage-ping) for implementation details. A database reviewer is expected to look out for obviously complex @@ -184,10 +190,6 @@ test its execution using `CREATE INDEX CONCURRENTLY` in the `#database-lab` Slac - [Check query plans](understanding_explain_plans.md) and suggest improvements to queries (changing the query, schema or adding indexes and similar) - General guideline is for queries to come in below 100ms execution time - - If queries rely on prior migrations that are not present yet on production - (eg indexes, columns), you can use a [one-off instance from the restore - pipeline](https://ops.gitlab.net/gitlab-com/gl-infra/gitlab-restore/postgres-gprd) - in order to establish a proper testing environment. If you don't have access to this project, reach out to #database on Slack to get advice on how to proceed. - Avoid N+1 problems and minimalize the [query count](merge_request_performance_guidelines.md#query-counts). ### Timing guidelines for migrations diff --git a/doc/development/db_dump.md b/doc/development/db_dump.md index 4095932e44c..b16cd4b08d1 100644 --- a/doc/development/db_dump.md +++ b/doc/development/db_dump.md @@ -1,3 +1,9 @@ +--- +stage: Enablement +group: Database +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Importing a database dump into a staging environment Sometimes it is useful to import the database from a production environment diff --git a/doc/development/distributed_tracing.md b/doc/development/distributed_tracing.md index 9a4c15c5c19..952435150d6 100644 --- a/doc/development/distributed_tracing.md +++ b/doc/development/distributed_tracing.md @@ -1,15 +1,12 @@ --- stage: Monitor -group: APM +group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # Distributed Tracing - development guidelines -NOTE: **Note:** -Distributed Tracing in GitLab is currently considered **experimental**, as it has not yet been tested at scale on GitLab.com. - -GitLab is instrumented for distributed tracing. +GitLab is instrumented for distributed tracing. Distributed Tracing in GitLab is currently considered **experimental**, as it has not yet been tested at scale on GitLab.com. According to [Open Tracing](https://opentracing.io/docs/overview/what-is-tracing/): diff --git a/doc/development/documentation/feature_flags.md b/doc/development/documentation/feature_flags.md index a4a6ee2fa0f..0f03ceeb4b5 100644 --- a/doc/development/documentation/feature_flags.md +++ b/doc/development/documentation/feature_flags.md @@ -52,7 +52,7 @@ For feature flags disabled by default, if they can be used by end users: do not say anything about it. - Say whether it's recommended for production use. - Document how to enable and disable it. -- Add a warning to the user saying that the feature is disabled. +- Add a warning to the user saying that the feature might be disabled. For example, for a feature disabled by default, disabled on GitLab.com, cannot be enabled for a single project, and is not ready for production use: @@ -250,7 +250,7 @@ be enabled by project, and is ready for production use: > - [Introduced](link-to-issue) in GitLab 12.0. > - It's [deployed behind a feature flag](<replace with path to>/user/feature_flags.md), enabled by default. > - It's enabled on GitLab.com. -> - It can be enabled or disable for a single project. +> - It can be enabled or disabled for a single project. > - It's recommended for production use. > - For GitLab self-managed instances, GitLab administrators can opt to [disable it](#anchor-to-section). **(CORE ONLY)** diff --git a/doc/development/documentation/graphql_styleguide.md b/doc/development/documentation/graphql_styleguide.md new file mode 100644 index 00000000000..11e6b159359 --- /dev/null +++ b/doc/development/documentation/graphql_styleguide.md @@ -0,0 +1,92 @@ +--- +type: reference, dev +stage: none +group: Development +info: "See the Technical Writers assigned to Development Guidelines: https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments-to-development-guidelines" +description: "Writing styles, markup, formatting, and other standards for GraphQL API's GitLab Documentation." +--- + +# GraphQL API + +GraphQL APIs are different from [RESTful APIs](restful_api_styleguide.md). Reference +information is generated in our [GraphQL reference](../../api/graphql/reference/index.md). + +However, it's helpful to include examples on how to use GraphQL for different +*use cases*, with samples that readers can use directly in the +[GraphiQL explorer](../api_graphql_styleguide.md#graphiql). + +This section describes the steps required to add your GraphQL examples to +GitLab documentation. + +## Add a dedicated GraphQL page + +To create a dedicated GraphQL page, create a new `.md` file in the +`doc/api/graphql/` directory. Give that file a functional name, such as +`import_from_specific_location.md`. + +## Start the page with an explanation + +Include a page title that describes the GraphQL functionality in a few words, +such as: + +```markdown +# Search for [substitute kind of data] +``` + +Describe the search. One sentence may be all you need. More information may +help readers learn how to use the example for their GitLab deployments. + +## Include a procedure using the GraphiQL explorer + +The GraphiQL explorer can help readers test queries with working deployments. +Set up the section with the following: + +- Use the following title: + + ```markdown + ## Set up the GraphiQL explorer + ``` + +- Include a code block with the query that anyone can include in their + instance of the GraphiQL explorer: + + ````markdown + ```graphql + query { + <insert queries here> + } + ``` + ```` + +- Tell the user what to do: + + ```markdown + 1. Open the GraphiQL explorer tool in the following URL: `https://gitlab.com/-/graphql-explorer`. + 1. Paste the `query` listed above into the left window of your GraphiQL explorer tool. + 1. Select **Play** to get the result shown here: + ``` + +- Include a screenshot of the result in the GraphiQL explorer. Follow the naming + convention described in the [Save the image](styleguide.md#save-the-image) section of the Documentation style guide. +- Follow up with an example of what you can do with the output. Make sure the + example is something that readers can do on their own deployments. +- Include a link to the [GraphQL API resources](../../api/graphql/reference/index.md). + +## Add the GraphQL example to the global navigation + +You should include a link for your new document in the global navigation (the list on the +left side of the documentation website). To do so, open a second MR, against the +[GitLab documentation repository](https://gitlab.com/gitlab-org/gitlab-docs/). + +We store our global navigation in the [`default-nav.yaml`](https://gitlab.com/gitlab-org/gitlab-docs/-/blob/master/content/_data/default-nav.yaml) file, in the +`content/_data` subdirectory. You can find the GraphQL section under the +following line: + +```yaml +- category_title: GraphQL +``` + +Be aware that CI tests for that second MR will fail with a bad link until the +main MR that adds the new GraphQL page is merged. Therefore, only merge the MR against the +[`gitlab-docs`](https://gitlab.com/gitlab-org/gitlab-docs) repository after the content has +been merged and live on `docs.gitlab.com`. diff --git a/doc/development/documentation/index.md b/doc/development/documentation/index.md index d6f24d6374d..e51f966ee6e 100644 --- a/doc/development/documentation/index.md +++ b/doc/development/documentation/index.md @@ -1,4 +1,7 @@ --- +stage: none +group: Documentation Guidelines +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers description: Learn how to contribute to GitLab Documentation. --- @@ -52,9 +55,9 @@ docs-only merge requests using the following guide: [Contributions to GitLab docs](workflow.md) are welcome from the entire GitLab community. -To ensure that GitLab docs are current, there are special processes and responsibilities for all [feature changes](feature-change-workflow.md), that is development work that impacts the appearance, usage, or administration of a feature. +To ensure that GitLab docs are current, there are special processes and responsibilities for all [feature changes](workflow.md), that is development work that impacts the appearance, usage, or administration of a feature. -However, anyone can contribute [documentation improvements](improvement-workflow.md) that are not associated with a feature change. For example, adding a new doc on how to accomplish a use case that's already possible with GitLab or with third-party tools and GitLab. +However, anyone can contribute [documentation improvements](workflow.md) that are not associated with a feature change. For example, adding a new doc on how to accomplish a use case that's already possible with GitLab or with third-party tools and GitLab. ## Markdown and styles @@ -128,7 +131,7 @@ The following metadata should be added when a page is moved to another location: - `redirect_to`: The relative path and filename (with an `.md` extension) of the location to which visitors should be redirected for a moved page. - [Learn more](#changing-document-location). + [Learn more](#move-or-rename-a-page). - `disqus_identifier`: Identifier for Disqus commenting system. Used to keep comments with a page that's been moved to a new URL. [Learn more](#redirections-for-pages-with-disqus-comments). @@ -156,17 +159,18 @@ Nanoc layout), which will be displayed at the top of the page if defined: [algorithm](https://gitlab.com/gitlab-org/gitlab-docs/-/blob/master/lib/helpers/reading_time.rb) to calculate the reading time based on the number of words. -## Changing document location +## Move or rename a page + +Moving or renaming a document is the same as changing its location. This process +requires specific steps to ensure that visitors can find the new +documentation page, whether they're using `/help` from a GitLab instance or by +visiting <https://docs.gitlab.com>. -Changing a document's location requires specific steps to ensure that -users can seamlessly access the new doc page, whether they are accessing content -on a GitLab instance domain at `/help` or at <https://docs.gitlab.com>. Be sure to assign a -technical writer if you have any questions during the process (such as -whether the move is necessary), and ensure that a technical writer reviews this -change prior to merging. +Be sure to assign a technical writer to a page move or rename MR. Technical +Writers can help with any questions and can review your change. -If you indeed need to change a document's location, do not remove the old -document, but instead replace all of its content with the following: +To change a document's location, don't remove the old document, but instead +replace all of its content with the following: ```markdown --- @@ -176,14 +180,18 @@ redirect_to: '../path/to/file/index.md' This document was moved to [another location](../path/to/file/index.md). ``` -Where `../path/to/file/index.md` is usually the relative path to the old document. +Replace `../path/to/file/index.md` with the relative path to the old document. + +The `redirect_to` variable supports both full and relative URLs; for example: -The `redirect_to` variable supports both full and relative URLs, for example -`https://docs.gitlab.com/ee/path/to/file.html`, `../path/to/file.html`, `path/to/file.md`. -It ensures that the redirect will work for <https://docs.gitlab.com> and any `*.md` paths -will be compiled to `*.html`. -The new line underneath the front matter informs the user that the document -changed location and is useful for someone that browses that file from the repository. +- `https://docs.gitlab.com/ee/path/to/file.html` +- `../path/to/file.html` +- `path/to/file.md` + +The redirect works for <https://docs.gitlab.com>, and any `*.md` paths are +changed to `*.html`. The description line following the `redirect_to` code +informs the visitor that the document changed location if the redirect process +doesn't complete successfully. For example, if you move `doc/workflow/lfs/index.md` to `doc/administration/lfs.md`, then the steps would be: @@ -208,9 +216,8 @@ For example, if you move `doc/workflow/lfs/index.md` to git grep -n "lfs/lfs_administration" ``` -NOTE: **Note:** -If the document being moved has any Disqus comments on it, there are extra steps -to follow documented just [below](#redirections-for-pages-with-disqus-comments). +1. If the document being moved has any Disqus comments on it, follow the steps + described in [Redirections for pages with Disqus comments](#redirections-for-pages-with-disqus-comments). Things to note: @@ -241,7 +248,8 @@ using the old URL as value. For example, let's say we moved the document available under `https://docs.gitlab.com/my-old-location/README.html` to a new location, `https://docs.gitlab.com/my-new-location/index.html`. -Into the **new document** front matter, we add the following: +Into the **new document** front matter, we add the following information. You must +include the file name in the `disqus_identifier` URL, even if it's `index.html` or `README.html`. ```yaml --- @@ -249,9 +257,6 @@ disqus_identifier: 'https://docs.gitlab.com/my-old-location/README.html' --- ``` -Note: it is necessary to include the file name in the `disqus_identifier` URL, -even if it's `index.html` or `README.html`. - ## Merge requests for GitLab documentation Before getting started, make sure you read the introductory section @@ -267,9 +272,8 @@ represents a good-faith effort to follow the template and style standards, and is believed to be accurate. Further needs for what would make the doc even better should be immediately addressed -in a follow-up MR or issue. +in a follow-up merge request or issue. -NOTE: **Note:** If the release version you want to add the documentation to has already been frozen or released, use the label `~"Pick into X.Y"` to get it merged into the correct release. Avoid picking into a past release as much as you can, as @@ -392,8 +396,7 @@ You will need at least Maintainer permissions to be able to run it.  -NOTE: **Note:** -You will need to push a branch to those repositories, it doesn't work for forks. +You must push a branch to those repositories, as it doesn't work for forks. The `review-docs-deploy*` job will: @@ -410,17 +413,16 @@ minutes and it should appear online, otherwise you can check the status of the remote pipeline from the link in the merge request's job output. If the pipeline failed or got stuck, drop a line in the `#docs` chat channel. -TIP: **Tip:** -Someone with no merge rights to the GitLab projects (think of forks from -contributors) cannot run the manual job. In that case, you can -ask someone from the GitLab team who has the permissions to do that for you. - -NOTE: **Note:** Make sure that you always delete the branch of the merge request you were working on. If you don't, the remote docs branch won't be removed either, and the server where the Review Apps are hosted will eventually be out of disk space. +TIP: **Tip:** +Someone with no merge rights to the GitLab projects (think of forks from +contributors) cannot run the manual job. In that case, you can +ask someone from the GitLab team who has the permissions to do that for you. + ### Troubleshooting review apps In case the review app URL returns 404, follow these steps to debug: @@ -462,7 +464,7 @@ If you want to know the in-depth details, here's what's really happening: The following GitLab features are used among others: - [Manual actions](../../ci/yaml/README.md#whenmanual) -- [Multi project pipelines](../../ci/multi_project_pipeline_graphs.md) +- [Multi project pipelines](../../ci/multi_project_pipelines.md) - [Review Apps](../../ci/review_apps/index.md) - [Artifacts](../../ci/yaml/README.md#artifacts) - [Specific runner](../../ci/runners/README.md#prevent-a-specific-runner-from-being-enabled-for-other-projects) @@ -668,7 +670,7 @@ build pipelines: ``` We recommend installing the version of `markdownlint-cli` currently used in the documentation - linting [Docker image](https://gitlab.com/gitlab-org/gitlab-docs/-/blob/master/dockerfiles/Dockerfile.gitlab-docs-lint#L38). + linting [Docker image](https://gitlab.com/gitlab-org/gitlab-docs/-/blob/master/.gitlab-ci.yml#L420). 1. Install [`vale`](https://github.com/errata-ai/vale/releases). For example, to install using `brew` for macOS, run: @@ -678,7 +680,7 @@ build pipelines: ``` We recommend installing the version of Vale currently used in the documentation linting - [Docker image](https://gitlab.com/gitlab-org/gitlab-docs/-/blob/master/dockerfiles/Dockerfile.gitlab-docs-lint#L16). + [Docker image](https://gitlab.com/gitlab-org/gitlab-docs/-/blob/master/.gitlab-ci.yml#L419). In addition to using markdownlint and Vale at the command line, these tools can be [integrated with your code editor](#configure-editors). diff --git a/doc/development/documentation/restful_api_styleguide.md b/doc/development/documentation/restful_api_styleguide.md new file mode 100644 index 00000000000..b12578b5d98 --- /dev/null +++ b/doc/development/documentation/restful_api_styleguide.md @@ -0,0 +1,180 @@ +--- +type: reference, dev +stage: none +group: Development +info: "See the Technical Writers assigned to Development Guidelines: https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments-to-development-guidelines" +description: "Writing styles, markup, formatting, and other standards for GitLab's RESTful APIs." +--- + +# RESTful API + +REST API resources are documented in Markdown under +[`/doc/api`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/doc/api). Each +resource has its own Markdown file, which is linked from `api_resources.md`. + +When modifying the Markdown, also update the corresponding +[OpenAPI definition](https://gitlab.com/gitlab-org/gitlab/-/tree/master/doc/api/openapi) +if one exists for the resource. If not, consider creating one. Match the latest +[OpenAPI 3.0.x specification](https://swagger.io/specification/). (For more +information, see the discussion in this +[issue](https://gitlab.com/gitlab-org/gitlab/-/issues/16023#note_370901810).) + +In the Markdown doc for a resource (AKA endpoint): + +- Every method must have the REST API request. For example: + + ```plaintext + GET /projects/:id/repository/branches + ``` + +- Every method must have a detailed [description of the parameters](#method-description). +- Every method must have a cURL example. +- Every method must have a response body (in JSON format). + +## API topic template + +The following can be used as a template to get started: + +````markdown +## Descriptive title + +One or two sentence description of what endpoint does. + +```plaintext +METHOD /endpoint +``` + +| Attribute | Type | Required | Description | +|:------------|:---------|:---------|:----------------------| +| `attribute` | datatype | yes/no | Detailed description. | +| `attribute` | datatype | yes/no | Detailed description. | + +Example request: + +```shell +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/endpoint?parameters" +``` + +Example response: + +```json +[ + { + } +] +``` +```` + +## Method description + +Use the following table headers to describe the methods. Attributes should +always be in code blocks using backticks (`` ` ``). + +```markdown +| Attribute | Type | Required | Description | +|:----------|:-----|:---------|:------------| +``` + +Rendered example: + +| Attribute | Type | Required | Description | +|:----------|:-------|:---------|:--------------------| +| `user` | string | yes | The GitLab username. | + +## cURL commands + +- Use `https://gitlab.example.com/api/v4/` as an endpoint. +- Wherever needed use this personal access token: `<your_access_token>`. +- Always put the request first. `GET` is the default so you don't have to + include it. +- Wrap the URL in double quotes (`"`). +- Prefer to use examples using the personal access token and don't pass data of + username and password. + +| Methods | Description | +|:------------------------------------------- |:------------------------------------------------------| +| `--header "PRIVATE-TOKEN: <your_access_token>"` | Use this method as is, whenever authentication needed. | +| `--request POST` | Use this method when creating new objects | +| `--request PUT` | Use this method when updating existing objects | +| `--request DELETE` | Use this method when removing existing objects | + +## cURL Examples + +The following sections include a set of [cURL](https://curl.haxx.se) examples +you can use in the API documentation. + +CAUTION: **Caution:** +Do not use information for real users, URLs, or tokens. For documentation, refer to our +relevant style guide sections on [Fake user information](styleguide.md#fake-user-information), +[Fake URLs](styleguide.md#fake-urls), and [Fake tokens](styleguide.md#fake-tokens). + +### Simple cURL command + +Get the details of a group: + +```shell +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/gitlab-org" +``` + +### cURL example with parameters passed in the URL + +Create a new project under the authenticated user's namespace: + +```shell +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects?name=foo" +``` + +### Post data using cURL's `--data` + +Instead of using `--request POST` and appending the parameters to the URI, you +can use cURL's `--data` option. The example below will create a new project +`foo` under the authenticated user's namespace. + +```shell +curl --data "name=foo" --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects" +``` + +### Post data using JSON content + +NOTE: **Note:** +In this example we create a new group. Watch carefully the single and double +quotes. + +```shell +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --header "Content-Type: application/json" --data '{"path": "my-group", "name": "My group"}' "https://gitlab.example.com/api/v4/groups" +``` + +### Post data using form-data + +Instead of using JSON or urlencode you can use multipart/form-data which +properly handles data encoding: + +```shell +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --form "title=ssh-key" --form "key=ssh-rsa AAAAB3NzaC1yc2EA..." "https://gitlab.example.com/api/v4/users/25/keys" +``` + +The above example is run by and administrator and will add an SSH public key +titled `ssh-key` to user's account which has an ID of 25. + +### Escape special characters + +Spaces or slashes (`/`) may sometimes result to errors, thus it is recommended +to escape them when possible. In the example below we create a new issue which +contains spaces in its title. Observe how spaces are escaped using the `%20` +ASCII code. + +```shell +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/42/issues?title=Hello%20Dude" +``` + +Use `%2F` for slashes (`/`). + +### Pass arrays to API calls + +The GitLab API sometimes accepts arrays of strings or integers. For example, to +exclude specific users when requesting a list of users for a project, you would +do something like this: + +```shell +curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" --data "skip_users[]=<user_id>" --data "skip_users[]=<user_id>" "https://gitlab.example.com/api/v4/projects/<project_id>/users" +``` diff --git a/doc/development/documentation/site_architecture/global_nav.md b/doc/development/documentation/site_architecture/global_nav.md index 22d97a9e2cf..9fce9b4e4b3 100644 --- a/doc/development/documentation/site_architecture/global_nav.md +++ b/doc/development/documentation/site_architecture/global_nav.md @@ -324,7 +324,6 @@ There are three main considerations on the logic built for the nav: - `https://docs.gitlab.com/ee/` - `https://docs.gitlab.com/omnibus/` - `https://docs.gitlab.com/runner/` - - `https://docs.gitlab.com/debug/` - `https://docs.gitlab.com/*` - [EE-only](#ee-only-docs): documentation only available in `/ee/`, not on `/ce/`, e.g.: - `https://docs.gitlab.com/ee/user/group/epics/` @@ -342,8 +341,8 @@ all the nav links to other pages: <% dir = @item.identifier.to_s[%r{(?<=/)[^/]+}] %> ``` -For instance, for `https://docs.gitlab.com/ce/user/index.html`, -`dir` == `ce`, and for `https://docs.gitlab.com/omnibus/README.html`, +For instance, for `https://docs.gitlab.com/ee/user/index.html`, +`dir` == `ee`, and for `https://docs.gitlab.com/omnibus/README.html`, `dir` == `omnibus`. #### Default URL diff --git a/doc/development/documentation/site_architecture/release_process.md b/doc/development/documentation/site_architecture/release_process.md index 98bb116aba6..d04d34ff786 100644 --- a/doc/development/documentation/site_architecture/release_process.md +++ b/doc/development/documentation/site_architecture/release_process.md @@ -121,10 +121,11 @@ versions (stable branches `X.Y` of the `gitlab-docs` project): pipelines succeed: NOTE: **Note:** - The `release-X-Y` branch needs to be present locally, otherwise the Rake - task will abort. + The `release-X-Y` branch needs to be present locally, + and you need to have switched to it, otherwise the Rake task will fail. ```shell + git checkout release-X-Y ./bin/rake release:dropdowns ``` diff --git a/doc/development/documentation/styleguide.md b/doc/development/documentation/styleguide.md index 984c64b9e9e..6075124ef40 100644 --- a/doc/development/documentation/styleguide.md +++ b/doc/development/documentation/styleguide.md @@ -251,7 +251,7 @@ The table below shows what kind of documentation goes where. | `doc/legal/` | Legal documents about contributing to GitLab. | | `doc/install/` | Contains instructions for installing GitLab. | | `doc/update/` | Contains instructions for updating GitLab. | -| `doc/topics/` | Indexes per topic (`doc/topics/topic-name/index.md`): all resources for that topic. | +| `doc/topics/` | Indexes per topic (`doc/topics/topic_name/index.md`): all resources for that topic. | ### Work with directories and files @@ -287,9 +287,8 @@ The table below shows what kind of documentation goes where. 1. The `doc/topics/` directory holds topic-related technical content. Create `doc/topics/topic_name/subtopic_name/index.md` when subtopics become necessary. General user- and admin- related documentation, should be placed accordingly. -1. The directories `/workflow/`, `/university/`, and `/articles/` have been - *deprecated* and the majority their documentation has been moved to their - correct location in small iterations. +1. The `/university/` directory is *deprecated* and the majority of its documentation + has been moved. If you are unsure where to place a document or a content addition, this should not stop you from authoring and contributing. You can use your best judgment and @@ -369,7 +368,7 @@ create an issue or an MR to propose a change to the user interface text. - milestones - reorder issues - runner, runners, shared runners - - a to-do, to-dos + - a to-do item, to dos - *Some features are capitalized*, typically nouns naming GitLab-specific capabilities or tools. For example: - GitLab CI/CD @@ -410,7 +409,7 @@ Use forms of *sign in*, instead of *log in* or *login*. For example: Exceptions to this rule include the concept of *single sign-on* and references to user interface elements. For example: -- To sign in to product X, enter your credentials, and then click **Log in**. +- To sign in to product X, enter your credentials, and then select **Log in**. ### Inclusive language @@ -418,8 +417,11 @@ We strive to create documentation that is inclusive. This section includes guidance and examples in the following categories: - [Gender-specific wording](#avoid-gender-specific-wording). + (Tested in [`InclusionGender.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/InclusionGender.yml).) - [Ableist language](#avoid-ableist-language). + (Tested in [`InclusionAbleism.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/InclusionAbleism.yml).) - [Cultural sensitivity](#culturally-sensitive-language). + (Tested in [`InclusionCultural.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/InclusionCultural.yml).) We write our developer documentation with inclusivity and diversity in mind. This page is not an exhaustive reference, but describes some general guidelines and @@ -433,12 +435,16 @@ a gender-neutral pronoun. Avoid the use of gender-specific pronouns, unless referring to a specific person. +<!-- vale gitlab.InclusionGender = NO --> + | Use | Avoid | |-----------------------------------|---------------------------------| | People, humanity | Mankind | | GitLab Team Members | Manpower | | You can install; They can install | He can install; She can install | +<!-- vale gitlab.InclusionGender = YES --> + If you need to set up [Fake user information](#fake-user-information), use diverse or non-gendered names with common surnames. @@ -446,6 +452,8 @@ diverse or non-gendered names with common surnames. Avoid terms that are also used in negative stereotypes for different groups. +<!-- vale gitlab.InclusionAbleism = NO --> + | Use | Avoid | |------------------------|----------------------| | Check for completeness | Sanity check | @@ -455,6 +463,8 @@ Avoid terms that are also used in negative stereotypes for different groups. | Active/Inactive | Enabled/Disabled | | On/Off | Enabled/Disabled | +<!-- vale gitlab.InclusionAbleism = YES --> + Credit: [Avoid ableist language](https://developers.google.com/style/inclusive-documentation#ableist-language) in the Google Developer Style Guide. @@ -464,13 +474,57 @@ Avoid terms that reflect negative cultural stereotypes and history. In most cases, you can replace terms such as `master` and `slave` with terms that are more precise and functional, such as `primary` and `secondary`. +<!-- vale gitlab.InclusionCultural = NO --> + | Use | Avoid | |----------------------|-----------------------| | Primary / secondary | Master / slave | | Allowlist / denylist | Blacklist / whitelist | +<!-- vale gitlab.InclusionCultural = YES --> + For more information see the following [Internet Draft specification](https://tools.ietf.org/html/draft-knodel-terminology-02). +### Fake user information + +You may need to include user information in entries such as a REST call or user profile. +**Do not** use real user information or email addresses in GitLab documentation. For email +addresses and names, do use: + +- **Email addresses**: Use an email address ending in `example.com`. +- **Names**: Use strings like `example_username`. Alternatively, use diverse or + non-gendered names with common surnames, such as `Sidney Jones`, `Zhang Wei`, + or `Alex Garcia`. + +### Fake URLs + +When including sample URLs in the documentation, use: + +- `example.com` when the domain name is generic. +- `gitlab.example.com` when referring to self-managed instances of GitLab. + +### Fake tokens + +There may be times where a token is needed to demonstrate an API call using +cURL or a variable used in CI. It is strongly advised not to use real tokens in +documentation even if the probability of a token being exploited is low. + +You can use the following fake tokens as examples: + +| Token type | Token value | +|:----------------------|:-------------------------------------------------------------------| +| Private user token | `<your_access_token>` | +| Personal access token | `n671WNGecHugsdEDPsyo` | +| Application ID | `2fcb195768c39e9a94cec2c2e32c59c0aad7a3365c10892e8116b5d83d4096b6` | +| Application secret | `04f294d1eaca42b8692017b426d53bbc8fe75f827734f0260710b83a556082df` | +| CI/CD variable | `Li8j-mLUVA3eZYjPfd_H` | +| Specific runner token | `yrnZW46BrtBFqM7xDzE7dddd` | +| Shared runner token | `6Vk7ZsosqQyfreAxXTZr` | +| Trigger token | `be20d8dcc028677c931e04f3871a9b` | +| Webhook secret token | `6XhDroRcYPM5by_h-HLY` | +| Health check token | `Tu7BgjR9qeZTEyRzGG2P` | +| Request profile token | `7VgpS4Ax5utVD2esNstz` | + ### Language to avoid When creating documentation, limit or avoid the use of the following verb @@ -529,6 +583,7 @@ tenses, words, and phrases: - Avoid the use of [racially-insensitive terminology or phrases](https://www.marketplace.org/2020/06/17/tech-companies-update-language-to-avoid-offensive-terms/). For example: - Use *primary* and *secondary* for database and server relationships. - Use *allowlist* and *denylist* to describe access control lists. +- Avoid the word *please*. For details, see [the Microsoft style guide](https://docs.microsoft.com/en-us/style-guide/a-z-word-list-term-collections/p/please). ### Word usage clarifications @@ -832,8 +887,10 @@ When creating tables of lists of features (such as whether or not features are available to certain roles on the [Permissions](../../user/permissions.md#project-members-permissions) page), use the following phrases (based on the SVG icons): -- *No*: **{dotted-circle}** No -- *Yes*: **{check-circle}** Yes +| Option | Markdown | Displayed result | +|--------|--------------------------|------------------------| +| No | `**{dotted-circle}** No` | **{dotted-circle}** No | +| Yes | `**{check-circle}** Yes` | **{check-circle}** Yes | ## Quotes @@ -890,8 +947,8 @@ search engine optimization (SEO), use the imperative, where possible. For guidelines on capitalizing headings, see the section on [capitalization](#capitalization). NOTE: **Note:** -If you change an existing title, be careful. Any such changes may affect not -only [links](#anchor-links) within the page, but may also affect links to the +If you change an existing title, be careful. These changes might affect not +only [links](#anchor-links) within the page, but might also affect links to the GitLab documentation from both the GitLab application and external sites. ### Anchor links @@ -1095,14 +1152,26 @@ document to ensure it links to the most recent version of the file. ## Navigation -To indicate the steps of navigation through the user interface: +When documenting navigation through the user interface: -- Use the exact word as shown in the UI, including any capital letters as-is. +- Use the exact wording as shown in the UI, including any capital letters as-is. - Use bold text for navigation items and the char "greater than" (`>`) as a - separator (for example, `Navigate to your project's **Settings > CI/CD**` ). + separator. For example: `Navigate to your project's **Settings > CI/CD**`. - If there are any expandable menus, make sure to mention that the user needs to - expand the tab to find the settings you're referring to (for example, - `Navigate to your project's **Settings > CI/CD** and expand **General pipelines**`). + expand the tab to find the settings you're referring to. For example: + `Navigate to your project's **Settings > CI/CD** and expand **General pipelines**`. + +### Navigational elements + +Use the following terms when referring to the main GitLab user interface +elements: + +- **Top menu**: This is the top menu that spans the width of the user interface. + It includes the GitLab logo, search field, counters, and the user's avatar. +- **Left sidebar**: This is the navigation sidebar on the left of the user + interface, specific to the project or group. +- **Right sidebar**: This is the navigation sidebar on the right of the user + interface, specific to the open issue, merge request, or epic. ## Images @@ -1162,9 +1231,6 @@ that: - Are accurate, succinct, and unique. - Don't use *image of …* or *graphic of…* to describe the image. -Also, if a heading immediately follows an image, be sure to add three dashes -(`---`) between the image and the heading. - ### Remove image shadow All images displayed on the [GitLab documentation site](https://docs.gitlab.com) @@ -1249,7 +1315,7 @@ reviewed and approved by a technical writer. 1. In YouTube, visit the video URL you want to display. Copy the regular URL from your browser (`https://www.youtube.com/watch?v=VIDEO-ID`) and replace the video title and link in the line under `<div class="video-fallback">`. -1. In YouTube, click **Share**, and then click **Embed**. +1. In YouTube, select **Share**, and then select **Embed**. 1. Copy the `<iframe>` source (`src`) **URL only** (`https://www.youtube.com/embed/VIDEO-ID`), and paste it, replacing the content of the `src` field in the @@ -1292,6 +1358,9 @@ the documentation site, but will be displayed on GitLab's `/help`. added to code blocks. To make things easier for the user, always add a full code block for things that can be useful to copy and paste, as they can easily do it with the button on code blocks. +- HTTP methods (`HTTP POST`) and HTTP status codes, both full (`404 File Not Found`) + and abbreviated (`404`), should be wrapped in inline code blocks when used in sentences. + For example: Send a `DELETE` request to delete the runner. Send a `POST` request to create one. - Add a blank line above and below code blocks. - When providing a shell command and its output, prefix the shell command with `$` and leave a blank line between the command and the output. @@ -1416,17 +1485,17 @@ interface: Use an icon when you find yourself having to describe an interface element. For example: -- Do: Click the Admin Area icon ( **{admin}** ). -- Don't: Click the Admin Area icon (the wrench icon). +- Do: Select the Admin Area icon ( **{admin}** ). +- Don't: Select the Admin Area icon (the wrench icon). ## Alert boxes When you need to call special attention to particular sentences, use the following markup to create highlighted alert boxes. -Note that the alert boxes only work for one paragraph only. Multiple paragraphs, -lists, headers and so on, will not render correctly. For multiple lines, use -[blockquotes](#blockquotes) instead. +Alert boxes work for one paragraph only. Multiple paragraphs, lists, and headers +won't render correctly. For multiple lines, use [blockquotes](#blockquotes) +instead. Alert boxes render only on the GitLab documentation site (<https://docs.gitlab.com>). Within GitLab itself, they will appear as plain Markdown text (like the examples @@ -1444,23 +1513,20 @@ guidelines, but for consistency you should try to use these values: ### Note -Notes catch the eye of most readers, and therefore should be used very sparingly. -In most cases, content considered for a note should be included: +Notes indicate additional information that is of special use to the reader. +Notes are most effective when used _sparingly_. -- As just another sentence in the previous paragraph or the most-relevant - paragraph. -- As its own standalone paragraph. -- As content under a new subheading that introduces the topic, making it more - visible or findable. +Try to avoid them. Too many notes can impact the scannability of a topic and +create an overly busy page. -#### When to use +Instead of adding a note, try one of these alternatives: -Use a note when there is a reason that most or all readers who browse the -section should see the content. That is, if missed, it’s likely to cause major -trouble for a minority of users or significant trouble for a majority of users. +- Re-write the sentence as part of the most-relevant paragraph. +- Put the information into its own standalone paragraph. +- Put the content under a new subheading that introduces the topic, which makes + it more visible. -Weigh the costs of distracting users to whom the content is not relevant against -the cost of users missing the content if it were not expressed as a note. +If you must use a note, use the following formatting: ```markdown NOTE: **Note:** @@ -1582,12 +1648,11 @@ application: The following are recommended verbs for specific uses with user interface elements: -| Recommended | Used for | Replaces | -|:--------------------|:---------------------------|:---------------------------| -| *click* | buttons, links, menu items | "hit", "press", "select" | -| *select* or *clear* | checkboxes | "enable", "click", "press" | -| *select* | dropdowns | "pick" | -| *expand* | expandable sections | "open" | +| Recommended | Used for | Replaces | +|:--------------------|:--------------------------------------|:---------------------------| +| *select* | buttons, links, menu items, dropdowns | "click, "press," "hit" | +| *select* or *clear* | checkboxes | "enable", "click", "press" | +| *expand* | expandable sections | "open" | ### Other Verbs @@ -1614,6 +1679,15 @@ heading level. ### Text for documentation requiring version text +When a feature is new or updated, you can add version information as a bulleted +item in the **Version history**, or as an inline reference with related text. + +#### Version text in the **Version History** + +If all content in a section is related, add version text in a bulleted list +following the heading for the section. To render correctly, it must be on its +own line and surrounded by blank lines. + - For features that need to declare the GitLab version that the feature was introduced. Text similar to the following should be added immediately below the heading as a blockquote: @@ -1663,9 +1737,20 @@ heading level. and replaced by [Feature name](link-to-feature-documentation). ``` -NOTE: **Note:** -Version text must be on its own line and surrounded by blank lines to render -correctly. +#### Inline version text + +If you're adding content to an existing topic, you can add version information +inline with the existing text. + +In this case, add `([introduced/deprecated](<link-to-issue>) in GitLab X.X)`. +If applicable, include the paid tier: `([introduced/deprecated](<link-to-issue>) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.4)` + +Including the issue link is encouraged, but isn't a requirement. For example: + +```markdown +The voting strategy (introduced in GitLab 13.4) requires +the primary and secondary voters to agree. +``` ### Versions in the past or future @@ -1807,7 +1892,7 @@ for the changes to take effect. If the document you are editing resides in a place other than the GitLab CE/EE `doc/` directory, instead of the relative link, use the full path: -`https://docs.gitlab.com/ce/administration/restart_gitlab.html`. Replace +`https://docs.gitlab.com/ee/administration/restart_gitlab.html`. Replace `reconfigure` with `restart` where appropriate. ### Installation guide @@ -1894,216 +1979,9 @@ steps aren't required, consider setting up a [table](#tables) with headers of Learn how to [document features deployed behind flags](feature_flags.md). For guidance on developing GitLab with feature flags, see [Feature flags in development of GitLab](../feature_flags/index.md). -## RESTful API - -REST API resources are documented in Markdown under -[`/doc/api`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/doc/api). Each -resource has its own Markdown file, which is linked from `api_resources.md`. - -When modifying the Markdown, also update the corresponding -[OpenAPI definition](https://gitlab.com/gitlab-org/gitlab/-/tree/master/doc/api/openapi) -if one exists for the resource. If not, consider creating one. Match the latest -[OpenAPI 3.0.x specification](https://swagger.io/specification/). (For more -information, see the discussion in this -[issue](https://gitlab.com/gitlab-org/gitlab/-/issues/16023#note_370901810).) - -In the Markdown doc for a resource (AKA endpoint): - -- Every method must have the REST API request. For example: - - ```plaintext - GET /projects/:id/repository/branches - ``` - -- Every method must have a detailed [description of the parameters](#method-description). -- Every method must have a cURL example. -- Every method must have a response body (in JSON format). - -### API topic template - -The following can be used as a template to get started: - -````markdown -## Descriptive title - -One or two sentence description of what endpoint does. - -```plaintext -METHOD /endpoint -``` - -| Attribute | Type | Required | Description | -|:------------|:---------|:---------|:----------------------| -| `attribute` | datatype | yes/no | Detailed description. | -| `attribute` | datatype | yes/no | Detailed description. | - -Example request: - -```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/endpoint?parameters" -``` - -Example response: - -```json -[ - { - } -] -``` -```` - -### Fake user information - -You may need to demonstrate an API call or a cURL command that includes the name -and email address of a user. Don't use real user information in API calls: - -- **Email addresses**: Use an email address ending in `example.com`. -- **Names**: Use strings like `Example Username`. Alternatively, use diverse or - non-gendered names with common surnames, such as `Sidney Jones`, `Zhang Wei`, - or `Maria Garcia`. - -### Fake URLs - -When including sample URLs in the documentation, use: - -- `example.com` when the domain name is generic. -- `gitlab.example.com` when referring to self-managed instances of GitLab. - -### Fake tokens - -There may be times where a token is needed to demonstrate an API call using -cURL or a variable used in CI. It is strongly advised not to use real tokens in -documentation even if the probability of a token being exploited is low. - -You can use the following fake tokens as examples: - -| Token type | Token value | -|:----------------------|:-------------------------------------------------------------------| -| Private user token | `<your_access_token>` | -| Personal access token | `n671WNGecHugsdEDPsyo` | -| Application ID | `2fcb195768c39e9a94cec2c2e32c59c0aad7a3365c10892e8116b5d83d4096b6` | -| Application secret | `04f294d1eaca42b8692017b426d53bbc8fe75f827734f0260710b83a556082df` | -| CI/CD variable | `Li8j-mLUVA3eZYjPfd_H` | -| Specific runner token | `yrnZW46BrtBFqM7xDzE7dddd` | -| Shared runner token | `6Vk7ZsosqQyfreAxXTZr` | -| Trigger token | `be20d8dcc028677c931e04f3871a9b` | -| Webhook secret token | `6XhDroRcYPM5by_h-HLY` | -| Health check token | `Tu7BgjR9qeZTEyRzGG2P` | -| Request profile token | `7VgpS4Ax5utVD2esNstz` | - -### Method description - -Use the following table headers to describe the methods. Attributes should -always be in code blocks using backticks (`` ` ``). - -```markdown -| Attribute | Type | Required | Description | -|:----------|:-----|:---------|:------------| -``` - -Rendered example: - -| Attribute | Type | Required | Description | -|:----------|:-------|:---------|:--------------------| -| `user` | string | yes | The GitLab username | - -### cURL commands - -- Use `https://gitlab.example.com/api/v4/` as an endpoint. -- Wherever needed use this personal access token: `<your_access_token>`. -- Always put the request first. `GET` is the default so you don't have to - include it. -- Wrap the URL in double quotes (`"`). -- Prefer to use examples using the personal access token and don't pass data of - username and password. - -| Methods | Description | -|:------------------------------------------- |:------------------------------------------------------| -| `--header "PRIVATE-TOKEN: <your_access_token>"` | Use this method as is, whenever authentication needed | -| `--request POST` | Use this method when creating new objects | -| `--request PUT` | Use this method when updating existing objects | -| `--request DELETE` | Use this method when removing existing objects | - -### cURL Examples - -The following sections include a set of [cURL](https://curl.haxx.se) examples -you can use in the API documentation. - -#### Simple cURL command - -Get the details of a group: - -```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/gitlab-org" -``` - -#### cURL example with parameters passed in the URL - -Create a new project under the authenticated user's namespace: - -```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects?name=foo" -``` - -#### Post data using cURL's `--data` - -Instead of using `--request POST` and appending the parameters to the URI, you -can use cURL's `--data` option. The example below will create a new project -`foo` under the authenticated user's namespace. - -```shell -curl --data "name=foo" --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects" -``` - -#### Post data using JSON content - -NOTE: **Note:** -In this example we create a new group. Watch carefully the single and double -quotes. - -```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --header "Content-Type: application/json" --data '{"path": "my-group", "name": "My group"}' "https://gitlab.example.com/api/v4/groups" -``` - -#### Post data using form-data - -Instead of using JSON or urlencode you can use multipart/form-data which -properly handles data encoding: - -```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --form "title=ssh-key" --form "key=ssh-rsa AAAAB3NzaC1yc2EA..." "https://gitlab.example.com/api/v4/users/25/keys" -``` - -The above example is run by and administrator and will add an SSH public key -titled `ssh-key` to user's account which has an ID of 25. - -#### Escape special characters - -Spaces or slashes (`/`) may sometimes result to errors, thus it is recommended -to escape them when possible. In the example below we create a new issue which -contains spaces in its title. Observe how spaces are escaped using the `%20` -ASCII code. - -```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/42/issues?title=Hello%20Dude" -``` - -Use `%2F` for slashes (`/`). - -#### Pass arrays to API calls - -The GitLab API sometimes accepts arrays of strings or integers. For example, to -exclude specific users when requesting a list of users for a project, you would -do something like this: - -```shell -curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" --data "skip_users[]=<user_id>" --data "skip_users[]=<user_id>" "https://gitlab.example.com/api/v4/projects/<project_id>/users" -``` - ## GraphQL API -GraphQL APIs are different from [RESTful APIs](#restful-api). Reference +GraphQL APIs are different from [RESTful APIs](restful_api_styleguide.md). Reference information is generated in our [GraphQL reference](../../api/graphql/reference/index.md). However, it's helpful to include examples on how to use GraphQL for different @@ -2158,7 +2036,7 @@ Set up the section with the following: ```markdown 1. Open the GraphiQL explorer tool in the following URL: `https://gitlab.com/-/graphql-explorer`. 1. Paste the `query` listed above into the left window of your GraphiQL explorer tool. - 1. Click Play to get the result shown here: + 1. Select Play to get the result shown here: ``` - Include a screenshot of the result in the GraphiQL explorer. Follow the naming diff --git a/doc/development/elasticsearch.md b/doc/development/elasticsearch.md index e70cf456101..639759e5014 100644 --- a/doc/development/elasticsearch.md +++ b/doc/development/elasticsearch.md @@ -9,7 +9,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w This area is to maintain a compendium of useful information when working with Elasticsearch. Information on how to enable Elasticsearch and perform the initial indexing is in -the [Elasticsearch integration documentation](../integration/elasticsearch.md#enabling-elasticsearch). +the [Elasticsearch integration documentation](../integration/elasticsearch.md#enabling-advanced-search). ## Deep Dive diff --git a/doc/development/emails.md b/doc/development/emails.md index cf7f49ee834..de9607fef64 100644 --- a/doc/development/emails.md +++ b/doc/development/emails.md @@ -27,7 +27,7 @@ Please note that [S/MIME signed](../administration/smime_signing_email.md) email ## Mailer previews Rails provides a way to preview our mailer templates in HTML and plaintext using -dummy data. +sample data. The previews live in [`app/mailers/previews`](https://gitlab.com/gitlab-org/gitlab-foss/tree/master/app/mailers/previews) and can be viewed at [`/rails/mailers`](http://localhost:3000/rails/mailers). diff --git a/doc/development/event_tracking/backend.md b/doc/development/event_tracking/backend.md index 93e135772ef..79ea80a52ea 100644 --- a/doc/development/event_tracking/backend.md +++ b/doc/development/event_tracking/backend.md @@ -1,5 +1,5 @@ --- -redirect_to: '../telemetry/index.md' +redirect_to: '../product_analytics/index.md' --- -This document was moved to [another location](../telemetry/index.md). +This document was moved to [another location](../product_analytics/index.md). diff --git a/doc/development/event_tracking/frontend.md b/doc/development/event_tracking/frontend.md index 93e135772ef..79ea80a52ea 100644 --- a/doc/development/event_tracking/frontend.md +++ b/doc/development/event_tracking/frontend.md @@ -1,5 +1,5 @@ --- -redirect_to: '../telemetry/index.md' +redirect_to: '../product_analytics/index.md' --- -This document was moved to [another location](../telemetry/index.md). +This document was moved to [another location](../product_analytics/index.md). diff --git a/doc/development/event_tracking/index.md b/doc/development/event_tracking/index.md index 93e135772ef..79ea80a52ea 100644 --- a/doc/development/event_tracking/index.md +++ b/doc/development/event_tracking/index.md @@ -1,5 +1,5 @@ --- -redirect_to: '../telemetry/index.md' +redirect_to: '../product_analytics/index.md' --- -This document was moved to [another location](../telemetry/index.md). +This document was moved to [another location](../product_analytics/index.md). diff --git a/doc/development/experiment_guide/index.md b/doc/development/experiment_guide/index.md index 07b803603a5..18b606450c2 100644 --- a/doc/development/experiment_guide/index.md +++ b/doc/development/experiment_guide/index.md @@ -16,46 +16,97 @@ In either case, an outcome of the experiment should be posted to the issue with ## Code reviews -Since the code of experiments will not be part of the codebase for a long time and we want to iterate fast to retrieve data, the code quality of experiments might sometimes not fulfill our standards but should not negatively impact the availability of GitLab whether the experiment is running or not. -Initially experiments will only be deployed to a fraction of users but we still want a flawless experience for those users. Therefore, experiments still require tests. - -For reviewers and maintainers: if you find code that would usually not make it through the review, but is temporarily acceptable, please mention your concerns but note that it's not necessary to change. -The author then adds a comment to this piece of code and adds a link to the issue that resolves the experiment. If the experiment is successful and becomes part of the product these follow up issues should be addressed. +Experiments' code quality can fail our standards for several reasons. These +reasons can include not being added to the codebase for a long time, or because +of fast iteration to retrieve data. However, having the experiment run (or not +run) shouldn't impact GitLab's availability. To avoid or identify issues, +experiments are initially deployed to a small number of users. Regardless, +experiments still need tests. + +If, as a reviewer or maintainer, you find code that would usually fail review +but is acceptable for now, mention your concerns with a note that there's no +need to change the code. The author can then add a comment to this piece of code +and link to the issue that resolves the experiment. If the experiment is +successful and becomes part of the product, any follow up issues should be +addressed. ## How to create an A/B test -- Add the experiment to the `Gitlab::Experimentation::EXPERIMENTS` hash in [`experimentation.rb`](https://gitlab.com/gitlab-org/gitlab/blob/master/lib%2Fgitlab%2Fexperimentation.rb): - - ```ruby - EXPERIMENTS = { - other_experiment: { - #... - }, - # Add your experiment here: - signup_flow: { - environment: ::Gitlab.dev_env_or_com?, # Target environment, defaults to enabled for development and GitLab.com - tracking_category: 'Growth::Acquisition::Experiment::SignUpFlow' # Used for providing the category when setting up tracking data - } - }.freeze - ``` - -- Use the experiment in a controller: - - ```ruby - class RegistrationController < ApplicationController - def show - # experiment_enabled?(:feature_name) is also available in views and helpers - if experiment_enabled?(:signup_flow) - # render the experiment - else - # render the original version - end - end - end - ``` - -- Track necessary events. See the [telemetry guide](../telemetry/index.md) for details. -- After the merge request is merged, use [`chatops`](../../ci/chatops/README.md) in the +### Implementation + +1. Add the experiment to the `Gitlab::Experimentation::EXPERIMENTS` hash in [`experimentation.rb`](https://gitlab.com/gitlab-org/gitlab/blob/master/lib%2Fgitlab%2Fexperimentation.rb): + + ```ruby + EXPERIMENTS = { + other_experiment: { + #... + }, + # Add your experiment here: + signup_flow: { + environment: ::Gitlab.dev_env_or_com?, # Target environment, defaults to enabled for development and GitLab.com + tracking_category: 'Growth::Acquisition::Experiment::SignUpFlow' # Used for providing the category when setting up tracking data + } + }.freeze + ``` + +1. Use the experiment in the code. + + - Use this standard for the experiment in a controller: + + ```ruby + class RegistrationController < ApplicationController + def show + # experiment_enabled?(:experiment_key) is also available in views and helpers + if experiment_enabled?(:signup_flow) + # render the experiment + else + # render the original version + end + end + end + ``` + + - Make the experiment available to the frontend in a controller: + + ```ruby + before_action do + push_frontend_experiment(:signup_flow) + end + ``` + + The above will check whether the experiment is enabled and push the result to the frontend. + + You can check the state of the feature flag in JavaScript: + + ```javascript + import { isExperimentEnabled } from '~/experimentation'; + + if ( isExperimentEnabled('signupFlow') ) { + // ... + } + ``` + + - It is also possible to run an experiment outside of the controller scope, for example in a worker: + + ```ruby + class SomeWorker + def perform + # Check if the experiment is enabled at all (the percentage_of_time_value > 0) + return unless Gitlab::Experimentation.enabled?(:experiment_key) + + # Since we cannot access cookies in a worker, we need to bucket models based on a unique, unchanging attribute instead. + # Use the following method to check if the experiment is enabled for a certain attribute, for example a username or email address: + if Gitlab::Experimentation.enabled_for_attribute?(:experiment_key, some_attribute) + # execute experimental code + else + # execute control code + end + end + end + ``` + +1. Track necessary events. See the [product analytics guide](../product_analytics/index.md) for details. +1. After the merge request is merged, use [`chatops`](../../ci/chatops/README.md) in the [appropriate channel](../feature_flags/controls.md#communicate-the-change) to start the experiment for 10% of the users. The feature flag should have the name of the experiment with the `_experiment_percentage` suffix appended. For visibility, please also share any commands run against production in the `#s_growth` channel: @@ -69,3 +120,19 @@ For visibility, please also share any commands run against production in the `#s ```shell /chatops run feature delete signup_flow_experiment_percentage ``` + +### Tests and test helpers + +Use the following in Jest to test the experiment is enabled. + +```javascript +import { withGonExperiment } from 'helpers/experimentation_helper'; + +describe('given experiment is enabled', () => { + withGonExperiment('signupFlow'); + + it('should do the experimental thing', () => { + expect(wrapper.find('.js-some-experiment-triggered-element')).toEqual(expect.any(Element)); + }); +}); +``` diff --git a/doc/development/fe_guide/architecture.md b/doc/development/fe_guide/architecture.md index 3d27f67a8a6..c7e1ba59f23 100644 --- a/doc/development/fe_guide/architecture.md +++ b/doc/development/fe_guide/architecture.md @@ -15,5 +15,5 @@ You can find the Frontend Architecture experts on the [team page](https://about. ## Examples -You can find documentation about the desired architecture for a new feature -built with Vue.js [here](vue.md). +You can find [documentation about the desired architecture](vue.md) for a new +feature built with Vue.js. diff --git a/doc/development/fe_guide/dependencies.md b/doc/development/fe_guide/dependencies.md index 7f078df887d..ba97e7e39be 100644 --- a/doc/development/fe_guide/dependencies.md +++ b/doc/development/fe_guide/dependencies.md @@ -18,9 +18,9 @@ automatically create merge requests for updating dependencies of several project up-to-date list of projects managed by the renovate bot in the project’s README. Some key dependencies updated using renovate are: -- [`@gitlab/ui`](https://gitlab.com/gitlab-org/gitlab-ui/) -- [`@gitlab/svgs`](https://gitlab.com/gitlab-org/gitlab-svgs/) -- [`@gitlab/eslint-config`](https://gitlab.com/gitlab-org/gitlab-eslint-config) +- [`@gitlab/ui`](https://gitlab.com/gitlab-org/gitlab-ui) +- [`@gitlab/svgs`](https://gitlab.com/gitlab-org/gitlab-svgs) +- [`@gitlab/eslint-plugin`](https://gitlab.com/gitlab-org/frontend/eslint-plugin) ### Blocked dependencies diff --git a/doc/development/fe_guide/event_tracking.md b/doc/development/fe_guide/event_tracking.md index 93e135772ef..79ea80a52ea 100644 --- a/doc/development/fe_guide/event_tracking.md +++ b/doc/development/fe_guide/event_tracking.md @@ -1,5 +1,5 @@ --- -redirect_to: '../telemetry/index.md' +redirect_to: '../product_analytics/index.md' --- -This document was moved to [another location](../telemetry/index.md). +This document was moved to [another location](../product_analytics/index.md). diff --git a/doc/development/fe_guide/graphql.md b/doc/development/fe_guide/graphql.md index 82cd19dce4f..ad3958d4496 100644 --- a/doc/development/fe_guide/graphql.md +++ b/doc/development/fe_guide/graphql.md @@ -187,7 +187,7 @@ As shown in the code example by using `produce`, we can perform any kind of dire `draftState`. Besides, `immer` guarantees that a new state which includes the changes to `draftState` will be generated. Finally, to verify whether the immutable cache update is working properly, we need to change -`assumeImmutableResults` to `true` in the `default client config` (see [Apollo Client](#apollo-client) for more info). +`assumeImmutableResults` to `true` in the `default client config` (see [Apollo Client](#apollo-client) for more info). If everything is working properly `assumeImmutableResults` should remain set to `true`. @@ -308,7 +308,7 @@ const resolvers = { export default resolvers; ``` -We need to pass resolvers object to our existing Apollo Client: +We need to pass a resolvers object to our existing Apollo Client: ```javascript // graphql.js @@ -319,13 +319,13 @@ import resolvers from './graphql/resolvers'; const defaultClient = createDefaultClient(resolvers); ``` -Now every single time on attempt to fetch a version, our client will fetch `id` and `sha` from the remote API endpoint and will assign our hardcoded values to `author` and `createdAt` version properties. With this data, frontend developers are able to work on UI part without being blocked by backend. When actual response is added to the API, a custom local resolver can be removed fast and the only change to query/fragment is `@client` directive removal. +For each attempt to fetch a version, our client will fetch `id` and `sha` from the remote API endpoint and will assign our hardcoded values to the `author` and `createdAt` version properties. With this data, frontend developers are able to work on their UI without being blocked by backend. When the actual response is added to the API, our custom local resolver can be removed and the only change to the query/fragment is to remove the `@client` directive. Read more about local state management with Apollo in the [Vue Apollo documentation](https://vue-apollo.netlify.app/guide/local-state.html#local-state). ### Using with Vuex -When Apollo Client is used within Vuex and fetched data is stored in the Vuex store, there is no need in keeping Apollo Client cache enabled. Otherwise we would have data from the API stored in two places - Vuex store and Apollo Client cache. More to say, with Apollo's default settings, a subsequent fetch from the GraphQL API could result in fetching data from Apollo cache (in the case where we have the same query and variables). To prevent this behavior, we need to disable Apollo Client cache passing a valid `fetchPolicy` option to its constructor: +When Apollo Client is used within Vuex and fetched data is stored in the Vuex store, there is no need to keep Apollo Client cache enabled. Otherwise we would have data from the API stored in two places - Vuex store and Apollo Client cache. With Apollo's default settings, a subsequent fetch from the GraphQL API could result in fetching data from Apollo cache (in the case where we have the same query and variables). To prevent this behavior, we need to disable Apollo Client cache by passing a valid `fetchPolicy` option to its constructor: ```javascript import fetchPolicies from '~/graphql_shared/fetch_policy_constants'; diff --git a/doc/development/fe_guide/icons.md b/doc/development/fe_guide/icons.md index b539293e9cf..67add5709d9 100644 --- a/doc/development/fe_guide/icons.md +++ b/doc/development/fe_guide/icons.md @@ -1,9 +1,10 @@ # Icons and SVG Illustrations -We manage our own Icon and Illustration library in the [`gitlab-svgs`](https://gitlab.com/gitlab-org/gitlab-svgs) repository. -This repository is published on [npm](https://www.npmjs.com/package/@gitlab/svgs) and managed as a dependency via yarn. -You can browse all available Icons and Illustrations [here](https://gitlab-org.gitlab.io/gitlab-svgs). -To upgrade to a new version run `yarn upgrade @gitlab/svgs`. +We manage our own icon and illustration library in the [`gitlab-svgs`](https://gitlab.com/gitlab-org/gitlab-svgs) +repository. This repository is published on [npm](https://www.npmjs.com/package/@gitlab/svgs), +and is managed as a dependency with yarn. You can browse all available +[icons and illustrations](https://gitlab-org.gitlab.io/gitlab-svgs). To upgrade +to a new version run `yarn upgrade @gitlab/svgs`. ## Icons @@ -21,10 +22,11 @@ To use a sprite Icon in HAML or Rails we use a specific helper function : sprite_icon(icon_name, size: nil, css_class: '') ``` -- **icon_name** Use the icon_name that you can find in the SVG Sprite - ([Overview is available here](https://gitlab-org.gitlab.io/gitlab-svgs)). -- **size (optional)** Use one of the following sizes : 16, 24, 32, 48, 72 (this will be translated into a `s16` class) -- **css_class (optional)** If you want to add additional CSS classes +- **icon_name**: Use the icon_name for the SVG sprite in the list of + ([GitLab SVGs](https://gitlab-org.gitlab.io/gitlab-svgs)). +- **size (optional)**: Use one of the following sizes : 16, 24, 32, 48, 72 (this + will be translated into a `s16` class) +- **css_class (optional)**: If you want to add additional CSS classes. **Example** @@ -66,10 +68,12 @@ export default { </template> ``` -- **name** Name of the Icon in the SVG Sprite ([Overview is available here](https://gitlab-org.gitlab.io/gitlab-svgs)). -- **size (optional)** Number value for the size which is then mapped to a specific CSS class - (Available Sizes: 8, 12, 16, 18, 24, 32, 48, 72 are mapped to `sXX` CSS classes) -- **class (optional)** Additional CSS Classes to add to the SVG tag. +- **name**: Name of the icon of the SVG sprite, as listed in the + ([GitLab SVG Previewer](https://gitlab-org.gitlab.io/gitlab-svgs)). +- **size: (optional)** Number value for the size which is then mapped to a + specific CSS class (Available sizes: 8, 12, 16, 18, 24, 32, 48, 72 are mapped + to `sXX` CSS classes) +- **class (optional)**: Additional CSS classes to add to the SVG tag. ### Usage in HTML/JS diff --git a/doc/development/fe_guide/index.md b/doc/development/fe_guide/index.md index ef23b6c4ed2..f909866d44e 100644 --- a/doc/development/fe_guide/index.md +++ b/doc/development/fe_guide/index.md @@ -76,6 +76,10 @@ How we use SVG for our [Icons and Illustrations](icons.md). General information about frontend [dependencies](dependencies.md) and how we manage them. +## Keyboard Shortcuts + +How we implement [keyboard shortcuts](keyboard_shortcuts.md) that can be customized and disabled. + ## Frontend FAQ Read the [frontend's FAQ](frontend_faq.md) for common small pieces of helpful information. diff --git a/doc/development/fe_guide/keyboard_shortcuts.md b/doc/development/fe_guide/keyboard_shortcuts.md new file mode 100644 index 00000000000..da9b3702a8d --- /dev/null +++ b/doc/development/fe_guide/keyboard_shortcuts.md @@ -0,0 +1,98 @@ +# Implementing keyboard shortcuts + +We use [Mousetrap](https://craig.is/killing/mice) to implement keyboard +shortcuts in GitLab. + +Mousetrap provides an API that allows keyboard shortcut strings (like +`mod+shift+p` or `p b`) to be bound to a JavaScript handler: + +```javascript +// Don't do this; see note below +Mousetrap.bind('p b', togglePerformanceBar) +``` + +However, associating a hard-coded key sequence to a handler (as shown above) +prevents these keyboard shortcuts from being customized or disabled by users. + +To allow keyboard shortcuts to be customized, commands are defined in +`~/behaviors/shortcuts/keybindings.js`. The `keysFor` method is responsible for +returning the correct key sequence for the provided command: + +```javascript +import { keysFor, TOGGLE_PERFORMANCE_BAR } from '~/behaviors/shortcuts/keybindings' + +Mousetrap.bind(keysFor(TOGGLE_PERFORMANCE_BAR), togglePerformanceBar); +``` + +## Shortcut customization + +`keybindings.js` stores keyboard shortcut customizations as a JSON string in +`localStorage`. When `keybindings.js` is first imported, it fetches any +customizations from `localStorage` and merges these customizations into the +default set of keybindings. There is no UI to edit these customizations. + +## Adding new shortcuts + +Because keyboard shortcuts can be customized or disabled by end users, +developers are encouraged to build _lots_ of keyboard shortcuts into GitLab. +Shortcuts that are less likely to be used should be +[disabled](#disabling-shortcuts) by default. + +To add a new shortcut, define and export a new command string in +`keybindings.js`: + +```javascript +export const MAKE_COFFEE = 'foodAndBeverage.makeCoffee'; +``` + +Next, add a new command definition under the appropriate group in the +`keybindingGroups` array: + +```javascript +{ + description: s__('KeyboardShortcuts|Make coffee'), + command: MAKE_COFFEE, + defaultKeys: ['mod+shift+c'], + customKeys: customizations[MAKE_COFFEE], +} +``` + +Finally, in the application code, import the `keysFor` function and the new +command and bind the shortcut to the handler using Mousetrap: + +```javascript +import { keysFor, MAKE_COFFEE } from '~/behaviors/shortcuts/keybindings' + +Mousetrap.bind(keysFor(MAKE_COFFEE), makeCoffee); +``` + +See the existing the command definitions in `keybindings.js` for more examples. + +## Disabling shortcuts + +A shortcut can be disabled, also known as _unassigned_, by assigning the +shortcut to an empty array `[]`. For example, to introduce a new shortcut that +is disabled by default, a command can be defined like this: + +```javascript +export const MAKE_MOCHA = 'foodAndBeverage.makeMocha'; + +{ + description: s__('KeyboardShortcuts|Make a mocha'), + command: MAKE_MOCHA, + defaultKeys: [], + customKeys: customizations[MAKE_MOCHA], +} +``` + +## Make cross-platform shortcuts + +It's difficult to make shortcuts that work well in all platforms and browsers. +This is one of the reasons that being able to customize and disable shortcuts is +so important. + +One important way to make keyboard shortcuts more portable is to use the `mod` +shortcut string, which resolves to `command` on Mac and `ctrl` otherwise. + +See [Mousetrap's documentation](https://craig.is/killing/mice#api.bind.combo) +for more information. diff --git a/doc/development/fe_guide/tooling.md b/doc/development/fe_guide/tooling.md index 5685ac5abcd..b93c0a9736b 100644 --- a/doc/development/fe_guide/tooling.md +++ b/doc/development/fe_guide/tooling.md @@ -101,7 +101,10 @@ Our code is automatically formatted with [Prettier](https://prettier.io) to foll ### Editor -The easiest way to include prettier in your workflow is by setting up your preferred editor (all major editors are supported) accordingly. We suggest setting up prettier to run automatically when each file is saved. Find [here](https://prettier.io/docs/en/editors.html) the best way to set it up in your preferred editor. +The recommended method to include Prettier in your workflow is to set up your +preferred editor (all major editors are supported) accordingly. We suggest +setting up Prettier to run when each file is saved. For instructions about using +Prettier in your preferred editor, see the [Prettier documentation](https://prettier.io/docs/en/editors.html). Please take care that you only let Prettier format the same file types as the global Yarn script does (`.js`, `.vue`, `.graphql`, and `.scss`). In VSCode by example you can easily exclude file formats in your settings file: diff --git a/doc/development/fe_guide/vuex.md b/doc/development/fe_guide/vuex.md index 4badf3f0845..528dcb3b7f4 100644 --- a/doc/development/fe_guide/vuex.md +++ b/doc/development/fe_guide/vuex.md @@ -32,8 +32,9 @@ When using Vuex at GitLab, separate these concerns into different files to impro └── mutation_types.js # mutation types ``` -The following example shows an application that lists and adds users to the state. -(For a more complex example implementation take a look at the security applications store in [here](https://gitlab.com/gitlab-org/gitlab/tree/master/ee/app/assets/javascripts/vue_shared/security_reports/store)) +The following example shows an application that lists and adds users to the +state. (For a more complex example implementation, review the security +applications stored in this [repository](https://gitlab.com/gitlab-org/gitlab/tree/master/ee/app/assets/javascripts/vue_shared/security_reports/store)). ### `index.js` diff --git a/doc/development/feature_categorization/index.md b/doc/development/feature_categorization/index.md index 3fd402ebe84..57e0ad8b772 100644 --- a/doc/development/feature_categorization/index.md +++ b/doc/development/feature_categorization/index.md @@ -23,9 +23,9 @@ product categories. When this occurs, you can automatically update and generate a new version of the file, which needs to be committed to the repository. -The [Scalabilitity +The [Scalability team](https://about.gitlab.com/handbook/engineering/infrastructure/team/scalability/) -currently maintains the `stages.yml` file. They will automatically be +currently maintains the `feature_categories.yml` file. They will automatically be notified on Slack when the file becomes outdated. ## Sidekiq workers @@ -75,38 +75,23 @@ A feature category can be specified on an entire controller using: ```ruby -class Projects::MergeRequestsController < ApplicationController - feature_category :source_code_management +class Boards::ListsController < ApplicationController + feature_category :kanban_boards end ``` The feature category can be limited to a list of actions using the -`only` argument, actions can be excluded using the `except` argument. +second argument: ```ruby -class Projects::MergeRequestsController < ApplicationController - feature_category :code_testing, only: [:metrics_reports] - feature_category :source_code_management, except: [:test_reports, :coverage_reports] +class DashboardController < ApplicationController + feature_category :issue_tracking, [:issues, :issues_calendar] + feature_category :code_review, [:merge_requests] end ``` -`except` and `only` arguments can not be combined. - -When specifying `except` all other actions will get the specified -category assigned. - -The assignment can also be scoped using `if` and `unless` procs: - -```ruby -class Projects::MergeRequestsController < ApplicationController - feature_category :source_code_management, - unless: -> (action) { action.include?("reports") } - if: -> (action) { action.include?("widget") } -end -``` - -In this case, both procs need to be satisfied for the action to get -the category assigned. +These forms cannot be mixed: if a controller has more than one category, +every single action must be listed. ### Excluding controller actions from feature categorization @@ -125,6 +110,5 @@ The `spec/controllers/every_controller_spec.rb` will iterate over all defined routes, and check the controller to see if a category is assigned to all actions. -The spec also validates if the used feature categories are known. And -if the actions used in `only` and `except` configuration still exist -as routes. +The spec also validates if the used feature categories are known. And if +the actions used in configuration still exist as routes. diff --git a/doc/development/feature_flags/controls.md b/doc/development/feature_flags/controls.md index 605b5919e0b..ef38a85bec0 100644 --- a/doc/development/feature_flags/controls.md +++ b/doc/development/feature_flags/controls.md @@ -88,15 +88,13 @@ parts of the company. The developer responsible needs to determine whether this is necessary and the appropriate level of communication. This depends on the feature and what sort of impact it might have. -As a guideline: +Guidelines: -- For simple features that are low-risk, and easily rolled back, then - just proceed to [enabling the feature in `#production`](#process). -- For features that will impact user experience consider notifying +1. If the feature meets the requirements for creating a [Change Management](https://about.gitlab.com/handbook/engineering/infrastructure/change-management/#feature-flags-and-the-change-management-process) issue, create a Change Management issue per [criticality guidelines](https://about.gitlab.com/handbook/engineering/infrastructure/change-management/#change-request-workflows). +1. For simple, low-risk, easily reverted features, proceed and [enable the feature in `#production`](#process). +1. For features that impact the user experience, consider notifying + `#support_gitlab-com` first. `#support_gitlab-com` beforehand. -- For features with significant downstream effects (e.g.: turning on/off - Elasticsearch indexing) consider coordinating with `#production` - beforehand. #### Process @@ -250,13 +248,26 @@ Changes to the issue format can be submitted in the ## Cleaning up -Once the change is deemed stable, submit a new merge request to remove the -feature flag. This ensures the change is available to all users and self-managed -instances. Make sure to add the ~"feature flag" label to this merge request so -release managers are aware the changes are hidden behind a feature flag. If the -merge request has to be picked into a stable branch, make sure to also add the -appropriate `~"Pick into X.Y"` label (e.g. `~"Pick into 13.0"`). -See [the process document](process.md#including-a-feature-behind-feature-flag-in-the-final-release) for further details. +A feature flag should be removed as soon as it is no longer needed. Each additional +feature flag in the codebase increases the complexity of the application +and reduces confidence in our testing suite covering all possible combinations. +Additionally, a feature flag overwritten in some of the environments can result +in undefined and untested system behavior. + +To remove a feature flag: + +1. Open a new merge request with the ~"feature flag" label so + release managers are aware the changes are hidden behind a feature flag. +1. If the merge request has to be picked into a stable branch, add the + appropriate `~"Pick into X.Y"` label, for example `~"Pick into 13.0"`. + See [the feature flag process](process.md#including-a-feature-behind-feature-flag-in-the-final-release) + for further details. +1. Remove all references to the feature flag from the codebase. +1. Remove the YAML definition for the feature from the repository. +1. Clean up the feature flag from all environments with `/chatops run feature delete some_feature`. +1. Close the rollout issue for the feature flag after the feature flag is removed from the codebase. + +### Cleanup ChatOps When a feature gate has been removed from the code base, the feature record still exists in the database that the flag was deployed too. diff --git a/doc/development/feature_flags/development.md b/doc/development/feature_flags/development.md index 29bd0ca0a7e..067e480f6f8 100644 --- a/doc/development/feature_flags/development.md +++ b/doc/development/feature_flags/development.md @@ -52,10 +52,10 @@ invocations: ```ruby # Check if feature flag is enabled -Feature.enabled?(:my_ops_flag, project, type: ops) +Feature.enabled?(:my_ops_flag, project, type: :ops) # Check if feature flag is disabled -Feature.disabled?(:my_ops_flag, project, type: ops) +Feature.disabled?(:my_ops_flag, project, type: :ops) # Push feature flag to Frontend push_frontend_feature_flag(:my_ops_flag, project, type: :ops) @@ -153,6 +153,11 @@ default_enabled: false TIP: **Tip:** To create a feature flag that is only used in EE, add the `--ee` flag: `bin/feature-flag --ee` +## Delete a feature flag + +See [cleaning up feature flags](controls.md#cleaning-up) for more information about +deleting feature flags. + ## Develop with a feature flag There are two main ways of using Feature Flags in the GitLab codebase: @@ -440,6 +445,21 @@ Feature.enabled?(:my_feature2) # => false Feature.enabled?(:my_feature2, project1) # => true ``` +### `have_pushed_frontend_feature_flags` + +Use `have_pushed_frontend_feature_flags` to test if [`push_frontend_feature_flag`](#frontend) +has added the feature flag to the HTML. + +For example, + +```ruby +stub_feature_flags(value_stream_analytics_path_navigation: false) + +visit group_analytics_cycle_analytics_path(group) + +expect(page).to have_pushed_frontend_feature_flags(valueStreamAnalyticsPathNavigation: false) +``` + ### `stub_feature_flags` vs `Feature.enable*` It is preferred to use `stub_feature_flags` to enable feature flags @@ -496,6 +516,14 @@ Feature.enabled?(:ci_live_trace) # => false Feature.enabled?(:ci_live_trace, gate) # => true ``` +You can also disable a feature flag for a specific actor: + +```ruby +gate = stub_feature_flag_gate('CustomActor') + +stub_feature_flags(ci_live_trace: false, thing: gate) +``` + ### Controlling feature flags engine in tests Our Flipper engine in the test environment works in a memory mode `Flipper::Adapters::Memory`. diff --git a/doc/development/frontend.md b/doc/development/frontend.md index f46cc377f95..8bb5cf7af62 100644 --- a/doc/development/frontend.md +++ b/doc/development/frontend.md @@ -1,4 +1,5 @@ +--- +redirect_to: 'fe_guide/index.md' +--- -# Frontend Development Guidelines - -This page has moved [here](fe_guide/index.md). +This document was moved to [another location](fe_guide/index.md). diff --git a/doc/development/geo.md b/doc/development/geo.md index 5b4af1c9931..06b032a8f66 100644 --- a/doc/development/geo.md +++ b/doc/development/geo.md @@ -1,3 +1,9 @@ +--- +stage: Enablement +group: Geo +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Geo (development) **(PREMIUM ONLY)** Geo connects GitLab instances together. One GitLab instance is @@ -420,7 +426,7 @@ We switch and filter from each event by the `event_name` field. ### Geo Log Cursor (GitLab 10.0 and up) -Since GitLab 10.0, [System Webhooks](#system-hooks-gitlab-87-to-95) are no longer +In GitLab 10.0 and later, [System Webhooks](#system-hooks-gitlab-87-to-95) are no longer used and Geo Log Cursor is used instead. The Log Cursor traverses the `Geo::EventLog` rows to see if there are changes since the last time the log was checked and will handle repository updates, deletes, diff --git a/doc/development/geo/framework.md b/doc/development/geo/framework.md index b720a6ca47e..55f4be07bb4 100644 --- a/doc/development/geo/framework.md +++ b/doc/development/geo/framework.md @@ -1,3 +1,9 @@ +--- +stage: Enablement +group: Geo +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Geo self-service framework (alpha) NOTE: **Note:** @@ -91,8 +97,6 @@ module Geo ::Packages::PackageFile end - # Change this to `true` to release replication of this model. Then remove - # this override in the next release. # The feature flag follows the format `geo_#{replicable_name}_replication`, # so here it would be `geo_package_file_replication` def self.replication_enabled_by_default? @@ -193,7 +197,9 @@ For example, to add support for files referenced by a `Widget` model with a file_store == ObjectStorage::Store::LOCAL end - def self.replicables_for_geo_node + # @param primary_key_in [Range, Widget] arg to pass to primary_key_in scope + # @return [ActiveRecord::Relation<Widget>] everything that should be synced to this node, restricted by primary key + def self.replicables_for_current_secondary(primary_key_in) # Should be implemented. The idea of the method is to restrict # the set of synced items depending on synchronization settings end @@ -220,8 +226,6 @@ For example, to add support for files referenced by a `Widget` model with a model_record.file end - # Change this to `true` to release replication of this model. Then remove - # this override in the next release. # The feature flag follows the format `geo_#{replicable_name}_replication`, # so here it would be `geo_widget_replication` def self.replication_enabled_by_default? @@ -637,7 +641,7 @@ the Admin Area UI, and Prometheus! include ::Types::Geo::RegistryType graphql_name 'WidgetRegistry' - description 'Represents the sync and verification state of a widget' + description 'Represents the Geo sync and verification state of a widget' field :widget_id, GraphQL::ID_TYPE, null: false, description: 'ID of the Widget' end @@ -676,6 +680,12 @@ the Admin Area UI, and Prometheus! } ``` +1. Update the GraphQL reference documentation: + + ```shell + bundle exec rake gitlab:graphql:compile_docs + ``` + Individual widget synchronization and verification data should now be available via the GraphQL API! @@ -693,3 +703,33 @@ To do: This should be done as part of Widget sync and verification data (aggregate and individual) should now be available in the Admin UI! + +#### Releasing the feature + +1. In `ee/app/replicators/geo/widget_replicator.rb`, delete the `self.replication_enabled_by_default?` method: + + ```ruby + module Geo + class WidgetReplicator < Gitlab::Geo::Replicator + ... + + # REMOVE THIS METHOD + def self.replication_enabled_by_default? + false + end + # REMOVE THIS METHOD + + ... + end + end + ``` + +1. In `ee/app/graphql/types/geo/geo_node_type.rb`, remove the `feature_flag` option for the released type: + + ```ruby + field :widget_registries, ::Types::Geo::WidgetRegistryType.connection_type, + null: true, + resolver: ::Resolvers::Geo::WidgetRegistriesResolver, + description: 'Find widget registries on this Geo node', + feature_flag: :geo_widget_replication # REMOVE THIS LINE + ``` diff --git a/doc/development/go_guide/index.md b/doc/development/go_guide/index.md index 6954b2bd6dc..15d25d2d1ed 100644 --- a/doc/development/go_guide/index.md +++ b/doc/development/go_guide/index.md @@ -130,7 +130,7 @@ become available, you will be able to share job templates like this Dependencies should be kept to the minimum. The introduction of a new dependency should be argued in the merge request, as per our [Approval Guidelines](../code_review.md#approval-guidelines). Both [License -Management](../../user/compliance/license_compliance/index.md) +Scanning](../../user/compliance/license_compliance/index.md) **(ULTIMATE)** and [Dependency Scanning](../../user/application_security/dependency_scanning/index.md) **(ULTIMATE)** should be activated on all projects to ensure new dependencies @@ -138,7 +138,7 @@ security status and license compatibility. ### Modules -Since Go 1.11, a standard dependency system is available behind the name [Go +In Go 1.11 and later, a standard dependency system is available behind the name [Go Modules](https://github.com/golang/go/wiki/Modules). It provides a way to define and lock dependencies for reproducible builds. It should be used whenever possible. @@ -472,7 +472,7 @@ In case we want to drop support for `go 1.11` in GitLab `12.10`, we need to veri We will not consider the active milestone, `12.10`, because a backport for `12.7` will be required in case of a critical security release. -1. If both [Omnibus and CNG](#updating-go-version) were using Go `1.12` since GitLab `12.7`, then we safely drop support for `1.11`. +1. If both [Omnibus and CNG](#updating-go-version) were using Go `1.12` in GitLab `12.7` and later, then we safely drop support for `1.11`. 1. If Omnibus or CNG were using `1.11` in GitLab `12.7`, then we still need to keep support for Go `1.11` for easier backporting of security fixes. ## Secure Team standards and style guidelines diff --git a/doc/development/gotchas.md b/doc/development/gotchas.md index f7b44e74c17..cc3db267d53 100644 --- a/doc/development/gotchas.md +++ b/doc/development/gotchas.md @@ -3,6 +3,21 @@ The purpose of this guide is to document potential "gotchas" that contributors might encounter or should avoid during development of GitLab CE and EE. +## Do not read files from app/assets directory + +In GitLab 10.8 and later, Omnibus has [dropped the `app/assets` directory](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/2456), +after asset compilation. The `ee/app/assets`, `vendor/assets` directories are dropped as well. + +This means that reading files from that directory will fail in Omnibus-installed GitLab instances: + +```ruby +file = Rails.root.join('app/assets/images/logo.svg') + +# This file does not exist, read will fail with: +# Errno::ENOENT: No such file or directory @ rb_sysopen +File.read(file) +``` + ## Do not assert against the absolute value of a sequence-generated attribute Consider the following factory: diff --git a/doc/development/graphql_guide/index.md b/doc/development/graphql_guide/index.md index 62650f7cd3f..9d7fb5ba0a8 100644 --- a/doc/development/graphql_guide/index.md +++ b/doc/development/graphql_guide/index.md @@ -8,6 +8,6 @@ feedback, and suggestions. - [GraphQL API development style guide](../api_graphql_styleguide.md): development style guide for GraphQL. -- [GraphQL API documentation style guide](../documentation/styleguide.md#graphql-api): documentation +- [GraphQL API documentation style guide](../documentation/graphql_styleguide.md): documentation style guide for GraphQL. - [GraphQL API](../../api/graphql/index.md): user documentation for the GitLab GraphQL API. diff --git a/doc/development/i18n/externalization.md b/doc/development/i18n/externalization.md index b5c5a199b1e..59399e54c3e 100644 --- a/doc/development/i18n/externalization.md +++ b/doc/development/i18n/externalization.md @@ -205,7 +205,7 @@ For the static text strings we suggest two patterns for using these translations When possible, you should opt for this pattern, as this allows you to import these strings directly into your component specs for re-use during testing. -- Internal component `$options` object `: +- Internal component `$options` object: ```javascript <script> @@ -432,7 +432,7 @@ To avoid this error, use the applicable HTML entity code (`<` or `>`) inst - In JavaScript: ```javascript - import { sanitize } from 'dompurify'; + import { sanitize } from '~/lib/dompurify'; const i18n = { LESS_THAN_ONE_HOUR: sanitize(__('In < 1 hour'), { ALLOWED_TAGS: [] }) }; diff --git a/doc/development/i18n/index.md b/doc/development/i18n/index.md index 929eded3f8e..2d84fe4536f 100644 --- a/doc/development/i18n/index.md +++ b/doc/development/i18n/index.md @@ -38,10 +38,10 @@ Voting for translations is also valuable, helping to confirm good and flag inacc See [Translation guidelines](translation.md). -### Proof reading +### Proofreading -Proof reading helps ensure the accuracy and consistency of translations. All -translations are proof read before being accepted. If a translations requires +Proofreading helps ensure the accuracy and consistency of translations. All +translations are proofread before being accepted. If a translations requires changes, you will be notified with a comment explaining why. See [Proofreading Translations](proofreader.md) for more information on who's diff --git a/doc/development/img/architecture_simplified.png b/doc/development/img/architecture_simplified.png Binary files differindex 46ae2b3c055..72d00b91129 100644 --- a/doc/development/img/architecture_simplified.png +++ b/doc/development/img/architecture_simplified.png diff --git a/doc/development/instrumentation.md b/doc/development/instrumentation.md index e420ae0c54f..bdbcd52eb61 100644 --- a/doc/development/instrumentation.md +++ b/doc/development/instrumentation.md @@ -1,6 +1,6 @@ --- stage: Monitor -group: APM +group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- @@ -11,7 +11,7 @@ blocks of Ruby code. Method instrumentation is the primary form of instrumentation with block-based instrumentation only being used when we want to drill down to specific regions of code within a method. -Please refer to [Telemetry](telemetry/index.md) if you are tracking product usage patterns. +Please refer to [Product Analytics](product_analytics/index.md) if you are tracking product usage patterns. ## Instrumenting Methods diff --git a/doc/development/integrations/secure.md b/doc/development/integrations/secure.md index dcfd0f40bf0..1094074cab6 100644 --- a/doc/development/integrations/secure.md +++ b/doc/development/integrations/secure.md @@ -374,12 +374,19 @@ which is shared by the analyzers that GitLab maintains. You can [contribute](htt new generic identifiers to if needed. Analyzers may also produce vendor-specific or product-specific identifiers, which don't belong in the [common library](https://gitlab.com/gitlab-org/security-products/analyzers/common). -The first item of the `identifiers` array is called the primary identifier. +The first item of the `identifiers` array is called the [primary +identifier](../../user/application_security/terminology/#primary-identifier). The primary identifier is particularly important, because it is used to [track vulnerabilities](#tracking-and-merging-vulnerabilities) as new commits are pushed to the repository. Identifiers are also used to [merge duplicate vulnerabilities](#tracking-and-merging-vulnerabilities) reported for the same commit, except for `CWE` and `WASC`. +Not all vulnerabilities have CVEs, and a CVE can be identified multiple times. As a result, a CVE +isn't a stable identifier and you shouldn't assume it as such when tracking vulnerabilities. + +The maximum number of identifiers for a vulnerability is set as 20. If a vulnerability has more than 20 identifiers, +the system will save only the first 20 of them. + ### Location The `location` indicates where the vulnerability has been detected. diff --git a/doc/development/integrations/secure_partner_integration.md b/doc/development/integrations/secure_partner_integration.md index 830cb84e257..19fd86f4bf6 100644 --- a/doc/development/integrations/secure_partner_integration.md +++ b/doc/development/integrations/secure_partner_integration.md @@ -36,7 +36,7 @@ best place to integrate your own product and its results into GitLab. - Pipeline jobs serve a variety of purposes. Jobs can do scanning for and have implications for app security, corporate policy, or compliance. When complete, the job reports back on its status and creates a - [job artifact](../../user/project/pipelines/job_artifacts.md) as a result. + [job artifact](../../ci/pipelines/job_artifacts.md) as a result. - The [Merge Request Security Widget](../../user/project/merge_requests/testing_and_reports_in_merge_requests.md#security-reports) displays the results of the pipeline's security checks and the developer can review them. The developer can review both a summary and a detailed version @@ -44,7 +44,7 @@ best place to integrate your own product and its results into GitLab. - If certain policies (such as [merge request approvals](../../user/project/merge_requests/merge_request_approvals.md)) are in place for a project, developers must resolve specific findings or get an approval from a specific list of people. -- The [security dashboard](../../user/application_security/security_dashboard/index.md#gitlab-security-dashboard) +- The [security dashboard](../../user/application_security/security_dashboard/index.md) also shows results which can developers can use to quickly see all the vulnerabilities that need to be addressed in the code. - When the developer reads the details about a vulnerability, they are @@ -78,7 +78,7 @@ and complete an integration with the Secure stage. to successfully display your own product's results with the rest of GitLab. - See detailed [technical directions](secure.md) for this step. - Read more about [job report artifacts](../../ci/pipelines/job_artifacts.md#artifactsreports). - - Read about [job artifacts](../../user/project/pipelines/job_artifacts.md). + - Read about [job artifacts](../../ci/pipelines/job_artifacts.md). - Your report artifact must be in one of our currently supported formats. For more information, see the [documentation on reports](secure.md#report). - Documentation for [SAST reports](../../user/application_security/sast/index.md#reports-json-format). @@ -89,7 +89,7 @@ and complete an integration with the Secure stage. and add the label `devops::secure`. - Once the job is completed, the data can be seen: - In the [Merge Request Security Report](../../user/project/merge_requests/testing_and_reports_in_merge_requests.md#security-reports) ([MR Security Report data flow](https://gitlab.com/snippets/1910005#merge-request-view)). - - While [browsing a Job Artifact](../../user/project/pipelines/job_artifacts.md). + - While [browsing a Job Artifact](../../ci/pipelines/job_artifacts.md). - In the [Security Dashboard](../../user/application_security/security_dashboard/index.md) ([Dashboard data flow](https://gitlab.com/snippets/1910005#project-and-group-dashboards)). 1. Optional: Provide a way to interact with results as Vulnerabilities: - Users can interact with the findings from your artifact within their workflow. They can dismiss the findings or accept them and create a backlog issue. diff --git a/doc/development/internal_api.md b/doc/development/internal_api.md index 4b46787c2c3..e25feda356d 100644 --- a/doc/development/internal_api.md +++ b/doc/development/internal_api.md @@ -52,7 +52,7 @@ POST /internal/allowed | `username` | string | no | Username from the certificate used to connect to GitLab Shell | | `project` | string | no (if `gl_repository` is passed) | Path to the project | | `gl_repository` | string | no (if `project` is passed) | Repository identifier (e.g. `project-7`) | -| `protocol` | string | yes | SSH when called from GitLab-shell, HTTP or SSH when called from Gitaly | +| `protocol` | string | yes | SSH when called from GitLab Shell, HTTP or SSH when called from Gitaly | | `action` | string | yes | Git command being run (`git-upload-pack`, `git-receive-pack`, `git-upload-archive`) | | `changes` | string | yes | `<oldrev> <newrev> <refname>` when called from Gitaly, the magic string `_any` when called from GitLab Shell | | `check_ip` | string | no | IP address from which call to GitLab Shell was made | @@ -223,6 +223,7 @@ Example response: - GitLab Geo - GitLab Shell's `bin/check` +- Gitaly ## Get new 2FA recovery codes using an SSH key diff --git a/doc/development/kubernetes.md b/doc/development/kubernetes.md index 64089d51c7b..a54798b4c35 100644 --- a/doc/development/kubernetes.md +++ b/doc/development/kubernetes.md @@ -15,14 +15,14 @@ This document provides various guidelines when developing for GitLab's Some Kubernetes operations, such as creating restricted project namespaces are performed on the GitLab Rails application. These -operations are performed using a [client library](#client-library). -These operations will carry an element of risk as the operations will be -run as the same user running the GitLab Rails application, see the -[security](#security) section below. +operations are performed using a [client library](#client-library), +and carry an element of risk. The operations are +run as the same user running the GitLab Rails application. For more information, +read the [security](#security) section below. Some Kubernetes operations, such as installing cluster applications are performed on one-off pods on the Kubernetes cluster itself. These -installation pods are currently named `install-<application_name>` and +installation pods are named `install-<application_name>` and are created within the `gitlab-managed-apps` namespace. In terms of code organization, we generally add objects that represent @@ -33,33 +33,32 @@ Kubernetes resources in We use the [`kubeclient`](https://rubygems.org/gems/kubeclient) gem to perform Kubernetes API calls. As the `kubeclient` gem does not support -different API Groups (e.g. `apis/rbac.authorization.k8s.io`) from a +different API Groups (such as `apis/rbac.authorization.k8s.io`) from a single client, we have created a wrapper class, [`Gitlab::Kubernetes::KubeClient`](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/kubernetes/kube_client.rb) -that will enable you to achieve this. +that enable you to achieve this. -Selected Kubernetes API groups are currently supported. Do add support +Selected Kubernetes API groups are supported. Do add support for new API groups or methods to [`Gitlab::Kubernetes::KubeClient`](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/kubernetes/kube_client.rb) if you need to use them. New API groups or API group versions can be -added to `SUPPORTED_API_GROUPS` - internally, this will create an +added to `SUPPORTED_API_GROUPS` - internally, this creates an internal client for that group. New methods can be added as a delegation to the relevant internal client. ### Performance considerations -All calls to the Kubernetes API must be in a background process. Do not -perform Kubernetes API calls within a web request as this will block -Unicorn and can easily lead to a Denial Of Service (DoS) attack in GitLab as +All calls to the Kubernetes API must be in a background process. Don't +perform Kubernetes API calls within a web request. This blocks +Unicorn, and can lead to a denial-of-service (DoS) attack in GitLab as the Kubernetes cluster response times are outside of our control. The easiest way to ensure your calls happen a background process is to delegate any such work to happen in a [Sidekiq worker](sidekiq_style_guide.md). -There are instances where you would like to make calls to Kubernetes and -return the response and as such a background worker does not seem to be -a good fit. For such cases you should make use of [reactive -caching](https://gitlab.com/gitlab-org/gitlab/blob/master/app/models/concerns/reactive_caching.rb). +You may want to make calls to Kubernetes and return the response, but a background +worker isn't a good fit. Consider using +[reactive caching](https://gitlab.com/gitlab-org/gitlab/blob/master/app/models/concerns/reactive_caching.rb). For example: ```ruby @@ -76,7 +75,7 @@ For example: ### Testing -We have some Webmock stubs in +We have some WebMock stubs in [`KubernetesHelpers`](https://gitlab.com/gitlab-org/gitlab/blob/master/spec/support/helpers/kubernetes_helpers.rb) which can help with mocking out calls to Kubernetes API in your tests. @@ -86,8 +85,8 @@ This section outlines the process for allowing a GitLab instance to create EKS c The following prerequisites are required: -A `Customer` AWS account. This is the account in which the -EKS cluster will be created. The following resources must be present: +A `Customer` AWS account. The EKS cluster is created in this account. The following +resources must be present: - A provisioning role that has permissions to create the cluster and associated resources. It must list the `GitLab` AWS account @@ -114,7 +113,7 @@ The process for creating a cluster is as follows: which takes somewhere between 10 and 15 minutes in most cases. 1. When the stack is ready, GitLab stores the cluster details and generates another set of temporary credentials, this time to allow connecting to - the cluster via Kubeclient. These credentials are valid for one minute. + the cluster via `kubeclient`. These credentials are valid for one minute. 1. GitLab configures the worker nodes so that they are able to authenticate to the cluster, and creates a service account for itself for future operations. 1. Credentials that are no longer required are removed. This deletes the following @@ -126,7 +125,7 @@ The process for creating a cluster is as follows: ## Security -### SSRF +### Server Side Request Forgery (SSRF) attacks As URLs for Kubernetes clusters are user controlled it is easily susceptible to Server Side Request Forgery (SSRF) attacks. You should @@ -153,11 +152,11 @@ Mitigation strategies include: app.make_errored!("Kubernetes error: #{e.error_code}") ``` -## Debugging +## Debugging Kubernetes integrations Logs related to the Kubernetes integration can be found in [`kubernetes.log`](../administration/logs.md#kuberneteslog). On a local -GDK install, this will be present in `log/kubernetes.log`. +GDK install, these logs are present in `log/kubernetes.log`. Some services such as [`Clusters::Applications::InstallService`](https://gitlab.com/gitlab-org/gitlab/blob/master/app/services/clusters/applications/install_service.rb#L18) @@ -176,7 +175,9 @@ kubectl logs <pod_name> --follow -n gitlab-managed-apps ## GitLab Managed Apps -GitLab provides [GitLab Managed Apps](../user/clusters/applications.md), a one-click install for various applications which can be added directly to your configured cluster. +GitLab provides [GitLab Managed Apps](../user/clusters/applications.md), a one-click +install for various applications which can be added directly to your configured cluster. **<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> -For an overview of how to add a new GitLab-managed app, see [How to add GitLab-managed-apps to Kubernetes integration](https://youtu.be/mKm-jkranEk).** +For an overview of how to add a new GitLab-managed app, see +[How to add GitLab-managed-apps to Kubernetes integration](https://youtu.be/mKm-jkranEk).** diff --git a/doc/development/logging.md b/doc/development/logging.md index 474a500da61..14812978f2d 100644 --- a/doc/development/logging.md +++ b/doc/development/logging.md @@ -1,6 +1,6 @@ --- stage: Monitor -group: APM +group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- @@ -287,7 +287,6 @@ method or variable shouldn't be evaluated right away) See our [HOWTO: Use Sidekiq metadata logs](https://www.youtube.com/watch?v=_wDllvO_IY0) for further knowledge on creating visualizations in Kibana. -NOTE: **Note:** The fields of the context are currently only logged for Sidekiq jobs triggered through web requests. See the [follow-up work](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/68) diff --git a/doc/development/migration_style_guide.md b/doc/development/migration_style_guide.md index 207dd02d258..71191d1d871 100644 --- a/doc/development/migration_style_guide.md +++ b/doc/development/migration_style_guide.md @@ -295,13 +295,16 @@ end Adding foreign key to `projects`: +We can use the `add_concurrenct_foreign_key` method in this case, as this helper method +has the lock retries built into it. + ```ruby include Gitlab::Database::MigrationHelpers +disable_ddl_transaction! + def up - with_lock_retries do - add_foreign_key :imports, :projects, column: :project_id, on_delete: :cascade # rubocop:disable Migration/AddConcurrentForeignKey - end + add_concurrent_foreign_key :imports, :projects, column: :project_id, on_delete: :cascade end def down @@ -316,10 +319,10 @@ Adding foreign key to `users`: ```ruby include Gitlab::Database::MigrationHelpers +disable_ddl_transaction! + def up - with_lock_retries do - add_foreign_key :imports, :users, column: :user_id, on_delete: :cascade # rubocop:disable Migration/AddConcurrentForeignKey - end + add_concurrent_foreign_key :imports, :users, column: :user_id, on_delete: :cascade end def down @@ -331,7 +334,7 @@ end **Usage with `disable_ddl_transaction!`** -Generally the `with_lock_retries` helper should work with `disabled_ddl_transaction!`. A custom RuboCop rule ensures that only allowed methods can be placed within the lock retries block. +Generally the `with_lock_retries` helper should work with `disable_ddl_transaction!`. A custom RuboCop rule ensures that only allowed methods can be placed within the lock retries block. ```ruby disable_ddl_transaction! @@ -348,7 +351,7 @@ end The RuboCop rule generally allows standard Rails migration methods, listed below. This example will cause a Rubocop offense: ```ruby -disabled_ddl_transaction! +disable_ddl_transaction! def up with_lock_retries do @@ -364,17 +367,7 @@ standard Rails migration helper methods. Calling more than one migration helper is not a problem if they're executed on the same table. Using the `with_lock_retries` helper method is advised when a database -migration involves one of the high-traffic tables: - -- `users` -- `projects` -- `namespaces` -- `gitlab_subscriptions` -- `issues` -- `merge_requests` -- `ci_pipelines` -- `ci_builds` -- `notes` +migration involves one of the [high-traffic tables](https://gitlab.com/gitlab-org/gitlab/-/blob/master/rubocop/rubocop-migrations.yml#L3). Example changes: @@ -612,7 +605,7 @@ See the style guide on [`NOT NULL` constraints](database/not_null_constraints.md ## Adding Columns With Default Values -With PostgreSQL 11 being the minimum version since GitLab 13.0, adding columns with default values has become much easier and +With PostgreSQL 11 being the minimum version in GitLab 13.0 and later, adding columns with default values has become much easier and the standard `add_column` helper should be used in all cases. Before PostgreSQL 11, adding a column with a default was problematic as it would @@ -647,7 +640,7 @@ tables: `namespaces`. This can be translated to: ```sql ALTER TABLE namespaces ALTER COLUMN request_access_enabled -DEFAULT false +SET DEFAULT false ``` In this particular case, the default value exists and we're just changing the metadata for diff --git a/doc/development/multi_version_compatibility.md b/doc/development/multi_version_compatibility.md index 5ca43b9b818..714be296b40 100644 --- a/doc/development/multi_version_compatibility.md +++ b/doc/development/multi_version_compatibility.md @@ -122,7 +122,7 @@ If we look at this schema from a database point of view, we can see two deployme And these deployments align perfectly with application changes. 1. At the beginning we have `Version N` on `Schema A`. -1. Then we have a _long_ transition periond with both `Version N` and `Version N+1` on `Schema B`. +1. Then we have a _long_ transition period with both `Version N` and `Version N+1` on `Schema B`. 1. When we only have `Version N+1` on `Schema B` the schema changes again. 1. Finally we have `Version N+1` on `Schema C`. diff --git a/doc/development/new_fe_guide/development/accessibility.md b/doc/development/new_fe_guide/development/accessibility.md index b9ee5c3a549..9c63ccad6e1 100644 --- a/doc/development/new_fe_guide/development/accessibility.md +++ b/doc/development/new_fe_guide/development/accessibility.md @@ -12,7 +12,7 @@ WAI-ARIA (the Accessible Rich Internet Applications specification) defines a way The `role` attribute describes the role the element plays in the context of the document. -Check the list of WAI-ARIA roles [here](https://www.w3.org/TR/wai-aria-1.1/#landmark_roles) +Review the list of [WAI-ARIA roles](https://www.w3.org/TR/wai-aria-1.1/#landmark_roles). ## Icons diff --git a/doc/development/new_fe_guide/development/index.md b/doc/development/new_fe_guide/development/index.md index 5dced3dc466..119dbc58012 100644 --- a/doc/development/new_fe_guide/development/index.md +++ b/doc/development/new_fe_guide/development/index.md @@ -12,6 +12,6 @@ Learn how to implement an accessible frontend. Learn how to keep our frontend performant. -## [Testing](testing.md) +## [Testing](../../testing_guide/frontend_testing.md) Learn how to keep our frontend tested. diff --git a/doc/development/packages.md b/doc/development/packages.md index 55e22d4bb5f..9eae99ff890 100644 --- a/doc/development/packages.md +++ b/doc/development/packages.md @@ -64,15 +64,16 @@ will have to be managed. Using instance level endpoints requires [stricter namin The current state of existing package registries availability is: -| Repository Type | Project Level | Group Level | Instance Level | -|-----------------|---------------|-------------|----------------| -| Maven | Yes | Yes | Yes | -| Conan | Yes | No - [open issue](https://gitlab.com/gitlab-org/gitlab/-/issues/11679) | Yes | -| NPM | No - [open issue](https://gitlab.com/gitlab-org/gitlab/-/issues/36853) | Yes | No - [open issue](https://gitlab.com/gitlab-org/gitlab/-/issues/36853) | -| NuGet | Yes | No - [open issue](https://gitlab.com/gitlab-org/gitlab/-/issues/36423) | No | -| PyPI | Yes | No | No | -| Go | Yes | No - [open issue](https://gitlab.com/gitlab-org/gitlab/-/issues/213900) | No - [open-issue](https://gitlab.com/gitlab-org/gitlab/-/issues/213902) | -| Composer | Yes | Yes | No | +| Repository Type | Project Level | Group Level | Instance Level | +|------------------|---------------|-------------|----------------| +| Maven | Yes | Yes | Yes | +| Conan | Yes | No - [open issue](https://gitlab.com/gitlab-org/gitlab/-/issues/11679) | Yes | +| NPM | No - [open issue](https://gitlab.com/gitlab-org/gitlab/-/issues/36853) | Yes | No - [open issue](https://gitlab.com/gitlab-org/gitlab/-/issues/36853) | +| NuGet | Yes | No - [open issue](https://gitlab.com/gitlab-org/gitlab/-/issues/36423) | No | +| PyPI | Yes | No | No | +| Go | Yes | No - [open issue](https://gitlab.com/gitlab-org/gitlab/-/issues/213900) | No - [open-issue](https://gitlab.com/gitlab-org/gitlab/-/issues/213902) | +| Composer | Yes | Yes | No | +| Generic | Yes | No | No | NOTE: **Note:** NPM is currently a hybrid of the instance level and group level. @@ -87,7 +88,7 @@ Composer package naming scope is Instance Level. To avoid name conflict for instance-level endpoints you will need to define a package naming convention that gives a way to identify the project that the package belongs to. This generally involves using the project ID or full project path in the package name. See -[Conan's naming convention](../user/packages/conan_repository/index.md#package-recipe-naming-convention-for-instance-level-remote) as an example. +[Conan's naming convention](../user/packages/conan_repository/index.md#package-recipe-naming-convention-for-instance-remotes) as an example. For group and project-level endpoints, naming can be less constrained and it will be up to the group and project members to be certain that there is no conflict between two package names. However, the system should prevent diff --git a/doc/development/performance.md b/doc/development/performance.md index 1dc7538c55b..3b59393bae6 100644 --- a/doc/development/performance.md +++ b/doc/development/performance.md @@ -247,8 +247,12 @@ The following configuration options can be configured: - `STACKPROF_ENABLED`: Enables stackprof signal handler on SIGUSR2 signal. Defaults to `false`. -- `STACKPROF_INTERVAL_US`: Sampling interval in microseconds. Defaults to - `10000` μs (100hz). +- `STACKPROF_MODE`: See [sampling modes](https://github.com/tmm1/stackprof#sampling). + Defaults to `cpu`. +- `STACKPROF_INTERVAL`: Sampling interval. Unit semantics depend on `STACKPROF_MODE`. + For `object` mode this is a per-event interval (every `n`th event will be sampled) + and defaults to `1000`. + For other modes such as `cpu` this is a frequency and defaults to `10000` μs (100hz). - `STACKPROF_FILE_PREFIX`: File path prefix where profiles are stored. Defaults to `$TMPDIR` (often corresponds to `/tmp`). - `STACKPROF_TIMEOUT_S`: Profiling timeout in seconds. Profiling will diff --git a/doc/development/pipelines.md b/doc/development/pipelines.md index 7756ef376fc..d12220d8c95 100644 --- a/doc/development/pipelines.md +++ b/doc/development/pipelines.md @@ -122,7 +122,6 @@ graph RL; click 2_2-5 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8404303&udv=0" subgraph "Needs `setup-test-env` & `compile-test-assets`"; 2_2-2 & 2_2-4 & 2_2-5 --> 1-6 & 1-3; - 2_2-3 --> 1-6 & 1-4; end 2_3-1["build-assets-image (2.5 minutes)"]; @@ -228,7 +227,6 @@ graph RL; click 2_2-5 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8404303&udv=0" subgraph "Needs `setup-test-env` & `compile-test-assets`"; 2_2-2 & 2_2-4 & 2_2-5 --> 1-6 & 1-3; - 2_2-3 --> 1-6 & 1-4; end 2_3-1["build-assets-image (2.5 minutes)"]; diff --git a/doc/development/policies.md b/doc/development/policies.md index 8f05948cb41..8dfd4763551 100644 --- a/doc/development/policies.md +++ b/doc/development/policies.md @@ -171,7 +171,7 @@ class ParentPolicy < BasePolicy condition(:speaks_spanish) { @subject.spoken_languages.include?(:es) } condition(:has_license) { @subject.driving_license.present? } condition(:enjoys_broccoli) { @subject.enjoyment_of(:broccoli) > 0 } - + rule { speaks_spanish }.enable :read_spanish rule { has_license }.enable :drive_car rule { enjoys_broccoli }.enable :eat_broccoli @@ -190,7 +190,7 @@ child policy, for example: ```ruby class ChildPolicy < BasePolicy delegate { @subject.parent } - + rule { default }.prevent :drive_car end ``` @@ -211,11 +211,11 @@ The solution it to override the `:eat_broccoli` ability in the child policy: ```ruby class ChildPolicy < BasePolicy delegate { @subject.parent } - + overrides :eat_broccoli - + condition(:good_kid) { @subject.behavior_level >= Child::GOOD } - + rule { good_kid }.enable :eat_broccoli end ``` diff --git a/doc/development/product_analytics/event_dictionary.md b/doc/development/product_analytics/event_dictionary.md new file mode 100644 index 00000000000..b049db21c30 --- /dev/null +++ b/doc/development/product_analytics/event_dictionary.md @@ -0,0 +1,32 @@ +--- +stage: Growth +group: Product Analytics +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Event Dictionary + +**Note: We've temporarily moved the Event Dictionary to a [Google Sheet](https://docs.google.com/spreadsheets/d/1VzE8R72Px_Y_LlE3Z05LxUlG_dumWe3vl-HeUo70TPw/edit?usp=sharing)**. The previous Markdown table exceeded 600 rows making it difficult to manage. In the future, our intention is to move this back into our docs using a [YAML file](https://gitlab.com/gitlab-org/gitlab-docs/-/issues/823). + +The event dictionary is a single source of truth for the metrics and events we collect for product usage data. The Event Dictionary lists all the metrics and events we track, why we're tracking them, and where they are tracked. + +This is a living document that is updated any time a new event is planned or implemented. It includes the following information. + +- Section, stage, or group +- Description +- Implementation status +- Availability by plan type +- Code path + +We're currently focusing our Event Dictionary on [Usage Ping](usage_ping.md). In the future, we will also include [Snowplow](snowplow.md). We currently have an initiative across the entire product organization to complete the [Event Dictionary for Usage Ping](https://gitlab.com/groups/gitlab-org/-/epics/4174). + +## Instructions + +1. Open the Event Dictionary and fill in all the **PM to edit** columns highlighted in yellow. +1. Check that all the metrics and events are assigned to the correct section, stage, or group. If a metric is used across many groups, assign it to the stage. If a metric is used across many stages, assign it to the section. If a metric is incorrectly assigned to another section, stage, or group, let the PM know you have reassigned it. If your group has no assigned metrics and events, check that your metrics and events are not incorrectly assigned to another PM. +1. Add descriptions of what your metrics and events are tracking. Work with your Engineering team or the Product Analytics team if you need help understanding this. +1. Add what plans this metric is available on. Work with your Engineering team or the Product Analytics team if you need help understanding this. + +## Planned metrics and events + +For future metrics and events you plan to track, please add them to the Event Dictionary and note the status as `Planned`, `In Progress`, or `Implemented`. Once you have confirmed the metric has been implemented and have confirmed the metric data is in our data warehouse, change the status to **Data Available**. diff --git a/doc/development/product_analytics/index.md b/doc/development/product_analytics/index.md new file mode 100644 index 00000000000..ab76d6f0561 --- /dev/null +++ b/doc/development/product_analytics/index.md @@ -0,0 +1,182 @@ +--- +stage: Growth +group: Product Analytics +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Product Analytics Guide + +At GitLab, we collect product usage data for the purpose of helping us build a better product. Data helps GitLab understand which parts of the product need improvement and which features we should build next. Product usage data also helps our team better understand the reasons why people use GitLab. With this knowledge we are able to make better product decisions. + +We encourage users to enable tracking, and we embrace full transparency with our tracking approach so it can be easily understood and trusted. + +By enabling tracking, users can: + +- Contribute back to the wider community. +- Help GitLab improve on the product. + +## Our tracking tools + +We use three methods to gather product usage data: + +- [Snowplow](#snowplow) +- [Usage Ping](#usage-ping) +- [Database import](#database-import) + +### Snowplow + +Snowplow is an enterprise-grade marketing and product analytics platform which helps track the way +users engage with our website and application. + +Snowplow consists of two components: + +- [Snowplow JS](https://github.com/snowplow/snowplow/wiki/javascript-tracker) tracks client-side + events. +- [Snowplow Ruby](https://github.com/snowplow/snowplow/wiki/ruby-tracker) tracks server-side events. + +For more details, read the [Snowplow](snowplow.md) guide. + +### Usage Ping + +Usage Ping is a method for GitLab Inc to collect usage data on a GitLab instance. Usage Ping is primarily composed of row counts for different tables in the instance’s database. By comparing these counts month over month (or week over week), we can get a rough sense for how an instance is using the different features within the product. This high-level data is used to help our product, support, and sales teams. + +For more details, read the [Usage Ping](usage_ping.md) guide. + +### Database import + +Database imports are full imports of data into GitLab's data warehouse. For GitLab.com, the PostgreSQL database is loaded into Snowflake data warehouse every 6 hours. For more details, see the [data team handbook](https://about.gitlab.com/handbook/business-ops/data-team/platform/#extract-and-load). + +## What data can be tracked + +Our different tracking tools allows us to track different types of events. The event types and examples of what data can be tracked are outlined below. + +The availability of event types and their tracking tools varies by segment. For example, on Self-Managed Users, we only have reporting using Database records via Usage Ping. + +| Event Types | SaaS Instance | SaaS Plan | SaaS Group | SaaS Session | SaaS User | SM Instance | SM Plan | SM Group | SM Session | SM User | +|----------------------------------------|---------------|-----------|------------|--------------|-----------|-------------|---------|----------|------------|---------| +| Snowplow (JS Pageview events) | ✅ | 📅 | 📅 | ✅ | 📅 | 📅 | 📅 | 📅 | 📅 | 📅 | +| Snowplow (JS UI events) | ✅ | 📅 | 📅 | ✅ | 📅 | 📅 | 📅 | 📅 | 📅 | 📅 | +| Snowplow (Ruby Pageview events) | ✅ | 📅 | 📅 | ✅ | 📅 | 📅 | 📅 | 📅 | 📅 | 📅 | +| Snowplow (Ruby CRUD / API events) | ✅ | 📅 | 📅 | ✅ | 📅 | 📅 | 📅 | 📅 | 📅 | 📅 | +| Usage Ping (Redis UI counters) | 🔄 | 🔄 | 🔄 | ✖️ | 🔄 | 🔄 | 🔄 | 🔄 | ✖️ | 🔄 | +| Usage Ping (Redis Pageview counters) | 🔄 | 🔄 | 🔄 | ✖️ | 🔄 | 🔄 | 🔄 | 🔄 | ✖️ | 🔄 | +| Usage Ping (Redis CRUD / API counters) | 🔄 | 🔄 | 🔄 | ✖️ | 🔄 | 🔄 | 🔄 | 🔄 | ✖️ | 🔄 | +| Usage Ping (Database counters) | ✅ | 🔄 | 📅 | ✖️ | ✅ | ✅ | ✅ | ✅ | ✖️ | ✅ | +| Usage Ping (Instance settings) | ✅ | 🔄 | 📅 | ✖️ | ✅ | ✅ | ✅ | ✅ | ✖️ | ✅ | +| Usage Ping (Integration settings) | ✅ | 🔄 | 📅 | ✖️ | ✅ | ✅ | ✅ | ✅ | ✖️ | ✅ | +| Database import (Database records) | ✅ | ✅ | ✅ | ✖️ | ✅ | ✖️ | ✖️ | ✖️ | ✖️ | ✖️ | + +[Source file](https://docs.google.com/spreadsheets/d/1e8Afo41Ar8x3JxAXJF3nL83UxVZ3hPIyXdt243VnNuE/edit?usp=sharing) + +**Legend** + +✅ Available, 🔄 In Progress, 📅 Planned, ✖️ Not Possible + +SaaS = GitLab.com. SM = Self-Managed instance + +### Pageview events + +- Number of sessions that visited the /dashboard/groups page + +### UI events + +- Number of sessions that clicked on a button or link +- Number of sessions that closed a modal + +UI events are any interface-driven actions from the browser including click data. + +### CRUD or API events + +- Number of Git pushes +- Number of GraphQL queries +- Number of requests to a Rails action or controller + +These are backend events that include the creation, read, update, deletion of records, and other events that might be triggered from layers other than those available in the interface. + +### Database records + +These are raw database records which can be explored using business intelligence tools like Sisense. The full list of available tables can be found in [structure.sql](https://gitlab.com/gitlab-org/gitlab/-/blob/master/db/structure.sql). + +### Instance settings + +These are settings of your instance such as the instance's Git version and if certain features are enabled such as `container_registry_enabled`. + +### Integration settings + +These are integrations your GitLab instance interacts with such as an [external storage provider](../../administration/static_objects_external_storage.md) or an [external container registry](../../administration/packages/container_registry.md#use-an-external-container-registry-with-gitlab-as-an-auth-endpoint). These services must be able to send data back into a GitLab instance for data to be tracked. + +## Reporting level + +Our reporting levels of aggregate or individual reporting varies by segment. For example, on Self-Managed Users, we can report at an aggregate user level using Usage Ping but not on an Individual user level. + +| Aggregated Reporting | SaaS Instance | SaaS Plan | SaaS Group | SaaS Session | SaaS User | SM Instance | SM Plan | SM Group | SM Session | SM User | +|----------------------|---------------|-----------|------------|--------------|-----------|-------------|---------|----------|------------|---------| +| Snowplow | ✅ | 📅 | 📅 | ✅ | 📅 | ✅ | 📅 | 📅 | ✅ | 📅 | +| Usage Ping | ✅ | 🔄 | 📅 | 📅 | ✅ | ✅ | ✅ | ✅ | 📅 | ✅ | +| Database import | ✅ | ✅ | ✅ | ✖️ | ✅ | ✖️ | ✖️ | ✖️ | ✖️ | ✖️ | + +| Identifiable Reporting | SaaS Instance | SaaS Plan | SaaS Group | SaaS Session | SaaS User | SM Instance | SM Plan | SM Group | SM Session | SM User | +|------------------------|---------------|-----------|------------|--------------|-----------|-------------|---------|----------|------------|---------| +| Snowplow | ✅ | 📅 | 📅 | ✅ | 📅 | ✖️ | ✖️ | ✖️ | ✖️ | ✖️ | +| Usage Ping | ✅ | 🔄 | 📅 | ✖️ | ✖️ | ✅ | ✅ | ✖️ | ✖️ | ✖️ | +| Database import | ✅ | ✅ | ✅ | ✖️ | ✅ | ✖️ | ✖️ | ✖️ | ✖️ | ✖️ | + +**Legend** + +✅ Available, 🔄 In Progress, 📅 Planned, ✖️ Not Possible + +SaaS = GitLab.com. SM = Self-Managed instance + +## Reporting time period + +Our reporting time periods varies by segment. For example, on Self-Managed Users, we can report all time counts and 28 day counts in Usage Ping. + +| Reporting Time Period | All Time | 28 Days | 7 Days | Daily | +|-----------------------|----------|---------|--------|-------| +| Snowplow | ✅ | ✅ | ✅ | ✅ | +| Usage Ping | ✅ | ✅ | 📅 | ✖️ | +| Database import | ✅ | ✅ | ✅ | ✅ | + +**Legend** + +✅ Available, 🔄 In Progress, 📅 Planned, ✖️ Not Possible + +## Systems overview + +The systems overview is a simplified diagram showing the interactions between GitLab Inc and self-managed instances. + + + +[Source file](https://app.diagrams.net/#G13DVpN-XnhWGz9tqReIj8pp1UE4ehk_EC) + +### GitLab Inc + +For Product Analytics purposes, GitLab Inc has three major components: + +1. [Data Infrastructure](https://about.gitlab.com/handbook/business-ops/data-team/platform/infrastructure/): This contains everything managed by our data team including Sisense Dashboards for visualization, Snowflake for Data Warehousing, incoming data sources such as PostgreSQL Pipeline and S3 Bucket, and lastly our data collectors [GitLab.com's Snowplow Collector](https://gitlab.com/gitlab-com/gl-infra/readiness/-/tree/master/library/snowplow/) and GitLab's Versions Application. +1. GitLab.com: This is the production GitLab application which is made up of a Client and Server. On the Client or browser side, a Snowplow JS Tracker (Frontend) is used to track client-side events. On the Server or application side, a Snowplow Ruby Tracker (Backend) is used to track server-side events. The server also contains Usage Ping which leverages a PostgreSQL database and a Redis in-memory data store to report on usage data. Lastly, the server also contains System Logs which are generated from running the GitLab application. +1. [Monitoring infrastructure](https://about.gitlab.com/handbook/engineering/monitoring/): This is the infrastructure used to ensure GitLab.com is operating smoothly. System Logs are sent from GitLab.com to our monitoring infrastructure and collected by a FluentD collector. From FluentD, logs are either sent to long term Google Cloud Services cold storage via Stackdriver, or, they are sent to our Elastic Cluster via Cloud Pub/Sub which can be explored in real-time using Kibana. + +### Self-managed + +For Product Analytics purposes, self-managed instances have two major components: + +1. Data infrastructure: Having a data infrastructure setup is optional on self-managed instances. If you'd like to collect Snowplow tracking events for your self-managed instance, you can setup your own self-managed Snowplow collector and configure your Snowplow events to point to your own collector. +1. GitLab: A self-managed GitLab instance contains all of the same components as GitLab.com mentioned above. + +### Differences between GitLab Inc and Self-managed + +As shown by the orange lines, on GitLab.com Snowplow JS, Snowplow Ruby, Usage Ping, and PostgreSQL database imports all flow into GitLab Inc's data infrastructure. However, on self-managed, only Usage Ping flows into GitLab Inc's data infrastructure. + +As shown by the green lines, on GitLab.com system logs flow into GitLab Inc's monitoring infrastructure. On self-managed, there are no logs sent to GitLab Inc's monitoring infrastructure. + +Note (1): Snowplow JS and Snowplow Ruby are available on self-managed, however, the Snowplow Collector endpoint is set to a self-managed Snowplow Collector which GitLab Inc does not have access to. + +## Additional information + +More useful links: + +- [Product Analytics Direction](https://about.gitlab.com/direction/product-analytics/) +- [Data Analysis Process](https://about.gitlab.com/handbook/business-ops/data-team/#data-analysis-process/) +- [Data for Product Managers](https://about.gitlab.com/handbook/business-ops/data-team/programs/data-for-product-managers/) +- [Data Infrastructure](https://about.gitlab.com/handbook/business-ops/data-team/platform/infrastructure/) diff --git a/doc/development/product_analytics/snowplow.md b/doc/development/product_analytics/snowplow.md new file mode 100644 index 00000000000..21d92566ffd --- /dev/null +++ b/doc/development/product_analytics/snowplow.md @@ -0,0 +1,429 @@ +--- +stage: Growth +group: Product Analytics +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Snowplow Guide + +This guide provides an overview of how Snowplow works, and implementation details. + +For more information about Product Analytics, see: + +- [Product Analytics Guide](index.md) +- [Usage Ping Guide](usage_ping.md) + +More useful links: + +- [Product Analytics Direction](https://about.gitlab.com/direction/product-analytics/) +- [Data Analysis Process](https://about.gitlab.com/handbook/business-ops/data-team/#data-analysis-process/) +- [Data for Product Managers](https://about.gitlab.com/handbook/business-ops/data-team/programs/data-for-product-managers/) +- [Data Infrastructure](https://about.gitlab.com/handbook/business-ops/data-team/platform/infrastructure/) + +## What is Snowplow + +Snowplow is an enterprise-grade marketing and product analytics platform which helps track the way users engage with our website and application. + +[Snowplow](https://github.com/snowplow/snowplow) consists of the following loosely-coupled sub-systems: + +- **Trackers** fire Snowplow events. Snowplow has 12 trackers, covering web, mobile, desktop, server, and IoT. +- **Collectors** receive Snowplow events from trackers. We have three different event collectors, synchronizing events either to Amazon S3, Apache Kafka, or Amazon Kinesis. +- **Enrich** cleans up the raw Snowplow events, enriches them and puts them into storage. We have an Hadoop-based enrichment process, and a Kinesis-based or Kafka-based process. +- **Storage** is where the Snowplow events live. We store the Snowplow events in a flat file structure on S3, and in the Redshift and PostgreSQL databases. +- **Data modeling** is where event-level data is joined with other data sets and aggregated into smaller data sets, and business logic is applied. This produces a clean set of tables which make it easier to perform analysis on the data. We have data models for Redshift and Looker. +- **Analytics** are performed on the Snowplow events or on the aggregate tables. + + + +## Snowplow schema + +We have many definitions of Snowplow's schema. We have an active issue to [standardize this schema](https://gitlab.com/gitlab-org/gitlab/-/issues/207930) including the following definitions: + +- Frontend and backend taxonomy as listed below +- [Structured event taxonomy](#structured-event-taxonomy) +- [Self describing events](https://github.com/snowplow/snowplow/wiki/Custom-events#self-describing-events) +- [Iglu schema](https://gitlab.com/gitlab-org/iglu/) +- [Snowplow authored events](https://github.com/snowplow/snowplow/wiki/Snowplow-authored-events) + +## Enabling Snowplow + +Tracking can be enabled at: + +- The instance level, which enables tracking on both the frontend and backend layers. +- User level, though user tracking can be disabled on a per-user basis. GitLab tracking respects the [Do Not Track](https://www.eff.org/issues/do-not-track) standard, so any user who has enabled the Do Not Track option in their browser is not tracked at a user level. + +We utilize Snowplow for the majority of our tracking strategy and it is enabled on GitLab.com. On a self-managed instance, Snowplow can be enabled by navigating to: + +- **Admin Area > Settings > General** in the UI. +- `admin/application_settings/integrations` in your browser. + +The following configuration is required: + +| Name | Value | +|---------------|---------------------------| +| Collector | `snowplow.trx.gitlab.net` | +| Site ID | `gitlab` | +| Cookie domain | `.gitlab.com` | + +## Snowplow request flow + +The following example shows a basic request/response flow between the following components: + +- Snowplow JS / Ruby Trackers on GitLab.com +- [GitLab.com Snowplow Collector](https://gitlab.com/gitlab-com/gl-infra/readiness/-/blob/master/library/snowplow/index.md) +- GitLab's S3 Bucket +- GitLab's Snowflake Data Warehouse +- Sisense: + +```mermaid +sequenceDiagram + participant Snowplow JS (Frontend) + participant Snowplow Ruby (Backend) + participant GitLab.com Snowplow Collector + participant S3 Bucket + participant Snowflake DW + participant Sisense Dashboards + Snowplow JS (Frontend) ->> GitLab.com Snowplow Collector: FE Tracking event + Snowplow Ruby (Backend) ->> GitLab.com Snowplow Collector: BE Tracking event + loop Process using Kinesis Stream + GitLab.com Snowplow Collector ->> GitLab.com Snowplow Collector: Log raw events + GitLab.com Snowplow Collector ->> GitLab.com Snowplow Collector: Enrich events + GitLab.com Snowplow Collector ->> GitLab.com Snowplow Collector: Write to disk + end + GitLab.com Snowplow Collector ->> S3 Bucket: Kinesis Firehose + S3 Bucket->>Snowflake DW: Import data + Snowflake DW->>Snowflake DW: Transform data using dbt + Snowflake DW->>Sisense Dashboards: Data available for querying +``` + +## Structured event taxonomy + +When adding new click events, we should add them in a way that's internally consistent. If we don't, it'll be very painful to perform analysis across features since each feature will be capturing events differently. + +The current method provides several attributes that are sent on each click event. Please try to follow these guidelines when specifying events to capture: + +| attribute | type | required | description | +| --------- | ------- | -------- | ----------- | +| category | text | true | The page or backend area of the application. Unless infeasible, please use the Rails page attribute by default in the frontend, and namespace + classname on the backend. | +| action | text | true | The action the user is taking, or aspect that's being instrumented. The first word should always describe the action or aspect: clicks should be `click`, activations should be `activate`, creations should be `create`, etc. Use underscores to describe what was acted on; for example, activating a form field would be `activate_form_input`. An interface action like clicking on a dropdown would be `click_dropdown`, while a behavior like creating a project record from the backend would be `create_project` | +| label | text | false | The specific element, or object that's being acted on. This is either the label of the element (e.g. a tab labeled 'Create from template' may be `create_from_template`) or a unique identifier if no text is available (e.g. closing the Groups dropdown in the top navbar might be `groups_dropdown_close`), or it could be the name or title attribute of a record being created. | +| property | text | false | Any additional property of the element, or object being acted on. | +| value | decimal | false | Describes a numeric value or something directly related to the event. This could be the value of an input (e.g. `10` when clicking `internal` visibility). | + +## Implementing Snowplow JS (Frontend) tracking + +GitLab provides `Tracking`, an interface that wraps the [Snowplow JavaScript Tracker](https://github.com/snowplow/snowplow/wiki/javascript-tracker) for tracking custom events. There are a few ways to utilize tracking, but each generally requires at minimum, a `category` and an `action`. Additional data can be provided that adheres to our [Structured event taxonomy](#structured-event-taxonomy). + +| field | type | default value | description | +|:-----------|:-------|:---------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `category` | string | document.body.dataset.page | Page or subsection of a page that events are being captured within. | +| `action` | string | 'generic' | Action the user is taking. Clicks should be `click` and activations should be `activate`, so for example, focusing a form field would be `activate_form_input`, and clicking a button would be `click_button`. | +| `data` | object | {} | Additional data such as `label`, `property`, `value`, and `context` as described in our [Structured event taxonomy](#structured-event-taxonomy). | + +### Tracking in HAML (or Vue Templates) + +When working within HAML (or Vue templates) we can add `data-track-*` attributes to elements of interest. All elements that have a `data-track-event` attribute automatically have event tracking bound on clicks. + +Below is an example of `data-track-*` attributes assigned to a button: + +```haml +%button.btn{ data: { track: { event: "click_button", label: "template_preview", property: "my-template" } } } +``` + +```html +<button class="btn" + data-track-event="click_button" + data-track-label="template_preview" + data-track-property="my-template" +/> +``` + +Event listeners are bound at the document level to handle click events on or within elements with these data attributes. This allows them to be properly handled on re-rendering and changes to the DOM. Note that because of the way these events are bound, click events should not be stopped from propagating up the DOM tree. If for any reason click events are being stopped from propagating, you need to implement your own listeners and follow the instructions in [Tracking in raw JavaScript](#tracking-in-raw-javascript). + +Below is a list of supported `data-track-*` attributes: + +| attribute | required | description | +|:----------------------|:---------|:------------| +| `data-track-event` | true | Action the user is taking. Clicks must be prepended with `click` and activations must be prepended with `activate`. For example, focusing a form field would be `activate_form_input` and clicking a button would be `click_button`. | +| `data-track-label` | false | The `label` as described in our [Structured event taxonomy](#structured-event-taxonomy). | +| `data-track-property` | false | The `property` as described in our [Structured event taxonomy](#structured-event-taxonomy). | +| `data-track-value` | false | The `value` as described in our [Structured event taxonomy](#structured-event-taxonomy). If omitted, this is the element's `value` property or an empty string. For checkboxes, the default value is the element's checked attribute or `false` when unchecked. | +| `data-track-context` | false | The `context` as described in our [Structured event taxonomy](#structured-event-taxonomy). | + +### Tracking within Vue components + +There's a tracking Vue mixin that can be used in components if more complex tracking is required. To use it, first import the `Tracking` library and request a mixin. + +```javascript +import Tracking from '~/tracking'; +const trackingMixin = Tracking.mixin({ label: 'right_sidebar' }); +``` + +You can provide default options that are passed along whenever an event is tracked from within your component. For instance, if all events within a component should be tracked with a given `label`, you can provide one at this time. Available defaults are `category`, `label`, `property`, and `value`. If no category is specified, `document.body.dataset.page` is used as the default. + +You can then use the mixin normally in your component with the `mixin` Vue declaration. The mixin also provides the ability to specify tracking options in `data` or `computed`. These override any defaults and allow the values to be dynamic from props, or based on state. + +```javascript +export default { + mixins: [trackingMixin], + // ...[component implementation]... + data() { + return { + expanded: false, + tracking: { + label: 'left_sidebar' + } + }; + }, +} +``` + +The mixin provides a `track` method that can be called within the template, or from component methods. An example of the whole implementation might look like the following. + +```javascript +export default { + mixins: [Tracking.mixin({ label: 'right_sidebar' })], + data() { + return { + expanded: false, + }; + }, + methods: { + toggle() { + this.expanded = !this.expanded; + this.track('click_toggle', { value: this.expanded }) + } + } +}; +``` + +And if needed within the template, you can use the `track` method directly as well. + +```html +<template> + <div> + <a class="toggle" @click.prevent="toggle">Toggle</a> + <div v-if="expanded"> + <p>Hello world!</p> + <a @click.prevent="track('click_action')">Track an event</a> + </div> + </div> +</template> +``` + +### Tracking in raw JavaScript + +Custom event tracking and instrumentation can be added by directly calling the `Tracking.event` static function. The following example demonstrates tracking a click on a button by calling `Tracking.event` manually. + +```javascript +import Tracking from '~/tracking'; + +const button = document.getElementById('create_from_template_button'); +button.addEventListener('click', () => { + Tracking.event('dashboard:projects:index', 'click_button', { + label: 'create_from_template', + property: 'template_preview', + value: 'rails', + }); +}) +``` + +### Tests and test helpers + +In Jest particularly in Vue tests, you can use the following: + +```javascript +import { mockTracking } from 'helpers/tracking_helper'; + +describe('MyTracking', () => { + let spy; + + beforeEach(() => { + spy = mockTracking('_category_', wrapper.element, jest.spyOn); + }); + + it('tracks an event when clicked on feedback', () => { + wrapper.find('.discover-feedback-icon').trigger('click'); + + expect(spy).toHaveBeenCalledWith('_category_', 'click_button', { + label: 'security-discover-feedback-cta', + property: '0', + }); + }); +}); +``` + +In obsolete Karma tests it's used as below: + +```javascript +import { mockTracking, triggerEvent } from 'spec/helpers/tracking_helper'; + +describe('my component', () => { + let trackingSpy; + + beforeEach(() => { + trackingSpy = mockTracking('_category_', vm.$el, spyOn); + }); + + const triggerEvent = () => { + // action which should trigger a event + }; + + it('tracks an event when toggled', () => { + expect(trackingSpy).not.toHaveBeenCalled(); + + triggerEvent('a.toggle'); + + expect(trackingSpy).toHaveBeenCalledWith('_category_', 'click_edit_button', { + label: 'right_sidebar', + property: 'confidentiality', + }); + }); +}); +``` + +## Implementing Snowplow Ruby (Backend) tracking + +GitLab provides `Gitlab::Tracking`, an interface that wraps the [Snowplow Ruby Tracker](https://github.com/snowplow/snowplow/wiki/ruby-tracker) for tracking custom events. + +Custom event tracking and instrumentation can be added by directly calling the `GitLab::Tracking.event` class method, which accepts the following arguments: + +| argument | type | default value | description | +|:-----------|:-------|:--------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `category` | string | 'application' | Area or aspect of the application. This could be `HealthCheckController` or `Lfs::FileTransformer` for instance. | +| `action` | string | 'generic' | The action being taken, which can be anything from a controller action like `create` to something like an Active Record callback. | +| `data` | object | {} | Additional data such as `label`, `property`, `value`, and `context` as described in [Structured event taxonomy](#structured-event-taxonomy). These are set as empty strings if you don't provide them. | + +Tracking can be viewed as either tracking user behavior, or can be utilized for instrumentation to monitor and visualize performance over time in an area or aspect of code. + +For example: + +```ruby +class Projects::CreateService < BaseService + def execute + project = Project.create(params) + + Gitlab::Tracking.event('Projects::CreateService', 'create_project', + label: project.errors.full_messages.to_sentence, + value: project.valid? + ) + end +end +``` + +### Unit testing + +Use the `expect_snowplow_event` helper when testing backend Snowplow events. See [testing best practices]( +https://docs.gitlab.com/ee/development/testing_guide/best_practices.html#test-snowplow-events) for details. + +### Performance + +We use the [AsyncEmitter](https://github.com/snowplow/snowplow/wiki/Ruby-Tracker#52-the-asyncemitter-class) when tracking events, which allows for instrumentation calls to be run in a background thread. This is still an active area of development. + +## Developing and testing Snowplow + +There are several tools for developing and testing Snowplow Event + +| Testing Tool | Frontend Tracking | Backend Tracking | Local Development Environment | Production Environment | Production Environment | +|----------------------------------------------|--------------------|---------------------|-------------------------------|------------------------|------------------------| +| Snowplow Analytics Debugger Chrome Extension | **{check-circle}** | **{dotted-circle}** | **{check-circle}** | **{check-circle}** | **{check-circle}** | +| Snowplow Inspector Chrome Extension | **{check-circle}** | **{dotted-circle}** | **{check-circle}** | **{check-circle}** | **{check-circle}** | +| Snowplow Micro | **{check-circle}** | **{check-circle}** | **{check-circle}** | **{dotted-circle}** | **{dotted-circle}** | +| Snowplow Mini | **{check-circle}** | **{check-circle}** | **{dotted-circle}** | **{status_preparing}** | **{status_preparing}** | + +**Legend** + +**{check-circle}** Available, **{status_preparing}** In progress, **{dotted-circle}** Not Planned + +### Preparing your MR for Review + +1. For frontend events, in the MR description section, add a screenshot of the event's relevant section using the [Snowplow Analytics Debugger](https://chrome.google.com/webstore/detail/snowplow-analytics-debugg/jbnlcgeengmijcghameodeaenefieedm) Chrome browser extension. +1. For backend events, please use Snowplow Micro and add the output of the Snowplow Micro good events `GET http://localhost:9090/micro/good`. + +### Snowplow Analytics Debugger Chrome Extension + +Snowplow Analytics Debugger is a browser extension for testing frontend events. This works on production, staging and local development environments. + +1. Install the [Snowplow Analytics Debugger](https://chrome.google.com/webstore/detail/snowplow-analytics-debugg/jbnlcgeengmijcghameodeaenefieedm) Chrome browser extension. +1. Open Chrome DevTools to the Snowplow Analytics Debugger tab. +1. Learn more at [Igloo Analytics](https://www.iglooanalytics.com/blog/snowplow-analytics-debugger-chrome-extension.html). + +### Snowplow Inspector Chrome Extension + +Snowplow Inspector Chrome Extension is a browser extension for testing frontend events. This works on production, staging and local development environments. + +1. Install [Snowplow Inspector](https://chrome.google.com/webstore/detail/snowplow-inspector/maplkdomeamdlngconidoefjpogkmljm?hl=en). +1. Open the Chrome extension by pressing the Snowplow Inspector icon beside the address bar. +1. Click around on a webpage with Snowplow and you should see JavaScript events firing in the inspector window. + +### Snowplow Micro + +Snowplow Micro is a very small version of a full Snowplow data collection pipeline: small enough that it can be launched by a test suite. Events can be recorded into Snowplow Micro just as they can a full Snowplow pipeline. Micro then exposes an API that can be queried. + +Snowplow Micro is a Docker-based solution for testing frontend and backend events in a local development environment. You need to modify GDK using the instructions below to set this up. + +- Read [Introducing Snowplow Micro](https://snowplowanalytics.com/blog/2019/07/17/introducing-snowplow-micro/) +- Look at the [Snowplow Micro repository](https://github.com/snowplow-incubator/snowplow-micro) +- Watch our [installation guide recording](https://www.youtube.com/watch?v=OX46fo_A0Ag) + +1. Install [Snowplow Micro](https://github.com/snowplow-incubator/snowplow-micro): + + ```shell + docker run --mount type=bind,source=$(pwd)/example,destination=/config -p 9090:9090 snowplow/snowplow-micro:latest --collector-config /config/micro.conf --iglu /config/iglu.json + ``` + +1. Install Snowplow Micro by cloning the settings in [this project](https://gitlab.com/gitlab-org/snowplow-micro-configuration): + + ```shell + git clone git@gitlab.com:gitlab-org/snowplow-micro-configuration.git + ./snowplow-micro.sh + ``` + +1. Update port in SQL to set `9090`: + + ```shell + gdk psql -d gitlabhq_development + update application_settings set snowplow_collector_hostname='localhost:9090', snowplow_enabled=true, snowplow_cookie_domain='.gitlab.com'; + ``` + +1. Update `app/assets/javascripts/tracking.js` to [remove this line](https://gitlab.com/snippets/1918635): + + ```javascript + forceSecureTracker: true + ``` + +1. Update `lib/gitlab/tracking.rb` to [add these lines](https://gitlab.com/snippets/1918635): + + ```ruby + protocol: 'http', + port: 9090, + ``` + +1. Update `lib/gitlab/tracking.rb` to [change async emitter from https to http](https://gitlab.com/snippets/1918635): + + ```ruby + SnowplowTracker::AsyncEmitter.new(Gitlab::CurrentSettings.snowplow_collector_hostname, protocol: 'http'), + ``` + +1. Enable Snowplow in the admin area, Settings::Integrations::Snowplow to point to: + `http://localhost:3000/admin/application_settings/integrations#js-snowplow-settings`. + +1. Restart GDK: + + ```shell + `gdk restart` + ``` + +1. Send a test Snowplow event from the Rails console: + + ```ruby + Gitlab::Tracking.self_describing_event('iglu:com.gitlab/pageview_context/jsonschema/1-0-0', { page_type: 'MY_TYPE' }, context: nil ) + ``` + +### Snowplow Mini + +[Snowplow Mini](https://github.com/snowplow/snowplow-mini) is an easily-deployable, single-instance version of Snowplow. + +Snowplow Mini can be used for testing frontend and backend events on a production, staging and local development environment. + +For GitLab.com, we're setting up a [QA and Testing environment](https://gitlab.com/gitlab-org/telemetry/-/issues/266) using Snowplow Mini. diff --git a/doc/development/product_analytics/usage_ping.md b/doc/development/product_analytics/usage_ping.md new file mode 100644 index 00000000000..d482af77d8a --- /dev/null +++ b/doc/development/product_analytics/usage_ping.md @@ -0,0 +1,937 @@ +--- +stage: Growth +group: Product Analytics +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Usage Ping Guide + +> - Introduced in GitLab Enterprise Edition 8.10. +> - More statistics were added in GitLab Enterprise Edition 8.12. +> - Moved to GitLab Core in 9.1. +> - More statistics were added in GitLab Ultimate 11.2. + +This guide describes Usage Ping's purpose and how it's implemented. + +For more information about Product Analytics, see: + +- [Product Analytics Guide](index.md) +- [Snowplow Guide](snowplow.md) + +More useful links: + +- [Product Analytics Direction](https://about.gitlab.com/direction/product-analytics/) +- [Data Analysis Process](https://about.gitlab.com/handbook/business-ops/data-team/#data-analysis-process/) +- [Data for Product Managers](https://about.gitlab.com/handbook/business-ops/data-team/programs/data-for-product-managers/) +- [Data Infrastructure](https://about.gitlab.com/handbook/business-ops/data-team/platform/infrastructure/) + +## What is Usage Ping? + +- GitLab sends a weekly payload containing usage data to GitLab Inc. Usage Ping provides high-level data to help our product, support, and sales teams. It does not send any project names, usernames, or any other specific data. The information from the usage ping is not anonymous, it is linked to the hostname of the instance. Sending usage ping is optional, and any instance can disable analytics. +- The usage data is primarily composed of row counts for different tables in the instance’s database. By comparing these counts month over month (or week over week), we can get a rough sense for how an instance is using the different features within the product. In addition to counts, other facts + that help us classify and understand GitLab installations are collected. +- Usage ping is important to GitLab as we use it to calculate our Stage Monthly Active Users (SMAU) which helps us measure the success of our stages and features. +- While usage ping is enabled, GitLab will gather data from the other instances and will be able to show usage statistics of your instance to your users. + +### Why should we enable Usage Ping? + +- The main purpose of Usage Ping is to build a better GitLab. Data about how GitLab is used is collected to better understand feature/stage adoption and usage, which helps us understand how GitLab is adding value and helps our team better understand the reasons why people use GitLab and with this knowledge we're able to make better product decisions. +- As a benefit of having the usage ping active, GitLab lets you analyze the users’ activities over time of your GitLab installation. +- As a benefit of having the usage ping active, GitLab provides you with The DevOps Report,which gives you an overview of your entire instance’s adoption of Concurrent DevOps from planning to monitoring. +- You will get better, more proactive support. (assuming that our TAMs and support organization used the data to deliver more value) +- You will get insight and advice into how to get the most value out of your investment in GitLab. Wouldn't you want to know that a number of features or values are not being adopted in your organization? +- You get a report that illustrates how you compare against other similar organizations (anonymized), with specific advice and recommendations on how to improve your DevOps processes. +- Usage Ping is enabled by default. To disable it, see [Disable Usage Ping](#disable-usage-ping). + +### Limitations + +- Usage Ping does not track frontend events things like page views, link clicks, or user sessions, and only focuses on aggregated backend events. +- Because of these limitations we recommend instrumenting your products with Snowplow for more detailed analytics on GitLab.com and use Usage Ping to track aggregated backend events on self-managed. + +## Usage Ping payload + +You can view the exact JSON payload sent to GitLab Inc. in the administration panel. To view the payload: + +1. Navigate to **Admin Area > Settings > Metrics and profiling**. +1. Expand the **Usage statistics** section. +1. Click the **Preview payload** button. + +For an example payload, see [Example Usage Ping payload](#example-usage-ping-payload). + +## Disable Usage Ping + +To disable Usage Ping in the GitLab UI, go to the **Settings** page of your administration panel and uncheck the **Usage Ping** checkbox. + +To disable Usage Ping and prevent it from being configured in the future through the administration panel, Omnibus installs can set the following in [`gitlab.rb`](https://docs.gitlab.com/omnibus/settings/configuration.html#configuration-options): + +```ruby +gitlab_rails['usage_ping_enabled'] = false +``` + +Source installations can set the following in `gitlab.yml`: + +```yaml +production: &base + # ... + gitlab: + # ... + usage_ping_enabled: false +``` + +## Usage Ping request flow + +The following example shows a basic request/response flow between a GitLab instance, the Versions Application, the License Application, Salesforce, GitLab's S3 Bucket, GitLab's Snowflake Data Warehouse, and Sisense: + +```mermaid +sequenceDiagram + participant GitLab Instance + participant Versions Application + participant Licenses Application + participant Salesforce + participant S3 Bucket + participant Snowflake DW + participant Sisense Dashboards + GitLab Instance->>Versions Application: Send Usage Ping + loop Process usage data + Versions Application->>Versions Application: Parse usage data + Versions Application->>Versions Application: Write to database + Versions Application->>Versions Application: Update license ping time + end + loop Process data for Salesforce + Versions Application-xLicenses Application: Request Zuora subscription id + Licenses Application-xVersions Application: Zuora subscription id + Versions Application-xSalesforce: Request Zuora account id by Zuora subscription id + Salesforce-xVersions Application: Zuora account id + Versions Application-xSalesforce: Usage data for the Zuora account + end + Versions Application->>S3 Bucket: Export Versions database + S3 Bucket->>Snowflake DW: Import data + Snowflake DW->>Snowflake DW: Transform data using dbt + Snowflake DW->>Sisense Dashboards: Data available for querying + Versions Application->>GitLab Instance: DevOps Report (Conversational Development Index) +``` + +## How Usage Ping works + +1. The Usage Ping [cron job](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/workers/gitlab_usage_ping_worker.rb#L30) is set in Sidekiq to run weekly. +1. When the cron job runs, it calls [`GitLab::UsageData.to_json`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/services/submit_usage_ping_service.rb#L22). +1. `GitLab::UsageData.to_json` [cascades down](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data.rb#L22) to ~400+ other counter method calls. +1. The response of all methods calls are [merged together](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data.rb#L14) into a single JSON payload in `GitLab::UsageData.to_json`. +1. The JSON payload is then [posted to the Versions application]( https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/services/submit_usage_ping_service.rb#L20). + +## Implementing Usage Ping + +Usage Ping consists of two kinds of data, counters and observations. Counters track how often a certain event +happened over time, such as how many CI pipelines have run. They are monotonic and always trend up. +Observations are facts collected from one or more GitLab instances and can carry arbitrary data. There are no +general guidelines around how to collect those, due to the individual nature of that data. + +There are several types of counters which are all found in `usage_data.rb`: + +- **Ordinary Batch Counters:** Simple count of a given ActiveRecord_Relation +- **Distinct Batch Counters:** Distinct count of a given ActiveRecord_Relation on given column +- **Sum Batch Counters:** Sum the values of a given ActiveRecord_Relation on given column +- **Alternative Counters:** Used for settings and configurations +- **Redis Counters:** Used for in-memory counts. + +NOTE: **Note:** +Only use the provided counter methods. Each counter method contains a built in fail safe to isolate each counter to avoid breaking the entire Usage Ping. + +### Why batch counting + +For large tables, PostgreSQL can take a long time to count rows due to MVCC [(Multi-version Concurrency Control)](https://en.wikipedia.org/wiki/Multiversion_concurrency_control). Batch counting is a counting method where a single large query is broken into multiple smaller queries. For example, instead of a single query querying 1,000,000 records, with batch counting, you can execute 100 queries of 10,000 records each. Batch counting is useful for avoiding database timeouts as each batch query is significantly shorter than one single long running query. + +For GitLab.com, there are extremely large tables with 15 second query timeouts, so we use batch counting to avoid encountering timeouts. Here are the sizes of some GitLab.com tables: + +| Table | Row counts in millions | +|------------------------------|------------------------| +| `merge_request_diff_commits` | 2280 | +| `ci_build_trace_sections` | 1764 | +| `merge_request_diff_files` | 1082 | +| `events` | 514 | + +There are two batch counting methods provided, `Ordinary Batch Counters` and `Distinct Batch Counters`. Batch counting requires indexes on columns to calculate max, min, and range queries. In some cases, a specialized index may need to be added on the columns involved in a counter. + +### Ordinary Batch Counters + +Handles `ActiveRecord::StatementInvalid` error + +Simple count of a given ActiveRecord_Relation, does a non-distinct batch count, smartly reduces batch_size and handles errors. + +Method: `count(relation, column = nil, batch: true, start: nil, finish: nil)` + +Arguments: + +- `relation` the ActiveRecord_Relation to perform the count +- `column` the column to perform the count on, by default is the primary key +- `batch`: default `true` in order to use batch counting +- `start`: custom start of the batch counting in order to avoid complex min calculations +- `end`: custom end of the batch counting in order to avoid complex min calculations + +Examples: + +```ruby +count(User.active) +count(::Clusters::Cluster.aws_installed.enabled, :cluster_id) +count(::Clusters::Cluster.aws_installed.enabled, :cluster_id, start: ::Clusters::Cluster.minimum(:id), finish: ::Clusters::Cluster.maximum(:id)) +``` + +### Distinct Batch Counters + +Handles `ActiveRecord::StatementInvalid` error + +Distinct count of a given ActiveRecord_Relation on given column, a distinct batch count, smartly reduces batch_size and handles errors. + +Method: `distinct_count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil)` + +Arguments: + +- `relation` the ActiveRecord_Relation to perform the count +- `column` the column to perform the distinct count, by default is the primary key +- `batch`: default `true` in order to use batch counting +- `batch_size`: if none set it will use default value 10000 from `Gitlab::Database::BatchCounter` +- `start`: custom start of the batch counting in order to avoid complex min calculations +- `end`: custom end of the batch counting in order to avoid complex min calculations + +Examples: + +```ruby +distinct_count(::Project, :creator_id) +distinct_count(::Note.with_suggestions.where(time_period), :author_id, start: ::User.minimum(:id), finish: ::User.maximum(:id)) +distinct_count(::Clusters::Applications::CertManager.where(time_period).available.joins(:cluster), 'clusters.user_id') +``` + +### Sum Batch Counters + +Handles `ActiveRecord::StatementInvalid` error + +Sum the values of a given ActiveRecord_Relation on given column and handles errors. + +Method: `sum(relation, column, batch_size: nil, start: nil, finish: nil)` + +Arguments: + +- `relation` the ActiveRecord_Relation to perform the operation +- `column` the column to sum on +- `batch_size`: if none set it will use default value 1000 from `Gitlab::Database::BatchCounter` +- `start`: custom start of the batch counting in order to avoid complex min calculations +- `end`: custom end of the batch counting in order to avoid complex min calculations + +Examples: + +```ruby +sum(JiraImportState.finished, :imported_issues_count) +``` + +### Grouping & Batch Operations + +The `count`, `distinct_count`, and `sum` batch counters can accept an `ActiveRecord::Relation` +object, which groups by a specified column. With a grouped relation, the methods do batch counting, +handle errors, and returns a hash table of key-value pairs. + +Examples: + +```ruby +count(Namespace.group(:type)) +# returns => {nil=>179, "Group"=>54} + +distinct_count(Project.group(:visibility_level), :creator_id) +# returns => {0=>1, 10=>1, 20=>11} + +sum(Issue.group(:state_id), :weight)) +# returns => {1=>3542, 2=>6820} +``` + +### Redis Counters + +Handles `::Redis::CommandError` and `Gitlab::UsageDataCounters::BaseCounter::UnknownEvent` +returns -1 when a block is sent or hash with all values -1 when a `counter(Gitlab::UsageDataCounters)` is sent +different behavior due to 2 different implementations of Redis counter + +Method: `redis_usage_data(counter, &block)` + +Arguments: + +- `counter`: a counter from `Gitlab::UsageDataCounters`, that has `fallback_totals` method implemented +- or a `block`: which is evaluated + +#### Ordinary Redis Counters + +Examples of implementation: + +- Using Redis methods [`INCR`](https://redis.io/commands/incr), [`GET`](https://redis.io/commands/get), and [`Gitlab::UsageDataCounters::WikiPageCounter`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/wiki_page_counter.rb) +- Using Redis methods [`HINCRBY`](https://redis.io/commands/hincrby), [`HGETALL`](https://redis.io/commands/hgetall), and [`Gitlab::UsageCounters::PodLogs`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_counters/pod_logs.rb) + +#### Redis HLL Counters + +With `Gitlab::UsageDataCounters::HLLRedisCounter` we have available data structures used to count unique values. + +Implemented using Redis methods [PFADD](https://redis.io/commands/pfadd) and [PFCOUNT](https://redis.io/commands/pfcount). + +##### Adding new events + +1. Define events in [`known_events.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/known_events.yml). + + Example event: + + ```yaml + - name: i_compliance_credential_inventory + category: compliance + redis_slot: compliance + expiry: 42 # 6 weeks + aggregation: weekly + ``` + + Keys: + + - `name`: unique event name. + + Name format `<prefix>_<redis_slot>_name`. + + Use one of the following prefixes for the event's name: + + - `g_` for group, as an event which is tracked for group. + - `p_` for project, as an event which is tracked for project. + - `i_` for instance, as an event which is tracked for instance. + - `a_` for events encompassing all `g_`, `p_`, `i_`. + - `o_` for other. + + Consider including in the event's name the Redis slot in order to be able to count totals for a specific category. + + Example names: `i_compliance_credential_inventory`, `g_analytics_contribution`. + + - `category`: event category. Used for getting total counts for events in a category, for easier + access to a group of events. + - `redis_slot`: optional Redis slot; default value: event name. Used if needed to calculate totals + for a group of metrics. Ensure keys are in the same slot. For example: + `i_compliance_credential_inventory` with `redis_slot: 'compliance'` will build Redis key + `i_{compliance}_credential_inventory-2020-34`. If `redis_slot` is not defined the Redis key will + be `{i_compliance_credential_inventory}-2020-34`. + - `expiry`: expiry time in days. Default: 29 days for daily aggregation and 6 weeks for weekly + aggregation. + - `aggregation`: aggregation `:daily` or `:weekly`. The argument defines how we build the Redis + keys for data storage. For `daily` we keep a key for metric per day of the year, for `weekly` we + keep a key for metric per week of the year. + +1. Track event in controller using `RedisTracking` module with `track_redis_hll_event(*controller_actions, name:, feature:, feature_default_enabled: false)`. + + Arguments: + + - `controller_actions`: controller actions we want to track. + - `name`: event name. + - `feature`: feature name, all metrics we track should be under feature flag. + - `feature_default_enabled`: feature flag is disabled by default, set to `true` for it to be enabled by default. + + Example usage: + + ```ruby + # controller + class ProjectsController < Projects::ApplicationController + include RedisTracking + + skip_before_action :authenticate_user!, only: :show + track_redis_hll_event :index, :show, name: 'g_compliance_example_feature_visitors', feature: :compliance_example_feature, feature_default_enabled: true + + def index + render html: 'index' + end + + def new + render html: 'new' + end + + def show + render html: 'show' + end + end + ``` + +1. Track event in API using `increment_unique_values(event_name, values)` helper method. + + In order to be able to track the event, Usage Ping must be enabled and the event feature `usage_data_<event_name>` must be enabled. + + Arguments: + + - `event_name`: event name. + - `values`: values counted, one value or array of values. + + Example usage: + + ```ruby + get ':id/registry/repositories' do + repositories = ContainerRepositoriesFinder.new( + user: current_user, subject: user_group + ).execute + + increment_unique_values('i_list_repositories', current_user.id) + + present paginate(repositories), with: Entities::ContainerRegistry::Repository, tags: params[:tags], tags_count: params[:tags_count] + end + ``` + +1. Track event using `track_usage_event(event_name, values) in services and graphql + + Increment unique values count using Redis HLL, for given event name. + + Example: + + [Track usage event for incident created in service](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/services/issues/update_service.rb) + + [Track usage event for incident created in graphql](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/graphql/mutations/alert_management/update_alert_status.rb) + + ```ruby + track_usage_event(:incident_management_incident_created, current_user.id) + ``` + +1. Track event using `UsageData` API + + Increment unique users count using Redis HLL, for given event name. + + Tracking events using the `UsageData` API requires the `usage_data_api` feature flag to be enabled, which is enabled by default. + + API requests are protected by checking for a valid CSRF token. + + In order to be able to increment the values the related feature `usage_data<event_name>` should be enabled. + + ```plaintext + POST /usage_data/increment_unique_users + ``` + + | Attribute | Type | Required | Description | + | :-------- | :--- | :------- | :---------- | + | `event` | string | yes | The event name it should be tracked | + + Response +w + Return 200 if tracking failed for any reason. + + - `200` if event was tracked or any errors + - `400 Bad request` if event parameter is missing + - `401 Unauthorized` if user is not authenticated + - `403 Forbidden` for invalid CSRF token provided + +1. Track events using JavaScript/Vue API helper which calls the API above + + Example usage for an existing event already defined in [known events](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/known_events.yml): + + Note that `usage_data_api` and `usage_data_#{event_name}` should be enabled in order to be able to track events + + ```javascript + import api from '~/api'; + + api.trackRedisHllUserEvent('my_already_defined_event_name'), + ``` + +1. Track event using base module `Gitlab::UsageDataCounters::HLLRedisCounter.track_event(entity_id, event_name)`. + + Arguments: + + - `entity_id`: value we count. For example: user_id, visitor_id. + - `event_name`: event name. + +1. Get event data using `Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names:, start_date:, end_date)`. + + Arguments: + + - `event_names`: the list of event names. + - `start_date`: start date of the period for which we want to get event data. + - `end_date`: end date of the period for which we want to get event data. + +Recommendations: + +- Key should expire in 29 days for daily and 42 days for weekly. +- If possible, data granularity should be a week. For example a key could be composed from the + metric's name and week of the year, `2020-33-{metric_name}`. +- Use a [feature flag](../../operations/feature_flags.md) to have a control over the impact when + adding new metrics. + +##### Known events in usage data payload + +All events added in [`known_events.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/known_events.yml) are automatically added to usage data generation under the `redis_hll_counters` key. This column is stored in [version-app as a JSON](https://gitlab.com/gitlab-services/version-gitlab-com/-/blob/master/db/schema.rb#L209). +For each event we add metrics for the weekly and monthly time frames, and totals for each where applicable: + +- `#{event_name}_weekly`: Data for 7 days for daily [aggregation](#adding-new-events) events and data for the last complete week for weekly [aggregation](#adding-new-events) events. +- `#{event_name}_monthly`: Data for 28 days for daily [aggregation](#adding-new-events) events and data for the last 4 complete weeks for weekly [aggregation](#adding-new-events) events. +- `#{category}_total_unique_counts_weekly`: Total unique counts for events in the same category for the last 7 days or the last complete week, if events are in the same Redis slot and we have more than one metric. +- `#{category}_total_unique_counts_monthly`: Total unique counts for events in same category for the last 28 days or the last 4 complete weeks, if events are in the same Redis slot and we have more than one metric. + +Example of `redis_hll_counters` data: + +```ruby +{:redis_hll_counters=> + {"compliance"=> + {"g_compliance_dashboard_weekly"=>0, + "g_compliance_dashboard_monthly"=>0, + "g_compliance_audit_events_weekly"=>0, + "g_compliance_audit_events_monthly"=>0, + "compliance_total_unique_counts_weekly"=>0, + "compliance_total_unique_counts_monthly"=>0}, + "analytics"=> + {"g_analytics_contribution_weekly"=>0, + "g_analytics_contribution_monthly"=>0, + "g_analytics_insights_weekly"=>0, + "g_analytics_insights_monthly"=>0, + "analytics_total_unique_counts_weekly"=>0, + "analytics_total_unique_counts_monthly"=>0}, + "ide_edit"=> + {"g_edit_by_web_ide_weekly"=>0, + "g_edit_by_web_ide_monthly"=>0, + "g_edit_by_sfe_weekly"=>0, + "g_edit_by_sfe_monthly"=>0, + "ide_edit_total_unique_counts_weekly"=>0, + "ide_edit_total_unique_counts_monthly"=>0}, + "search"=> + {"i_search_total_weekly"=>0, "i_search_total_monthly"=>0, "i_search_advanced_weekly"=>0, "i_search_advanced_monthly"=>0, "i_search_paid_weekly"=>0, "i_search_paid_monthly"=>0, "search_total_unique_counts_weekly"=>0, "search_total_unique_counts_monthly"=>0}, + "source_code"=>{"wiki_action_weekly"=>0, "wiki_action_monthly"=>0} + } +``` + +Example usage: + +```ruby +# Redis Counters +redis_usage_data(Gitlab::UsageDataCounters::WikiPageCounter) +redis_usage_data { ::Gitlab::UsageCounters::PodLogs.usage_totals[:total] } + +# Define events in known_events.yml https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/known_events.yml + +# Tracking events +Gitlab::UsageDataCounters::HLLRedisCounter.track_event(visitor_id, 'expand_vulnerabilities') + +# Get unique events for metric +redis_usage_data { Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: 'expand_vulnerabilities', start_date: 28.days.ago, end_date: Date.current) } +``` + +### Alternative Counters + +Handles `StandardError` and fallbacks into -1 this way not all measures fail if we encounter one exception. +Mainly used for settings and configurations. + +Method: `alt_usage_data(value = nil, fallback: -1, &block)` + +Arguments: + +- `value`: a simple static value in which case the value is simply returned. +- or a `block`: which is evaluated +- `fallback: -1`: the common value used for any metrics that are failing. + +Example of usage: + +```ruby +alt_usage_data { Gitlab::VERSION } +alt_usage_data { Gitlab::CurrentSettings.uuid } +alt_usage_data(999) +``` + +### Prometheus Queries + +In those cases where operational metrics should be part of Usage Ping, a database or Redis query is unlikely +to provide useful data. Instead, Prometheus might be more appropriate, since most of GitLab's architectural +components publish metrics to it that can be queried back, aggregated, and included as usage data. + +NOTE: **Note:** +Prometheus as a data source for Usage Ping is currently only available for single-node Omnibus installations +that are running the [bundled Prometheus](../../administration/monitoring/prometheus/index.md) instance. + +In order to query Prometheus for metrics, a helper method is available that will `yield` a fully configured +`PrometheusClient`, given it is available as per the note above: + +```ruby +with_prometheus_client do |client| + response = client.query('<your query>') + ... +end +``` + +Please refer to [the `PrometheusClient` definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/prometheus_client.rb) +for how to use its API to query for data. + +## Developing and testing Usage Ping + +### 1. Use your Rails console to manually test counters + +```ruby +# count +Gitlab::UsageData.count(User.active) +Gitlab::UsageData.count(::Clusters::Cluster.aws_installed.enabled, :cluster_id) + +# count distinct +Gitlab::UsageData.distinct_count(::Project, :creator_id) +Gitlab::UsageData.distinct_count(::Note.with_suggestions.where(time_period), :author_id, start: ::User.minimum(:id), finish: ::User.maximum(:id)) +``` + +### 2. Generate the SQL query + +Your Rails console will return the generated SQL queries. + +Example: + +```ruby +pry(main)> Gitlab::UsageData.count(User.active) + (2.6ms) SELECT "features"."key" FROM "features" + (15.3ms) SELECT MIN("users"."id") FROM "users" WHERE ("users"."state" IN ('active')) AND ("users"."user_type" IS NULL OR "users"."user_type" IN (6, 4)) + (2.4ms) SELECT MAX("users"."id") FROM "users" WHERE ("users"."state" IN ('active')) AND ("users"."user_type" IS NULL OR "users"."user_type" IN (6, 4)) + (1.9ms) SELECT COUNT("users"."id") FROM "users" WHERE ("users"."state" IN ('active')) AND ("users"."user_type" IS NULL OR "users"."user_type" IN (6, 4)) AND "users"."id" BETWEEN 1 AND 100000 +``` + +### 3. Optimize queries with #database-lab + +Paste the SQL query into `#database-lab` to see how the query performs at scale. + +- `#database-lab` is a Slack channel which uses a production-sized environment to test your queries. +- GitLab.com’s production database has a 15 second timeout. +- Any single query must stay below 1 second execution time with cold caches. +- Add a specialized index on columns involved to reduce the execution time. + +In order to have an understanding of the query's execution we add in the MR description the following information: + +- For counters that have a `time_period` test we add information for both cases: + - `time_period = {}` for all time periods + - `time_period = { created_at: 28.days.ago..Time.current }` for last 28 days period +- Execution plan and query time before and after optimization +- Query generated for the index and time +- Migration output for up and down execution + +We also use `#database-lab` and [explain.depesz.com](https://explain.depesz.com/). For more details, see the [database review guide](../database_review.md#preparation-when-adding-or-modifying-queries). + +#### Optimization recommendations and examples + +- Use specialized indexes [example 1](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/26871), [example 2](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/26445). +- Use defined `start` and `finish`, and simple queries, because these values can be memoized and reused, [example](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/37155). +- Avoid joins and write the queries as simply as possible, [example](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/36316). +- Set a custom `batch_size` for `distinct_count`, [example](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/38000). + +### 4. Add the metric definition + +When adding, changing, or updating metrics, please update the [Event Dictionary's **Usage Ping** table](event_dictionary.md). + +### 5. Add new metric to Versions Application + +Check if new metrics need to be added to the Versions Application. See `usage_data` [schema](https://gitlab.com/gitlab-services/version-gitlab-com/-/blob/master/db/schema.rb#L147) and usage data [parameters accepted](https://gitlab.com/gitlab-services/version-gitlab-com/-/blob/master/app/services/usage_ping.rb). Any metrics added under the `counts` key are saved in the `stats` column. + +### 6. Add the feature label + +Add the `feature` label to the Merge Request for new Usage Ping metrics. These are user-facing changes and are part of expanding the Usage Ping feature. + +### 7. Add a changelog file + +Ensure you comply with the [Changelog entries guide](../changelog.md). + +### 8. Ask for a Product Analytics Review + +On GitLab.com, we have DangerBot setup to monitor Product Analytics related files and DangerBot will recommend a Product Analytics review. Mention `@gitlab-org/growth/product_analytics/engineers` in your MR for a review. + +### 9. Verify your metric + +On GitLab.com, the Product Analytics team regularly monitors Usage Ping. They may alert you that your metrics need further optimization to run quicker and with greater success. You may also use the [Usage Ping QA dashboard](https://app.periscopedata.com/app/gitlab/632033/Usage-Ping-QA) to check how well your metric performs. The dashboard allows filtering by GitLab version, by "Self-managed" & "Saas" and shows you how many failures have occurred for each metric. Whenever you notice a high failure rate, you may re-optimize your metric. + +### Optional: Test Prometheus based Usage Ping + +If the data submitted includes metrics [queried from Prometheus](#prometheus-queries) that you would like to inspect and verify, +then you need to ensure that a Prometheus server is running locally, and that furthermore the respective GitLab components +are exporting metrics to it. If you do not need to test data coming from Prometheus, no further action +is necessary, since Usage Ping should degrade gracefully in the absence of a running Prometheus server. + +There are currently three kinds of components that may export data to Prometheus, and which are included in Usage Ping: + +- [`node_exporter`](https://github.com/prometheus/node_exporter) - Exports node metrics from the host machine +- [`gitlab-exporter`](https://gitlab.com/gitlab-org/gitlab-exporter) - Exports process metrics from various GitLab components +- various GitLab services such as Sidekiq and the Rails server that export their own metrics + +#### Test with an Omnibus container + +This is the recommended approach to test Prometheus based Usage Ping. + +The easiest way to verify your changes is to build a new Omnibus image from your code branch via CI, then download the image +and run a local container instance: + +1. From your merge request, click on the `qa` stage, then trigger the `package-and-qa` job. This job will trigger an Omnibus +build in a [downstream pipeline of the `omnibus-gitlab-mirror` project](https://gitlab.com/gitlab-org/build/omnibus-gitlab-mirror/-/pipelines). +1. In the downstream pipeline, wait for the `gitlab-docker` job to finish. +1. Open the job logs and locate the full container name including the version. It will take the following form: `registry.gitlab.com/gitlab-org/build/omnibus-gitlab-mirror/gitlab-ee:<VERSION>`. +1. On your local machine, make sure you are logged in to the GitLab Docker registry. You can find the instructions for this in +[Authenticate to the GitLab Container Registry](../../user/packages/container_registry/index.md#authenticate-with-the-container-registry). +1. Once logged in, download the new image via `docker pull registry.gitlab.com/gitlab-org/build/omnibus-gitlab-mirror/gitlab-ee:<VERSION>` +1. For more information about working with and running Omnibus GitLab containers in Docker, please refer to [GitLab Docker images](https://docs.gitlab.com/omnibus/docker/README.html) in the Omnibus documentation. + +#### Test with GitLab development toolkits + +This is the less recommended approach, since it comes with a number of difficulties when emulating a real GitLab deployment. + +The [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit) is not currently set up to run a Prometheus server or `node_exporter` alongside other GitLab components. If you would +like to do so, [Monitoring the GDK with Prometheus](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/master/doc/howto/prometheus/index.md#monitoring-the-gdk-with-prometheus) is a good start. + +The [GCK](https://gitlab.com/gitlab-org/gitlab-compose-kit) has limited support for testing Prometheus based Usage Ping. +By default, it already comes with a fully configured Prometheus service that is set up to scrape a number of components, +but with the following limitations: + +- It does not currently run a `gitlab-exporter` instance, so several `process_*` metrics from services such as Gitaly may be missing. +- While it runs a `node_exporter`, `docker-compose` services emulate hosts, meaning that it would normally report itself to not be associated +with any of the other services that are running. That is not how node metrics are reported in a production setup, where `node_exporter` +always runs as a process alongside other GitLab components on any given node. From Usage Ping's perspective none of the node data would therefore +appear to be associated to any of the services running, since they all appear to be running on different hosts. To alleviate this problem, the `node_exporter` in GCK was arbitrarily "assigned" to the `web` service, meaning only for this service `node_*` metrics will appear in Usage Ping. + +## Example Usage Ping payload + +The following is example content of the Usage Ping payload. + +```json +{ + "uuid": "0000000-0000-0000-0000-000000000000", + "hostname": "example.com", + "version": "12.10.0-pre", + "installation_type": "omnibus-gitlab", + "active_user_count": 999, + "recorded_at": "2020-04-17T07:43:54.162+00:00", + "edition": "EEU", + "license_md5": "00000000000000000000000000000000", + "license_id": null, + "historical_max_users": 999, + "licensee": { + "Name": "ABC, Inc.", + "Email": "email@example.com", + "Company": "ABC, Inc." + }, + "license_user_count": 999, + "license_starts_at": "2020-01-01", + "license_expires_at": "2021-01-01", + "license_plan": "ultimate", + "license_add_ons": { + }, + "license_trial": false, + "counts": { + "assignee_lists": 999, + "boards": 999, + "ci_builds": 999, + ... + }, + "container_registry_enabled": true, + "dependency_proxy_enabled": false, + "gitlab_shared_runners_enabled": true, + "gravatar_enabled": true, + "influxdb_metrics_enabled": true, + "ldap_enabled": false, + "mattermost_enabled": false, + "omniauth_enabled": true, + "prometheus_enabled": false, + "prometheus_metrics_enabled": false, + "reply_by_email_enabled": "incoming+%{key}@incoming.gitlab.com", + "signup_enabled": true, + "web_ide_clientside_preview_enabled": true, + "ingress_modsecurity_enabled": true, + "projects_with_expiration_policy_disabled": 999, + "projects_with_expiration_policy_enabled": 999, + ... + "elasticsearch_enabled": true, + "license_trial_ends_on": null, + "geo_enabled": false, + "git": { + "version": { + "major": 2, + "minor": 26, + "patch": 1 + } + }, + "gitaly": { + "version": "12.10.0-rc1-93-g40980d40", + "servers": 56, + "clusters": 14, + "filesystems": [ + "EXT_2_3_4" + ] + }, + "gitlab_pages": { + "enabled": true, + "version": "1.17.0" + }, + "container_registry_server": { + "vendor": "gitlab", + "version": "2.9.1-gitlab" + }, + "database": { + "adapter": "postgresql", + "version": "9.6.15", + "pg_system_id": 6842684531675334351 + }, + "avg_cycle_analytics": { + "issue": { + "average": 999, + "sd": 999, + "missing": 999 + }, + "plan": { + "average": null, + "sd": 999, + "missing": 999 + }, + "code": { + "average": null, + "sd": 999, + "missing": 999 + }, + "test": { + "average": null, + "sd": 999, + "missing": 999 + }, + "review": { + "average": null, + "sd": 999, + "missing": 999 + }, + "staging": { + "average": null, + "sd": 999, + "missing": 999 + }, + "production": { + "average": null, + "sd": 999, + "missing": 999 + }, + "total": 999 + }, + "analytics_unique_visits": { + "g_analytics_contribution": 999, + ... + }, + "usage_activity_by_stage": { + "configure": { + "project_clusters_enabled": 999, + ... + }, + "create": { + "merge_requests": 999, + ... + }, + "manage": { + "events": 999, + ... + }, + "monitor": { + "clusters": 999, + ... + }, + "package": { + "projects_with_packages": 999 + }, + "plan": { + "issues": 999, + ... + }, + "release": { + "deployments": 999, + ... + }, + "secure": { + "user_container_scanning_jobs": 999, + ... + }, + "verify": { + "ci_builds": 999, + ... + } + }, + "usage_activity_by_stage_monthly": { + "configure": { + "project_clusters_enabled": 999, + ... + }, + "create": { + "merge_requests": 999, + ... + }, + "manage": { + "events": 999, + ... + }, + "monitor": { + "clusters": 999, + ... + }, + "package": { + "projects_with_packages": 999 + }, + "plan": { + "issues": 999, + ... + }, + "release": { + "deployments": 999, + ... + }, + "secure": { + "user_container_scanning_jobs": 999, + ... + }, + "verify": { + "ci_builds": 999, + ... + } + }, + "topology": { + "duration_s": 0.013836685999194742, + "application_requests_per_hour": 4224, + "query_apdex_weekly_average": 0.996, + "failures": [], + "nodes": [ + { + "node_memory_total_bytes": 33269903360, + "node_memory_utilization": 0.35, + "node_cpus": 16, + "node_cpu_utilization": 0.2, + "node_uname_info": { + "machine": "x86_64", + "sysname": "Linux", + "release": "4.19.76-linuxkit" + }, + "node_services": [ + { + "name": "web", + "process_count": 16, + "process_memory_pss": 233349888, + "process_memory_rss": 788220927, + "process_memory_uss": 195295487, + "server": "puma" + }, + { + "name": "sidekiq", + "process_count": 1, + "process_memory_pss": 734080000, + "process_memory_rss": 750051328, + "process_memory_uss": 731533312 + }, + ... + ], + ... + }, + ... + ] + } +} +``` + +## Notable changes + +In GitLab 13.5, `pg_system_id` was added to send the [PostgreSQL system identifier](https://www.2ndquadrant.com/en/blog/support-for-postgresqls-system-identifier-in-barman/). + +## Exporting Usage Ping SQL queries and definitions + +Two Rake tasks exist to export Usage Ping definitions. + +- The Rake tasks export the raw SQL queries for `count`, `distinct_count`, `sum`. +- The Rake tasks export the Redis counter class or the line of the Redis block for `redis_usage_data`. +- The Rake tasks calculate the `alt_usage_data` metrics. + +In the home directory of your local GitLab installation run the following Rake tasks for the YAML and JSON versions respectively: + +```shell +# for YAML export +bin/rake gitlab:usage_data:dump_sql_in_yaml + +# for JSON export +bin/rake gitlab:usage_data:dump_sql_in_json + +# You may pipe the output into a file +bin/rake gitlab:usage_data:dump_sql_in_yaml > ~/Desktop/usage-metrics-2020-09-02.yaml +``` diff --git a/doc/development/profiling.md b/doc/development/profiling.md index f7ead94d725..f5a4d1edb92 100644 --- a/doc/development/profiling.md +++ b/doc/development/profiling.md @@ -91,9 +91,6 @@ that builds on this to add some additional niceties, such as allowing configuration with a single YAML file for multiple URLs, and uploading of the profile and log output to S3. -For GitLab.com, you can find the latest results here (restricted to GitLab Team members only): -`https://redash.gitlab.com/dashboard/gitlab-profiler-statistics` - ## Sherlock Sherlock is a custom profiling tool built into GitLab. Sherlock is _only_ diff --git a/doc/development/prometheus.md b/doc/development/prometheus.md index 902d4e6a1d0..3e7acaf6d94 100644 --- a/doc/development/prometheus.md +++ b/doc/development/prometheus.md @@ -1,6 +1,6 @@ --- stage: Monitor -group: APM +group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- @@ -47,3 +47,13 @@ using a Prometheus managed application in Kubernetes: ``` 1. Open `localhost:9090` in your browser to display the Prometheus user interface. + +## Script access to Prometheus + +You can script the access to Prometheus, extracting the name of the pod automatically like this: + +```shell +POD_INFORMATION=$(kubectl get pods -n gitlab-managed-apps | grep 'prometheus-prometheus-server') +POD_NAME=$(echo $POD_INFORMATION | awk '{print $1;}') +kubectl port-forward $POD_NAME 9090:9090 -n gitlab-managed-apps +``` diff --git a/doc/development/prometheus_metrics.md b/doc/development/prometheus_metrics.md index a39d19d8750..8fcc025b35b 100644 --- a/doc/development/prometheus_metrics.md +++ b/doc/development/prometheus_metrics.md @@ -1,6 +1,6 @@ --- stage: Monitor -group: APM +group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- @@ -35,11 +35,6 @@ After you add or change an existing common metric, you must [re-run the import s Or, you can create a database migration: -NOTE: **Note:** -If a query metric (which is identified by `id:`) is removed it will not be removed from database by default. -You might want to add additional database migration that makes a decision what to do with removed one. -For example: you might be interested in migrating all dependent data to a different metric. - ```ruby class ImportCommonMetrics < ActiveRecord::Migration[4.2] include Gitlab::Database::MigrationHelpers @@ -56,6 +51,10 @@ class ImportCommonMetrics < ActiveRecord::Migration[4.2] end ``` +If a query metric (which is identified by `id:`) is removed it will not be removed from database by default. +You might want to add additional database migration that makes a decision what to do with removed one. +For example: you might be interested in migrating all dependent data to a different metric. + ## GitLab Prometheus metrics GitLab provides [Prometheus metrics](../administration/monitoring/prometheus/gitlab_metrics.md) diff --git a/doc/development/reactive_caching.md b/doc/development/reactive_caching.md index f3386305e93..cf125c46565 100644 --- a/doc/development/reactive_caching.md +++ b/doc/development/reactive_caching.md @@ -85,9 +85,7 @@ The ReactiveCaching concern can be used in models as well as `project_services` 1. Implement the `calculate_reactive_cache` method in your model/service. 1. Call `with_reactive_cache` in your model/service where the cached value is needed. -1. If the `calculate_reactive_cache` method above submits requests to external services -(e.g. Prometheus, K8s), make sure to change the -[`reactive_cache_work_type` accordingly](#selfreactive_cache_work_type). +1. Set the [`reactive_cache_work_type` accordingly](#selfreactive_cache_work_type). ### In controllers @@ -252,7 +250,7 @@ self.reactive_cache_hard_limit = 5.megabytes - This is the type of work performed by the `calculate_reactive_cache` method. Based on this attribute, it's able to pick the right worker to process the caching job. Make sure to set it as `:external_dependency` if the work performs any external request -(e.g. Kubernetes, Sentry). +(e.g. Kubernetes, Sentry); otherwise set it to `:no_dependency`. #### `self.reactive_cache_worker_finder` diff --git a/doc/development/redis.md b/doc/development/redis.md index d205082b9c6..a0ae84beb8d 100644 --- a/doc/development/redis.md +++ b/doc/development/redis.md @@ -68,6 +68,10 @@ which is enabled for the `cache` and `shared_state` ## Redis in structured logging +For GitLab Team Members: There are [basic](https://www.youtube.com/watch?v=Uhdj19Dc6vU) and +[advanced](https://youtu.be/jw1Wv2IJxzs) videos that show how you can work with the Redis +structured logging fields on GitLab.com. + Our [structured logging](logging.md#use-structured-json-logging) for web requests and Sidekiq jobs contains fields for the duration, call count, bytes written, and bytes read per Redis instance, along with a total for @@ -96,10 +100,14 @@ requests that read the most data from the cache, we can just sort by ### The slow log +TIP: **Tip:** +There is a [video showing how to see the slow log](https://youtu.be/BBI68QuYRH8) (GitLab internal) +on GitLab.com + On GitLab.com, entries from the [Redis slow log](https://redis.io/commands/slowlog) are available in the `pubsub-redis-inf-gprd*` index with the [`redis.slowlog` -tag](https://log.gprd.gitlab.net/app/kibana#/discover?_g=(filters:!(),refreshInterval:(pause:!t,value:0),time:(from:now-1d,to:now))&_a=(columns:!(json.type,json.command,json.exec_time),filters:!(('$state':(store:appState),meta:(alias:!n,disabled:!f,index:AWSQX_Vf93rHTYrsexmk,key:json.tag,negate:!f,params:(query:redis.slowlog),type:phrase),query:(match:(json.tag:(query:redis.slowlog,type:phrase))))),index:AWSQX_Vf93rHTYrsexmk)). +tag](https://log.gprd.gitlab.net/app/kibana#/discover?_g=(filters:!(),refreshInterval:(pause:!t,value:0),time:(from:now-1d,to:now))&_a=(columns:!(json.type,json.command,json.exec_time_s),filters:!(('$state':(store:appState),meta:(alias:!n,disabled:!f,index:AWSQX_Vf93rHTYrsexmk,key:json.tag,negate:!f,params:(query:redis.slowlog),type:phrase),query:(match:(json.tag:(query:redis.slowlog,type:phrase))))),index:AWSQX_Vf93rHTYrsexmk)). This shows commands that have taken a long time and may be a performance concern. @@ -113,7 +121,7 @@ passing to fluentd (and ultimately Elasticsearch). The [Redis Keyspace Analyzer](https://gitlab.com/gitlab-com/gl-infra/redis-keyspace-analyzer) project contains tools for dumping the full key list and memory usage of a Redis -instance, and then analyzing those lists while elimating potentially sensitive +instance, and then analyzing those lists while eliminating potentially sensitive data from the results. It can be used to find the most frequent key patterns, or those that use the most memory. diff --git a/doc/development/reference_processing.md b/doc/development/reference_processing.md index 527fb94f228..cf587043cae 100644 --- a/doc/development/reference_processing.md +++ b/doc/development/reference_processing.md @@ -77,6 +77,22 @@ a minimum implementation of `AbstractReferenceFilter` should define: and an identifier, find the object. For example, this in a reference filter for merge requests, this might be `project.merge_requests.where(iid: iid)`. +### Add a new reference prefix and filter + +For reference filters for new objects, use a prefix format following the pattern +`^<object_type>#`, because: + +1. Varied single-character prefixes are hard for users to track. Especially for + lower-use object types, this can diminish value for the feature. +1. Suitable single-character prefixes are limited. +1. Following a consistent pattern allows users to infer the existence of new features. + +To add a reference prefix for a new object `apple`,which has both a name and ID, +format the reference as: + +- `^apple#123` for identification by ID. +- `^apple#"Granny Smith"` for identification by name. + ### Performance #### Find object optimization diff --git a/doc/development/secure_coding_guidelines.md b/doc/development/secure_coding_guidelines.md index 1961d1dcc34..e35bda82aaa 100644 --- a/doc/development/secure_coding_guidelines.md +++ b/doc/development/secure_coding_guidelines.md @@ -1,3 +1,10 @@ +--- +type: reference, dev +stage: none +group: Development +info: "See the Technical Writers assigned to Development Guidelines: https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments-to-development-guidelines" +--- + # Secure Coding Guidelines This document contains descriptions and guidelines for addressing security @@ -82,7 +89,8 @@ This Ruby Regex specialty can have security impact, as often regular expressions #### Examples -GitLab specific examples can be found [here](https://gitlab.com/gitlab-org/gitlab/-/issues/36029#note_251262187) and [there](https://gitlab.com/gitlab-org/gitlab/-/issues/33569). +GitLab-specific examples can be found in the following [path traversal](https://gitlab.com/gitlab-org/gitlab/-/issues/36029#note_251262187) +and [open redirect](https://gitlab.com/gitlab-org/gitlab/-/issues/33569) issues. Another example would be this fictional Ruby on Rails controller: @@ -392,5 +400,34 @@ In order to prevent Path Traversal vulnerabilities, user-controlled filenames or #### GitLab specific validations -- [`Gitlab::Utils.check_path_traversal`](https://gitlab.com/gitlab-org/security/gitlab/-/blob/master/lib/gitlab/utils.rb#L12-24) can be used to validate user input against Path Traversal vulnerabilities. Remember to add further validation when setting the `allowed_absolute` option to `true`. -- [`file_path` API validator](https://gitlab.com/gitlab-org/security/gitlab/-/blob/master/lib/api/validations/validators/file_path.rb) to validate user input when working with the Grape gem. +The methods `Gitlab::Utils.check_path_traversal!()` and `Gitlab::Utils.check_allowed_absolute_path!()` +can be used to validate user-supplied paths and prevent vulnerabilities. +`check_path_traversal!()` will detect their Path Traversal payloads and accepts URL-encoded paths. +`check_allowed_absolute_path!()` will check if a path is absolute and whether it is inside the allowed path list. By default, absolute +paths are not allowed, so you need to pass a list of allowed absolute paths to the `path_allowlist` +parameter when using `check_allowed_absolute_path!()`. + +To use a combination of both checks, follow the example below: + +```ruby +path = Gitlab::Utils.check_path_traversal!(path) +Gitlab::Utils.check_allowed_absolute_path!(path, path_allowlist) +``` + +In the REST API, we have the [`FilePath`](https://gitlab.com/gitlab-org/security/gitlab/-/blob/master/lib/api/validations/validators/file_path.rb) +validator that can be used to perform the checking on any file path argument the endpoints have. +It can be used as follows: + +```ruby +requires :file_path, type: String, file_path: { allowlist: ['/foo/bar/', '/home/foo/', '/app/home'] } +``` + +The Path Traversal check can also be used to forbid any absolute path: + +```ruby +requires :file_path, type: String, file_path: true +``` + +NOTE: **Note:** +Absolute paths are not allowed by default. If allowing an absolute path is required, you +need to provide an array of paths to the parameter `allowlist`. diff --git a/doc/development/shell_scripting_guide/index.md b/doc/development/shell_scripting_guide/index.md index 622c90d7a97..704b46c7efa 100644 --- a/doc/development/shell_scripting_guide/index.md +++ b/doc/development/shell_scripting_guide/index.md @@ -1,7 +1,5 @@ # Shell scripting standards and style guidelines -## Overview - GitLab consists of many various services and sub-projects. The majority of their backend code is written in [Ruby](https://www.ruby-lang.org) and [Go](https://golang.org). However, some of them use shell scripts for @@ -15,7 +13,7 @@ should be eventually harmonized with this guide. If there are any per-project deviations from this guide, they should be described in the `README.md` or `PROCESS.md` file for such a project. -### Avoid using shell scripts +## Avoid using shell scripts CAUTION: **Caution:** This is a must-read section. diff --git a/doc/development/sidekiq_style_guide.md b/doc/development/sidekiq_style_guide.md index fdea27f43ca..24570cfc07b 100644 --- a/doc/development/sidekiq_style_guide.md +++ b/doc/development/sidekiq_style_guide.md @@ -215,6 +215,85 @@ From the rails console: Feature.enable!(:disable_authorized_projects_deduplication) ``` +## Limited capacity worker + +It is possible to limit the number of concurrent running jobs for a worker class +by using the `LimitedCapacity::Worker` concern. + +The worker must implement three methods: + +- `perform_work` - the concern implements the usual `perform` method and calls +`perform_work` if there is any capacity available. +- `remaining_work_count` - number of jobs that will have work to perform. +- `max_running_jobs` - maximum number of jobs allowed to run concurrently. + +```ruby +class MyDummyWorker + include ApplicationWorker + include LimitedCapacity::Worker + + def perform_work(*args) + end + + def remaining_work_count(*args) + 5 + end + + def max_running_jobs + 25 + end +end +``` + +Additional to the regular worker, a cron worker must be defined as well to +backfill the queue with jobs. the arguments passed to `perform_with_capacity` +will be passed along to the `perform_work` method. + +```ruby +class ScheduleMyDummyCronWorker + include ApplicationWorker + include CronjobQueue + + def perform(*args) + MyDummyWorker.perform_with_capacity(*args) + end +end +``` + +### How many jobs are running? + +It will be running `max_running_jobs` at almost all times. + +The cron worker will check the remaining capacity on each execution and it +will schedule at most `max_running_jobs` jobs. Those jobs on completion will +re-enqueue themselves immediately, but not on failure. The cron worker is in +charge of replacing those failed jobs. + +### Handling errors and idempotence + +This concern disables Sidekiq retries, logs the errors, and sends the job to the +dead queue. This is done to have only one source that produces jobs and because +the retry would occupy a slot with a job that will be performed in the distant future. + +We let the cron worker enqueue new jobs, this could be seen as our retry and +back off mechanism because the job might fail again if executed immediately. +This means that for every failed job, we will be running at a lower capacity +until the cron worker fills the capacity again. If it is important for the +worker not to get a backlog, exceptions must be handled in `#perform_work` and +the job should not raise. + +The jobs are deduplicated using the `:none` strategy, but the worker is not +marked as `idempotent!`. + +### Metrics + +This concern exposes three Prometheus metrics of gauge type with the worker class +name as label: + +- `limited_capacity_worker_running_jobs` +- `limited_capacity_worker_max_running_jobs` +- `limited_capacity_worker_remaining_work_count` + ## Job urgency Jobs can have an `urgency` attribute set, which can be `:high`, @@ -617,7 +696,7 @@ during deployment before all Rails and Sidekiq nodes have the updated code. #### Deprecate and remove an argument -**Before you remove arguments from the `perform_async` and `perform` methods.**, deprecate them. The +**Before you remove arguments from the `perform_async` and `perform` methods.**, deprecate them. The following example deprecates and then removes `arg2` from the `perform_async` method: 1. Provide a default value (usually `nil`) and use a comment to mark the diff --git a/doc/development/sql.md b/doc/development/sql.md index 3b969c7d27a..55a8192578b 100644 --- a/doc/development/sql.md +++ b/doc/development/sql.md @@ -220,7 +220,7 @@ Project.select(:id, :user_id).joins(:merge_requests) Never use ActiveRecord's `pluck` to pluck a set of values into memory only to use them as an argument for another query. For example, this will execute an -extra unecessary database query and load a lot of unecessary data into memory: +extra unnecessary database query and load a lot of unnecessary data into memory: ```ruby projects = Project.all.pluck(:id) diff --git a/doc/development/swapping_tables.md b/doc/development/swapping_tables.md index 5c62900dbff..f633721ea9d 100644 --- a/doc/development/swapping_tables.md +++ b/doc/development/swapping_tables.md @@ -1,3 +1,9 @@ +--- +stage: Enablement +group: Database +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Swapping Tables Sometimes you need to replace one table with another. For example, when diff --git a/doc/development/telemetry/event_dictionary.md b/doc/development/telemetry/event_dictionary.md index d8cc32ea8d0..53b7ddea095 100644 --- a/doc/development/telemetry/event_dictionary.md +++ b/doc/development/telemetry/event_dictionary.md @@ -1,32 +1,5 @@ --- -stage: Growth -group: Telemetry -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +redirect_to: '../product_analytics/event_dictionary.md' --- -# Event Dictionary - -**Note: We've temporarily moved the Event Dictionary to a [Google Sheet](https://docs.google.com/spreadsheets/d/1VzE8R72Px_Y_LlE3Z05LxUlG_dumWe3vl-HeUo70TPw/edit?usp=sharing)**. The previous Markdown table exceeded 600 rows making it difficult to manage. In the future, our intention is to move this back into our docs using a [YAML file](https://gitlab.com/gitlab-org/gitlab-docs/-/issues/823). - -The event dictionary is a single source of truth for the metrics and events we collect for product usage data. The Event Dictionary lists all the metrics and events we track, why we're tracking them, and where they are tracked. - -This is a living document that is updated any time a new event is planned or implemented. It includes the following information. - -- Section, stage, or group -- Description -- Implementation status -- Availability by plan type -- Code path - -We're currently focusing our Event Dictionary on [Usage Ping](usage_ping.md). In the future, we will also include [Snowplow](snowplow.md). We currently have an initiative across the entire product organization to complete the [Event Dictionary for Usage Ping](https://gitlab.com/groups/gitlab-org/-/epics/4174). - -## Instructions - -1. Open the Event Dictionary and fill in all the **PM to edit** columns highlighted in yellow. -1. Check that all the metrics and events are assigned to the correct section, stage, or group. If a metric is used across many groups, assign it to the stage. If a metric is used across many stages, assign it to the section. If a metric is incorrectly assigned to another section, stage, or group, let the PM know you have reassigned it. If your group has no assigned metrics and events, check that your metrics and events are not incorrectly assigned to another PM. -1. Add descriptions of what your metrics and events are tracking. Work with your Engineering team or the Telemetry team if you need help understanding this. -1. Add what plans this metric is available on. Work with your Engineering team or the Telemetry team if you need help understanding this. - -## Planned metrics and events - -For future metrics and events you plan to track, please add them to the Event Dictionary and note the status as `Planned`, `In Progress`, or `Implemented`. Once you have confirmed the metric has been implemented and have confirmed the metric data is in our data warehouse, change the status to **Data Available**. +This document was moved to [another location](../product_analytics/event_dictionary.md). diff --git a/doc/development/telemetry/index.md b/doc/development/telemetry/index.md index b5032ce3730..79ea80a52ea 100644 --- a/doc/development/telemetry/index.md +++ b/doc/development/telemetry/index.md @@ -1,182 +1,5 @@ --- -stage: Growth -group: Telemetry -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +redirect_to: '../product_analytics/index.md' --- -# Telemetry Guide - -At GitLab, we collect product usage data for the purpose of helping us build a better product. Data helps GitLab understand which parts of the product need improvement and which features we should build next. Product usage data also helps our team better understand the reasons why people use GitLab. With this knowledge we are able to make better product decisions. - -We encourage users to enable tracking, and we embrace full transparency with our tracking approach so it can be easily understood and trusted. - -By enabling tracking, users can: - -- Contribute back to the wider community. -- Help GitLab improve on the product. - -## Our tracking tools - -We use three methods to gather product usage data: - -- [Snowplow](#snowplow) -- [Usage Ping](#usage-ping) -- [Database import](#database-import) - -### Snowplow - -Snowplow is an enterprise-grade marketing and product analytics platform which helps track the way -users engage with our website and application. - -Snowplow consists of two components: - -- [Snowplow JS](https://github.com/snowplow/snowplow/wiki/javascript-tracker) tracks client-side - events. -- [Snowplow Ruby](https://github.com/snowplow/snowplow/wiki/ruby-tracker) tracks server-side events. - -For more details, read the [Snowplow](snowplow.md) guide. - -### Usage Ping - -Usage Ping is a method for GitLab Inc to collect usage data on a GitLab instance. Usage Ping is primarily composed of row counts for different tables in the instance’s database. By comparing these counts month over month (or week over week), we can get a rough sense for how an instance is using the different features within the product. This high-level data is used to help our product, support, and sales teams. - -For more details, read the [Usage Ping](usage_ping.md) guide. - -### Database import - -Database imports are full imports of data into GitLab's data warehouse. For GitLab.com, the PostgreSQL database is loaded into Snowflake data warehouse every 6 hours. For more details, see the [data team handbook](https://about.gitlab.com/handbook/business-ops/data-team/platform/#extract-and-load). - -## What data can be tracked - -Our different tracking tools allows us to track different types of events. The event types and examples of what data can be tracked are outlined below. - -The availability of event types and their tracking tools varies by segment. For example, on Self-Managed Users, we only have reporting using Database records via Usage Ping. - -| Event Types | SaaS Instance | SaaS Plan | SaaS Group | SaaS Session | SaaS User | SM Instance | SM Plan | SM Group | SM Session | SM User | -|----------------------------------------|---------------|-----------|------------|--------------|-----------|-------------|---------|----------|------------|---------| -| Snowplow (JS Pageview events) | ✅ | 📅 | 📅 | ✅ | 📅 | 📅 | 📅 | 📅 | 📅 | 📅 | -| Snowplow (JS UI events) | ✅ | 📅 | 📅 | ✅ | 📅 | 📅 | 📅 | 📅 | 📅 | 📅 | -| Snowplow (Ruby Pageview events) | ✅ | 📅 | 📅 | ✅ | 📅 | 📅 | 📅 | 📅 | 📅 | 📅 | -| Snowplow (Ruby CRUD / API events) | ✅ | 📅 | 📅 | ✅ | 📅 | 📅 | 📅 | 📅 | 📅 | 📅 | -| Usage Ping (Redis UI counters) | 🔄 | 🔄 | 🔄 | ✖️ | 🔄 | 🔄 | 🔄 | 🔄 | ✖️ | 🔄 | -| Usage Ping (Redis Pageview counters) | 🔄 | 🔄 | 🔄 | ✖️ | 🔄 | 🔄 | 🔄 | 🔄 | ✖️ | 🔄 | -| Usage Ping (Redis CRUD / API counters) | 🔄 | 🔄 | 🔄 | ✖️ | 🔄 | 🔄 | 🔄 | 🔄 | ✖️ | 🔄 | -| Usage Ping (Database counters) | ✅ | 🔄 | 📅 | ✖️ | ✅ | ✅ | ✅ | ✅ | ✖️ | ✅ | -| Usage Ping (Instance settings) | ✅ | 🔄 | 📅 | ✖️ | ✅ | ✅ | ✅ | ✅ | ✖️ | ✅ | -| Usage Ping (Integration settings) | ✅ | 🔄 | 📅 | ✖️ | ✅ | ✅ | ✅ | ✅ | ✖️ | ✅ | -| Database import (Database records) | ✅ | ✅ | ✅ | ✖️ | ✅ | ✖️ | ✖️ | ✖️ | ✖️ | ✖️ | - -[Source file](https://docs.google.com/spreadsheets/d/1e8Afo41Ar8x3JxAXJF3nL83UxVZ3hPIyXdt243VnNuE/edit?usp=sharing) - -**Legend** - -✅ Available, 🔄 In Progress, 📅 Planned, ✖️ Not Possible - -SaaS = GitLab.com. SM = Self-Managed instance - -### Pageview events - -- Number of sessions that visited the /dashboard/groups page - -### UI events - -- Number of sessions that clicked on a button or link -- Number of sessions that closed a modal - -UI events are any interface-driven actions from the browser including click data. - -### CRUD or API events - -- Number of Git pushes -- Number of GraphQL queries -- Number of requests to a Rails action or controller - -These are backend events that include the creation, read, update, deletion of records, and other events that might be triggered from layers other than those available in the interface. - -### Database records - -These are raw database records which can be explored using business intelligence tools like Sisense. The full list of available tables can be found in [structure.sql](https://gitlab.com/gitlab-org/gitlab/-/blob/master/db/structure.sql). - -### Instance settings - -These are settings of your instance such as the instance's Git version and if certain features are enabled such as `container_registry_enabled`. - -### Integration settings - -These are integrations your GitLab instance interacts with such as an [external storage provider](../../administration/static_objects_external_storage.md) or an [external container registry](../../administration/packages/container_registry.md#use-an-external-container-registry-with-gitlab-as-an-auth-endpoint). These services must be able to send data back into a GitLab instance for data to be tracked. - -## Reporting level - -Our reporting levels of aggregate or individual reporting varies by segment. For example, on Self-Managed Users, we can report at an aggregate user level using Usage Ping but not on an Individual user level. - -| Aggregated Reporting | SaaS Instance | SaaS Plan | SaaS Group | SaaS Session | SaaS User | SM Instance | SM Plan | SM Group | SM Session | SM User | -|----------------------|---------------|-----------|------------|--------------|-----------|-------------|---------|----------|------------|---------| -| Snowplow | ✅ | 📅 | 📅 | ✅ | 📅 | ✅ | 📅 | 📅 | ✅ | 📅 | -| Usage Ping | ✅ | 🔄 | 📅 | 📅 | ✅ | ✅ | ✅ | ✅ | 📅 | ✅ | -| Database import | ✅ | ✅ | ✅ | ✖️ | ✅ | ✖️ | ✖️ | ✖️ | ✖️ | ✖️ | - -| Identifiable Reporting | SaaS Instance | SaaS Plan | SaaS Group | SaaS Session | SaaS User | SM Instance | SM Plan | SM Group | SM Session | SM User | -|------------------------|---------------|-----------|------------|--------------|-----------|-------------|---------|----------|------------|---------| -| Snowplow | ✅ | 📅 | 📅 | ✅ | 📅 | ✖️ | ✖️ | ✖️ | ✖️ | ✖️ | -| Usage Ping | ✅ | 🔄 | 📅 | ✖️ | ✖️ | ✅ | ✅ | ✖️ | ✖️ | ✖️ | -| Database import | ✅ | ✅ | ✅ | ✖️ | ✅ | ✖️ | ✖️ | ✖️ | ✖️ | ✖️ | - -**Legend** - -✅ Available, 🔄 In Progress, 📅 Planned, ✖️ Not Possible - -SaaS = GitLab.com. SM = Self-Managed instance - -## Reporting time period - -Our reporting time periods varies by segment. For example, on Self-Managed Users, we can report all time counts and 28 day counts in Usage Ping. - -| Reporting Time Period | All Time | 28 Days | 7 Days | Daily | -|-----------------------|----------|---------|--------|-------| -| Snowplow | ✅ | ✅ | ✅ | ✅ | -| Usage Ping | ✅ | ✅ | 📅 | ✖️ | -| Database import | ✅ | ✅ | ✅ | ✅ | - -**Legend** - -✅ Available, 🔄 In Progress, 📅 Planned, ✖️ Not Possible - -## Systems overview - -The systems overview is a simplified diagram showing the interactions between GitLab Inc and self-managed instances. - - - -[Source file](https://app.diagrams.net/#G13DVpN-XnhWGz9tqReIj8pp1UE4ehk_EC) - -### GitLab Inc - -For Telemetry purposes, GitLab Inc has three major components: - -1. [Data Infrastructure](https://about.gitlab.com/handbook/business-ops/data-team/platform/infrastructure/): This contains everything managed by our data team including Sisense Dashboards for visualization, Snowflake for Data Warehousing, incoming data sources such as PostgreSQL Pipeline and S3 Bucket, and lastly our data collectors [GitLab.com's Snowplow Collector](https://gitlab.com/gitlab-com/gl-infra/readiness/-/tree/master/library/snowplow/) and GitLab's Versions Application. -1. GitLab.com: This is the production GitLab application which is made up of a Client and Server. On the Client or browser side, a Snowplow JS Tracker (Frontend) is used to track client-side events. On the Server or application side, a Snowplow Ruby Tracker (Backend) is used to track server-side events. The server also contains Usage Ping which leverages a PostgreSQL database and a Redis in-memory data store to report on usage data. Lastly, the server also contains System Logs which are generated from running the GitLab application. -1. [Monitoring infrastructure](https://about.gitlab.com/handbook/engineering/monitoring/): This is the infrastructure used to ensure GitLab.com is operating smoothly. System Logs are sent from GitLab.com to our monitoring infrastructure and collected by a FluentD collector. From FluentD, logs are either sent to long term Google Cloud Services cold storage via Stackdriver, or, they are sent to our Elastic Cluster via Cloud Pub/Sub which can be explored in real-time using Kibana. - -### Self-managed - -For Telemetry purposes, self-managed instances have two major components: - -1. Data infrastructure: Having a data infrastructure setup is optional on self-managed instances. If you'd like to collect Snowplow tracking events for your self-managed instance, you can setup your own self-managed Snowplow collector and configure your Snowplow events to point to your own collector. -1. GitLab: A self-managed GitLab instance contains all of the same components as GitLab.com mentioned above. - -### Differences between GitLab Inc and Self-managed - -As shown by the orange lines, on GitLab.com Snowplow JS, Snowplow Ruby, Usage Ping, and PostgreSQL database imports all flow into GitLab Inc's data infrastructure. However, on self-managed, only Usage Ping flows into GitLab Inc's data infrastructure. - -As shown by the green lines, on GitLab.com system logs flow into GitLab Inc's monitoring infrastructure. On self-managed, there are no logs sent to GitLab Inc's monitoring infrastructure. - -Note (1): Snowplow JS and Snowplow Ruby are available on self-managed, however, the Snowplow Collector endpoint is set to a self-managed Snowplow Collector which GitLab Inc does not have access to. - -## Additional information - -More useful links: - -- [Telemetry Direction](https://about.gitlab.com/direction/telemetry/) -- [Data Analysis Process](https://about.gitlab.com/handbook/business-ops/data-team/#data-analysis-process/) -- [Data for Product Managers](https://about.gitlab.com/handbook/business-ops/data-team/programs/data-for-product-managers/) -- [Data Infrastructure](https://about.gitlab.com/handbook/business-ops/data-team/platform/infrastructure/) +This document was moved to [another location](../product_analytics/index.md). diff --git a/doc/development/telemetry/snowplow.md b/doc/development/telemetry/snowplow.md index f427d7d1488..fd75d29286b 100644 --- a/doc/development/telemetry/snowplow.md +++ b/doc/development/telemetry/snowplow.md @@ -1,410 +1,5 @@ --- -stage: Growth -group: Telemetry -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +redirect_to: '../product_analytics/snowplow.md' --- -# Snowplow Guide - -This guide provides an overview of how Snowplow works, and implementation details. - -For more information about Telemetry, see: - -- [Telemetry Guide](index.md) -- [Usage Ping Guide](usage_ping.md) - -More useful links: - -- [Telemetry Direction](https://about.gitlab.com/direction/telemetry/) -- [Data Analysis Process](https://about.gitlab.com/handbook/business-ops/data-team/#data-analysis-process/) -- [Data for Product Managers](https://about.gitlab.com/handbook/business-ops/data-team/programs/data-for-product-managers/) -- [Data Infrastructure](https://about.gitlab.com/handbook/business-ops/data-team/platform/infrastructure/) - -## What is Snowplow - -Snowplow is an enterprise-grade marketing and product analytics platform which helps track the way users engage with our website and application. - -[Snowplow](https://github.com/snowplow/snowplow) consists of the following loosely-coupled sub-systems: - -- **Trackers** fire Snowplow events. Snowplow has 12 trackers, covering web, mobile, desktop, server, and IoT. -- **Collectors** receive Snowplow events from trackers. We have three different event collectors, synchronizing events either to Amazon S3, Apache Kafka, or Amazon Kinesis. -- **Enrich** cleans up the raw Snowplow events, enriches them and puts them into storage. We have an Hadoop-based enrichment process, and a Kinesis-based or Kafka-based process. -- **Storage** is where the Snowplow events live. We store the Snowplow events in a flat file structure on S3, and in the Redshift and PostgreSQL databases. -- **Data modeling** is where event-level data is joined with other data sets and aggregated into smaller data sets, and business logic is applied. This produces a clean set of tables which make it easier to perform analysis on the data. We have data models for Redshift and Looker. -- **Analytics** are performed on the Snowplow events or on the aggregate tables. - - - -## Snowplow schema - -We have many definitions of Snowplow's schema. We have an active issue to [standardize this schema](https://gitlab.com/gitlab-org/gitlab/-/issues/207930) including the following definitions: - -- Frontend and backend taxonomy as listed below -- [Feature instrumentation taxonomy](https://about.gitlab.com/handbook/business-ops/data-team/programs/data-for-product-managers/#sts=Taxonomy) -- [Self describing events](https://github.com/snowplow/snowplow/wiki/Custom-events#self-describing-events) -- [Iglu schema](https://gitlab.com/gitlab-org/iglu/) -- [Snowplow authored events](https://github.com/snowplow/snowplow/wiki/Snowplow-authored-events) - -## Enabling Snowplow - -Tracking can be enabled at: - -- The instance level, which enables tracking on both the frontend and backend layers. -- User level, though user tracking can be disabled on a per-user basis. GitLab tracking respects the [Do Not Track](https://www.eff.org/issues/do-not-track) standard, so any user who has enabled the Do Not Track option in their browser is not tracked at a user level. - -We utilize Snowplow for the majority of our tracking strategy and it is enabled on GitLab.com. On a self-managed instance, Snowplow can be enabled by navigating to: - -- **Admin Area > Settings > Integrations** in the UI. -- `admin/application_settings/integrations` in your browser. - -The following configuration is required: - -| Name | Value | -|---------------|---------------------------| -| Collector | `snowplow.trx.gitlab.net` | -| Site ID | `gitlab` | -| Cookie domain | `.gitlab.com` | - -## Snowplow request flow - -The following example shows a basic request/response flow between the following components: - -- Snowplow JS / Ruby Trackers on GitLab.com -- [GitLab.com Snowplow Collector](https://gitlab.com/gitlab-com/gl-infra/readiness/-/blob/master/library/snowplow/index.md) -- GitLab's S3 Bucket -- GitLab's Snowflake Data Warehouse -- Sisense: - -```mermaid -sequenceDiagram - participant Snowplow JS (Frontend) - participant Snowplow Ruby (Backend) - participant GitLab.com Snowplow Collector - participant S3 Bucket - participant Snowflake DW - participant Sisense Dashboards - Snowplow JS (Frontend) ->> GitLab.com Snowplow Collector: FE Tracking event - Snowplow Ruby (Backend) ->> GitLab.com Snowplow Collector: BE Tracking event - loop Process using Kinesis Stream - GitLab.com Snowplow Collector ->> GitLab.com Snowplow Collector: Log raw events - GitLab.com Snowplow Collector ->> GitLab.com Snowplow Collector: Enrich events - GitLab.com Snowplow Collector ->> GitLab.com Snowplow Collector: Write to disk - end - GitLab.com Snowplow Collector ->> S3 Bucket: Kinesis Firehose - S3 Bucket->>Snowflake DW: Import data - Snowflake DW->>Snowflake DW: Transform data using dbt - Snowflake DW->>Sisense Dashboards: Data available for querying -``` - -## Implementing Snowplow JS (Frontend) tracking - -GitLab provides `Tracking`, an interface that wraps the [Snowplow JavaScript Tracker](https://github.com/snowplow/snowplow/wiki/javascript-tracker) for tracking custom events. There are a few ways to utilize tracking, but each generally requires at minimum, a `category` and an `action`. Additional data can be provided that adheres to our [Feature instrumentation taxonomy](https://about.gitlab.com/handbook/business-ops/data-team/programs/data-for-product-managers/#sts=Taxonomy). - -| field | type | default value | description | -|:-----------|:-------|:---------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `category` | string | document.body.dataset.page | Page or subsection of a page that events are being captured within. | -| `action` | string | 'generic' | Action the user is taking. Clicks should be `click` and activations should be `activate`, so for example, focusing a form field would be `activate_form_input`, and clicking a button would be `click_button`. | -| `data` | object | {} | Additional data such as `label`, `property`, `value`, and `context` as described [in our Feature Instrumentation taxonomy](https://about.gitlab.com/handbook/business-ops/data-team/programs/data-for-product-managers/#sts=Taxonomy). | - -### Tracking in HAML (or Vue Templates) - -When working within HAML (or Vue templates) we can add `data-track-*` attributes to elements of interest. All elements that have a `data-track-event` attribute automatically have event tracking bound on clicks. - -Below is an example of `data-track-*` attributes assigned to a button: - -```haml -%button.btn{ data: { track: { event: "click_button", label: "template_preview", property: "my-template" } } } -``` - -```html -<button class="btn" - data-track-event="click_button" - data-track-label="template_preview" - data-track-property="my-template" -/> -``` - -Event listeners are bound at the document level to handle click events on or within elements with these data attributes. This allows them to be properly handled on re-rendering and changes to the DOM. Note that because of the way these events are bound, click events should not be stopped from propagating up the DOM tree. If for any reason click events are being stopped from propagating, you need to implement your own listeners and follow the instructions in [Tracking in raw JavaScript](#tracking-in-raw-javascript). - -Below is a list of supported `data-track-*` attributes: - -| attribute | required | description | -|:----------------------|:---------|:------------| -| `data-track-event` | true | Action the user is taking. Clicks must be prepended with `click` and activations must be prepended with `activate`. For example, focusing a form field would be `activate_form_input` and clicking a button would be `click_button`. | -| `data-track-label` | false | The `label` as described [in our Feature Instrumentation taxonomy](https://about.gitlab.com/handbook/business-ops/data-team/programs/data-for-product-managers/#sts=Taxonomy). | -| `data-track-property` | false | The `property` as described [in our Feature Instrumentation taxonomy](https://about.gitlab.com/handbook/business-ops/data-team/programs/data-for-product-managers/#sts=Taxonomy). | -| `data-track-value` | false | The `value` as described [in our Feature Instrumentation taxonomy](https://about.gitlab.com/handbook/business-ops/data-team/programs/data-for-product-managers/#sts=Taxonomy). If omitted, this is the element's `value` property or an empty string. For checkboxes, the default value is the element's checked attribute or `false` when unchecked. | -| `data-track-context` | false | The `context` as described [in our Feature Instrumentation taxonomy](https://about.gitlab.com/handbook/business-ops/data-team/programs/data-for-product-managers/#sts=Taxonomy). | - -### Tracking within Vue components - -There's a tracking Vue mixin that can be used in components if more complex tracking is required. To use it, first import the `Tracking` library and request a mixin. - -```javascript -import Tracking from '~/tracking'; -const trackingMixin = Tracking.mixin({ label: 'right_sidebar' }); -``` - -You can provide default options that are passed along whenever an event is tracked from within your component. For instance, if all events within a component should be tracked with a given `label`, you can provide one at this time. Available defaults are `category`, `label`, `property`, and `value`. If no category is specified, `document.body.dataset.page` is used as the default. - -You can then use the mixin normally in your component with the `mixin` Vue declaration. The mixin also provides the ability to specify tracking options in `data` or `computed`. These override any defaults and allow the values to be dynamic from props, or based on state. - -```javascript -export default { - mixins: [trackingMixin], - // ...[component implementation]... - data() { - return { - expanded: false, - tracking: { - label: 'left_sidebar' - } - }; - }, -} -``` - -The mixin provides a `track` method that can be called within the template, or from component methods. An example of the whole implementation might look like the following. - -```javascript -export default { - mixins: [Tracking.mixin({ label: 'right_sidebar' })], - data() { - return { - expanded: false, - }; - }, - methods: { - toggle() { - this.expanded = !this.expanded; - this.track('click_toggle', { value: this.expanded }) - } - } -}; -``` - -And if needed within the template, you can use the `track` method directly as well. - -```html -<template> - <div> - <a class="toggle" @click.prevent="toggle">Toggle</a> - <div v-if="expanded"> - <p>Hello world!</p> - <a @click.prevent="track('click_action')">Track an event</a> - </div> - </div> -</template> -``` - -### Tracking in raw JavaScript - -Custom event tracking and instrumentation can be added by directly calling the `Tracking.event` static function. The following example demonstrates tracking a click on a button by calling `Tracking.event` manually. - -```javascript -import Tracking from '~/tracking'; - -const button = document.getElementById('create_from_template_button'); -button.addEventListener('click', () => { - Tracking.event('dashboard:projects:index', 'click_button', { - label: 'create_from_template', - property: 'template_preview', - value: 'rails', - }); -}) -``` - -### Tests and test helpers - -In Jest particularly in Vue tests, you can use the following: - -```javascript -import { mockTracking } from 'helpers/tracking_helper'; - -describe('MyTracking', () => { - let spy; - - beforeEach(() => { - spy = mockTracking('_category_', wrapper.element, jest.spyOn); - }); - - it('tracks an event when clicked on feedback', () => { - wrapper.find('.discover-feedback-icon').trigger('click'); - - expect(spy).toHaveBeenCalledWith('_category_', 'click_button', { - label: 'security-discover-feedback-cta', - property: '0', - }); - }); -}); -``` - -In obsolete Karma tests it's used as below: - -```javascript -import { mockTracking, triggerEvent } from 'spec/helpers/tracking_helper'; - -describe('my component', () => { - let trackingSpy; - - beforeEach(() => { - trackingSpy = mockTracking('_category_', vm.$el, spyOn); - }); - - const triggerEvent = () => { - // action which should trigger a event - }; - - it('tracks an event when toggled', () => { - expect(trackingSpy).not.toHaveBeenCalled(); - - triggerEvent('a.toggle'); - - expect(trackingSpy).toHaveBeenCalledWith('_category_', 'click_edit_button', { - label: 'right_sidebar', - property: 'confidentiality', - }); - }); -}); -``` - -## Implementing Snowplow Ruby (Backend) tracking - -GitLab provides `Gitlab::Tracking`, an interface that wraps the [Snowplow Ruby Tracker](https://github.com/snowplow/snowplow/wiki/ruby-tracker) for tracking custom events. - -Custom event tracking and instrumentation can be added by directly calling the `GitLab::Tracking.event` class method, which accepts the following arguments: - -| argument | type | default value | description | -|:-----------|:-------|:--------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `category` | string | 'application' | Area or aspect of the application. This could be `HealthCheckController` or `Lfs::FileTransformer` for instance. | -| `action` | string | 'generic' | The action being taken, which can be anything from a controller action like `create` to something like an Active Record callback. | -| `data` | object | {} | Additional data such as `label`, `property`, `value`, and `context` as described in [Instrumentation at GitLab](https://about.gitlab.com/handbook/business-ops/data-team/programs/data-for-product-managers/#sts=Taxonomy). These are set as empty strings if you don't provide them. | - -Tracking can be viewed as either tracking user behavior, or can be utilized for instrumentation to monitor and visualize performance over time in an area or aspect of code. - -For example: - -```ruby -class Projects::CreateService < BaseService - def execute - project = Project.create(params) - - Gitlab::Tracking.event('Projects::CreateService', 'create_project', - label: project.errors.full_messages.to_sentence, - value: project.valid? - ) - end -end -``` - -### Performance - -We use the [AsyncEmitter](https://github.com/snowplow/snowplow/wiki/Ruby-Tracker#52-the-asyncemitter-class) when tracking events, which allows for instrumentation calls to be run in a background thread. This is still an active area of development. - -## Developing and testing Snowplow - -There are several tools for developing and testing Snowplow Event - -| Testing Tool | Frontend Tracking | Backend Tracking | Local Development Environment | Production Environment | Production Environment | -|----------------------------------------------|--------------------|---------------------|-------------------------------|------------------------|------------------------| -| Snowplow Analytics Debugger Chrome Extension | **{check-circle}** | **{dotted-circle}** | **{check-circle}** | **{check-circle}** | **{check-circle}** | -| Snowplow Inspector Chrome Extension | **{check-circle}** | **{dotted-circle}** | **{check-circle}** | **{check-circle}** | **{check-circle}** | -| Snowplow Micro | **{check-circle}** | **{check-circle}** | **{check-circle}** | **{dotted-circle}** | **{dotted-circle}** | -| Snowplow Mini | **{check-circle}** | **{check-circle}** | **{dotted-circle}** | **{status_preparing}** | **{status_preparing}** | - -**Legend** - -**{check-circle}** Available, **{status_preparing}** In progress, **{dotted-circle}** Not Planned - -### Preparing your MR for Review - -1. For frontend events, in the MR description section, add a screenshot of the event's relevant section using the [Snowplow Analytics Debugger](https://chrome.google.com/webstore/detail/snowplow-analytics-debugg/jbnlcgeengmijcghameodeaenefieedm) Chrome browser extension. -1. For backend events, please use Snowplow Micro and add the output of the Snowplow Micro good events `GET http://localhost:9090/micro/good`. - -### Snowplow Analytics Debugger Chrome Extension - -Snowplow Analytics Debugger is a browser extension for testing frontend events. This works on production, staging and local development environments. - -1. Install the [Snowplow Analytics Debugger](https://chrome.google.com/webstore/detail/snowplow-analytics-debugg/jbnlcgeengmijcghameodeaenefieedm) Chrome browser extension. -1. Open Chrome DevTools to the Snowplow Analytics Debugger tab. -1. Learn more at [Igloo Analytics](https://www.iglooanalytics.com/blog/snowplow-analytics-debugger-chrome-extension.html). - -### Snowplow Inspector Chrome Extension - -Snowplow Inspector Chrome Extension is a browser extension for testing frontend events. This works on production, staging and local development environments. - -1. Install [Snowplow Inspector](https://chrome.google.com/webstore/detail/snowplow-inspector/maplkdomeamdlngconidoefjpogkmljm?hl=en). -1. Open the Chrome extension by pressing the Snowplow Inspector icon beside the address bar. -1. Click around on a webpage with Snowplow and you should see JavaScript events firing in the inspector window. - -### Snowplow Micro - -Snowplow Micro is a very small version of a full Snowplow data collection pipeline: small enough that it can be launched by a test suite. Events can be recorded into Snowplow Micro just as they can a full Snowplow pipeline. Micro then exposes an API that can be queried. - -Snowplow Micro is a Docker-based solution for testing frontend and backend events in a local development environment. You need to modify GDK using the instructions below to set this up. - -- Read [Introducing Snowplow Micro](https://snowplowanalytics.com/blog/2019/07/17/introducing-snowplow-micro/) -- Look at the [Snowplow Micro repository](https://github.com/snowplow-incubator/snowplow-micro) -- Watch our [installation guide recording](https://www.youtube.com/watch?v=OX46fo_A0Ag) - -1. Install [Snowplow Micro](https://github.com/snowplow-incubator/snowplow-micro): - - ```shell - docker run --mount type=bind,source=$(pwd)/example,destination=/config -p 9090:9090 snowplow/snowplow-micro:latest --collector-config /config/micro.conf --iglu /config/iglu.json - ``` - -1. Install snowplow micro by cloning the settings in [this project](https://gitlab.com/a_akgun/snowplow-micro): - - ```shell - git clone git@gitlab.com:a_akgun/snowplow-micro.git - ./snowplow-micro.sh - ``` - -1. Update port in SQL to set `9090`: - - ```shell - gdk psql -d gitlabhq_development - update application_settings set snowplow_collector_hostname='localhost:9090', snowplow_enabled=true, snowplow_cookie_domain='.gitlab.com'; - ``` - -1. Update `app/assets/javascripts/tracking.js` to [remove this line](https://gitlab.com/snippets/1918635): - - ```javascript - forceSecureTracker: true - ``` - -1. Update `lib/gitlab/tracking.rb` to [add these lines](https://gitlab.com/snippets/1918635): - - ```ruby - protocol: 'http', - port: 9090, - ``` - -1. Update `lib/gitlab/tracking.rb` to [change async emitter from https to http](https://gitlab.com/snippets/1918635): - - ```ruby - SnowplowTracker::AsyncEmitter.new(Gitlab::CurrentSettings.snowplow_collector_hostname, protocol: 'http'), - ``` - -1. Enable Snowplow in the admin area, Settings::Integrations::Snowplow to point to: - `http://localhost:3000/admin/application_settings/integrations#js-snowplow-settings`. - -1. Restart GDK: - - ```shell - `gdk restart` - ``` - -1. Send a test Snowplow event from the Rails console: - - ```ruby - Gitlab::Tracking.self_describing_event('iglu:com.gitlab/pageview_context/jsonschema/1-0-0', { page_type: 'MY_TYPE' }, context: nil ) - ``` - -### Snowplow Mini - -[Snowplow Mini](https://github.com/snowplow/snowplow-mini) is an easily-deployable, single-instance version of Snowplow. - -Snowplow Mini can be used for testing frontend and backend events on a production, staging and local development environment. - -For GitLab.com, we're setting up a [QA and Testing environment](https://gitlab.com/gitlab-org/telemetry/-/issues/266) using Snowplow Mini. +This document was moved to [another location](../product_analytics/snowplow.md). diff --git a/doc/development/telemetry/usage_ping.md b/doc/development/telemetry/usage_ping.md index ff4e7e0797b..1026e7a5a66 100644 --- a/doc/development/telemetry/usage_ping.md +++ b/doc/development/telemetry/usage_ping.md @@ -1,887 +1,5 @@ --- -stage: Growth -group: Telemetry -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +redirect_to: '../product_analytics/usage_ping.md' --- -# Usage Ping Guide - -> - Introduced in GitLab Enterprise Edition 8.10. -> - More statistics were added in GitLab Enterprise Edition 8.12. -> - Moved to GitLab Core in 9.1. -> - More statistics were added in GitLab Ultimate 11.2. - -This guide describes Usage Ping's purpose and how it's implemented. - -For more information about Telemetry, see: - -- [Telemetry Guide](index.md) -- [Snowplow Guide](snowplow.md) - -More useful links: - -- [Telemetry Direction](https://about.gitlab.com/direction/telemetry/) -- [Data Analysis Process](https://about.gitlab.com/handbook/business-ops/data-team/#data-analysis-process/) -- [Data for Product Managers](https://about.gitlab.com/handbook/business-ops/data-team/programs/data-for-product-managers/) -- [Data Infrastructure](https://about.gitlab.com/handbook/business-ops/data-team/platform/infrastructure/) - -## What is Usage Ping? - -- GitLab sends a weekly payload containing usage data to GitLab Inc. Usage Ping provides high-level data to help our product, support, and sales teams. It does not send any project names, usernames, or any other specific data. The information from the usage ping is not anonymous, it is linked to the hostname of the instance. Sending usage ping is optional, and any instance can disable analytics. -- The usage data is primarily composed of row counts for different tables in the instance’s database. By comparing these counts month over month (or week over week), we can get a rough sense for how an instance is using the different features within the product. In addition to counts, other facts - that help us classify and understand GitLab installations are collected. -- Usage ping is important to GitLab as we use it to calculate our Stage Monthly Active Users (SMAU) which helps us measure the success of our stages and features. -- Once usage ping is enabled, GitLab will gather data from the other instances and will be able to show usage statistics of your instance to your users. - -### Why should we enable Usage Ping? - -- The main purpose of Usage Ping is to build a better GitLab. Data about how GitLab is used is collected to better understand feature/stage adoption and usage, which helps us understand how GitLab is adding value and helps our team better understand the reasons why people use GitLab and with this knowledge we're able to make better product decisions. -- As a benefit of having the usage ping active, GitLab lets you analyze the users’ activities over time of your GitLab installation. -- As a benefit of having the usage ping active, GitLab provides you with The DevOps Report,which gives you an overview of your entire instance’s adoption of Concurrent DevOps from planning to monitoring. -- You will get better, more proactive support. (assuming that our TAMs and support organization used the data to deliver more value) -- You will get insight and advice into how to get the most value out of your investment in GitLab. Wouldn't you want to know that a number of features or values are not being adopted in your organization? -- You get a report that illustrates how you compare against other similar organizations (anonymized), with specific advice and recommendations on how to improve your DevOps processes. -- Usage Ping is enabled by default. To disable it, see [Disable Usage Ping](#disable-usage-ping). - -### Limitations - -- Usage Ping does not track frontend events things like page views, link clicks, or user sessions, and only focuses on aggregated backend events. -- Because of these limitations we recommend instrumenting your products with Snowplow for more detailed analytics on GitLab.com and use Usage Ping to track aggregated backend events on self-managed. - -## Usage Ping payload - -You can view the exact JSON payload sent to GitLab Inc. in the administration panel. To view the payload: - -1. Navigate to **Admin Area > Settings > Metrics and profiling**. -1. Expand the **Usage statistics** section. -1. Click the **Preview payload** button. - -For an example payload, see [Example Usage Ping payload](#example-usage-ping-payload). - -## Disable Usage Ping - -To disable Usage Ping in the GitLab UI, go to the **Settings** page of your administration panel and uncheck the **Usage Ping** checkbox. - -To disable Usage Ping and prevent it from being configured in the future through the administration panel, Omnibus installs can set the following in [`gitlab.rb`](https://docs.gitlab.com/omnibus/settings/configuration.html#configuration-options): - -```ruby -gitlab_rails['usage_ping_enabled'] = false -``` - -Source installations can set the following in `gitlab.yml`: - -```yaml -production: &base - # ... - gitlab: - # ... - usage_ping_enabled: false -``` - -## Usage Ping request flow - -The following example shows a basic request/response flow between a GitLab instance, the Versions Application, the License Application, Salesforce, GitLab's S3 Bucket, GitLab's Snowflake Data Warehouse, and Sisense: - -```mermaid -sequenceDiagram - participant GitLab Instance - participant Versions Application - participant Licenses Application - participant Salesforce - participant S3 Bucket - participant Snowflake DW - participant Sisense Dashboards - GitLab Instance->>Versions Application: Send Usage Ping - loop Process usage data - Versions Application->>Versions Application: Parse usage data - Versions Application->>Versions Application: Write to database - Versions Application->>Versions Application: Update license ping time - end - loop Process data for Salesforce - Versions Application-xLicenses Application: Request Zuora subscription id - Licenses Application-xVersions Application: Zuora subscription id - Versions Application-xSalesforce: Request Zuora account id by Zuora subscription id - Salesforce-xVersions Application: Zuora account id - Versions Application-xSalesforce: Usage data for the Zuora account - end - Versions Application->>S3 Bucket: Export Versions database - S3 Bucket->>Snowflake DW: Import data - Snowflake DW->>Snowflake DW: Transform data using dbt - Snowflake DW->>Sisense Dashboards: Data available for querying - Versions Application->>GitLab Instance: DevOps Report (Conversational Development Index) -``` - -## How Usage Ping works - -1. The Usage Ping [cron job](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/workers/gitlab_usage_ping_worker.rb#L30) is set in Sidekiq to run weekly. -1. When the cron job runs, it calls [`GitLab::UsageData.to_json`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/services/submit_usage_ping_service.rb#L22). -1. `GitLab::UsageData.to_json` [cascades down](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data.rb#L22) to ~400+ other counter method calls. -1. The response of all methods calls are [merged together](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data.rb#L14) into a single JSON payload in `GitLab::UsageData.to_json`. -1. The JSON payload is then [posted to the Versions application]( https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/services/submit_usage_ping_service.rb#L20). - -## Implementing Usage Ping - -Usage Ping consists of two kinds of data, counters and observations. Counters track how often a certain event -happened over time, such as how many CI pipelines have run. They are monotonic and always trend up. -Observations are facts collected from one or more GitLab instances and can carry arbitrary data. There are no -general guidelines around how to collect those, due to the individual nature of that data. - -There are four types of counters which are all found in `usage_data.rb`: - -- **Ordinary Batch Counters:** Simple count of a given ActiveRecord_Relation -- **Distinct Batch Counters:** Distinct count of a given ActiveRecord_Relation on given column -- **Alternative Counters:** Used for settings and configurations -- **Redis Counters:** Used for in-memory counts. - -NOTE: **Note:** -Only use the provided counter methods. Each counter method contains a built in fail safe to isolate each counter to avoid breaking the entire Usage Ping. - -### Why batch counting - -For large tables, PostgreSQL can take a long time to count rows due to MVCC [(Multi-version Concurrency Control)](https://en.wikipedia.org/wiki/Multiversion_concurrency_control). Batch counting is a counting method where a single large query is broken into multiple smaller queries. For example, instead of a single query querying 1,000,000 records, with batch counting, you can execute 100 queries of 10,000 records each. Batch counting is useful for avoiding database timeouts as each batch query is significantly shorter than one single long running query. - -For GitLab.com, there are extremely large tables with 15 second query timeouts, so we use batch counting to avoid encountering timeouts. Here are the sizes of some GitLab.com tables: - -| Table | Row counts in millions | -|------------------------------|------------------------| -| `merge_request_diff_commits` | 2280 | -| `ci_build_trace_sections` | 1764 | -| `merge_request_diff_files` | 1082 | -| `events` | 514 | - -There are two batch counting methods provided, `Ordinary Batch Counters` and `Distinct Batch Counters`. Batch counting requires indexes on columns to calculate max, min, and range queries. In some cases, a specialized index may need to be added on the columns involved in a counter. - -### Ordinary Batch Counters - -Handles `ActiveRecord::StatementInvalid` error - -Simple count of a given ActiveRecord_Relation, does a non-distinct batch count, smartly reduces batch_size and handles errors. - -Method: `count(relation, column = nil, batch: true, start: nil, finish: nil)` - -Arguments: - -- `relation` the ActiveRecord_Relation to perform the count -- `column` the column to perform the count on, by default is the primary key -- `batch`: default `true` in order to use batch counting -- `start`: custom start of the batch counting in order to avoid complex min calculations -- `end`: custom end of the batch counting in order to avoid complex min calculations - -Examples: - -```ruby -count(User.active) -count(::Clusters::Cluster.aws_installed.enabled, :cluster_id) -count(::Clusters::Cluster.aws_installed.enabled, :cluster_id, start: ::Clusters::Cluster.minimum(:id), finish: ::Clusters::Cluster.maximum(:id)) -``` - -### Distinct Batch Counters - -Handles `ActiveRecord::StatementInvalid` error - -Distinct count of a given ActiveRecord_Relation on given column, a distinct batch count, smartly reduces batch_size and handles errors. - -Method: `distinct_count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil)` - -Arguments: - -- `relation` the ActiveRecord_Relation to perform the count -- `column` the column to perform the distinct count, by default is the primary key -- `batch`: default `true` in order to use batch counting -- `batch_size`: if none set it will use default value 10000 from `Gitlab::Database::BatchCounter` -- `start`: custom start of the batch counting in order to avoid complex min calculations -- `end`: custom end of the batch counting in order to avoid complex min calculations - -Examples: - -```ruby -distinct_count(::Project, :creator_id) -distinct_count(::Note.with_suggestions.where(time_period), :author_id, start: ::User.minimum(:id), finish: ::User.maximum(:id)) -distinct_count(::Clusters::Applications::CertManager.where(time_period).available.joins(:cluster), 'clusters.user_id') -``` - -### Redis Counters - -Handles `::Redis::CommandError` and `Gitlab::UsageDataCounters::BaseCounter::UnknownEvent` -returns -1 when a block is sent or hash with all values -1 when a `counter(Gitlab::UsageDataCounters)` is sent -different behavior due to 2 different implementations of Redis counter - -Method: `redis_usage_data(counter, &block)` - -Arguments: - -- `counter`: a counter from `Gitlab::UsageDataCounters`, that has `fallback_totals` method implemented -- or a `block`: which is evaluated - -#### Ordinary Redis Counters - -Examples of implementation: - -- Using Redis methods [`INCR`](https://redis.io/commands/incr), [`GET`](https://redis.io/commands/get), and [`Gitlab::UsageDataCounters::WikiPageCounter`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/wiki_page_counter.rb) -- Using Redis methods [`HINCRBY`](https://redis.io/commands/hincrby), [`HGETALL`](https://redis.io/commands/hgetall), and [`Gitlab::UsageCounters::PodLogs`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_counters/pod_logs.rb) - -#### Redis HLL Counters - -With `Gitlab::UsageDataCounters::HLLRedisCounter` we have available data structures used to count unique values. - -Implemented using Redis methods [PFADD](https://redis.io/commands/pfadd) and [PFCOUNT](https://redis.io/commands/pfcount). - -##### Adding new events - -1. Define events in [`known_events.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/known_events.yml). - - Example event: - - ```yaml - - name: i_compliance_credential_inventory - category: compliance - redis_slot: compliance - expiry: 42 # 6 weeks - aggregation: weekly - ``` - - Keys: - - - `name`: unique event name. - - Name format `<prefix>_<redis_slot>_name`. - - Use one of the following prefixes for the event's name: - - - `g_` for group, as an event which is tracked for group. - - `p_` for project, as an event which is tracked for project. - - `i_` for instance, as an event which is tracked for instance. - - `a_` for events encompassing all `g_`, `p_`, `i_`. - - `o_` for other. - - Consider including in the event's name the Redis slot in order to be able to count totals for a specific category. - - Example names: `i_compliance_credential_inventory`, `g_analytics_contribution`. - - - `category`: event category. Used for getting total counts for events in a category, for easier - access to a group of events. - - `redis_slot`: optional Redis slot; default value: event name. Used if needed to calculate totals - for a group of metrics. Ensure keys are in the same slot. For example: - `i_compliance_credential_inventory` with `redis_slot: 'compliance'` will build Redis key - `i_{compliance}_credential_inventory-2020-34`. If `redis_slot` is not defined the Redis key will - be `{i_compliance_credential_inventory}-2020-34`. - - `expiry`: expiry time in days. Default: 29 days for daily aggregation and 6 weeks for weekly - aggregation. - - `aggregation`: aggregation `:daily` or `:weekly`. The argument defines how we build the Redis - keys for data storage. For `daily` we keep a key for metric per day of the year, for `weekly` we - keep a key for metric per week of the year. - -1. Track event in controller using `RedisTracking` module with `track_redis_hll_event(*controller_actions, name:, feature:, feature_default_enabled: false)`. - - Arguments: - - - `controller_actions`: controller actions we want to track. - - `name`: event name. - - `feature`: feature name, all metrics we track should be under feature flag. - - `feature_default_enabled`: feature flag is disabled by default, set to `true` for it to be enabled by default. - - Example usage: - - ```ruby - # controller - class ProjectsController < Projects::ApplicationController - include RedisTracking - - skip_before_action :authenticate_user!, only: :show - track_redis_hll_event :index, :show, name: 'i_analytics_dev_ops_score', feature: :g_compliance_dashboard_feature, feature_default_enabled: true - - def index - render html: 'index' - end - - def new - render html: 'new' - end - - def show - render html: 'show' - end - end - ``` - -1. Track event in API using `increment_unique_values(event_name, values)` helper method. - - In order to be able to track the event, Usage Ping must be enabled and the event feature `usage_data_<event_name>` must be enabled. - - Arguments: - - - `event_name`: event name. - - `values`: values counted, one value or array of values. - - Example usage: - - ```ruby - get ':id/registry/repositories' do - repositories = ContainerRepositoriesFinder.new( - user: current_user, subject: user_group - ).execute - - increment_unique_values('i_list_repositories', current_user.id) - - present paginate(repositories), with: Entities::ContainerRegistry::Repository, tags: params[:tags], tags_count: params[:tags_count] - end - ``` - -1. Track event using `track_usage_event(event_name, values) in services and graphql - - Increment unique values count using Redis HLL, for given event name. - - Example: - - [Track usage event for incident created in service](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/services/issues/update_service.rb) - - [Track usage event for incident created in graphql](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/graphql/mutations/alert_management/update_alert_status.rb) - - ```ruby - track_usage_event(:incident_management_incident_created, current_user.id) - ``` - -1. Track event using `UsageData` API - - Increment unique users count using Redis HLL, for given event name. - - Tracking events using the `UsageData` API requires the `usage_data_api` feature flag to be enabled, which is disabled by default. - - API requests are protected by checking for a valid CSRF token. - - In order to be able to increment the values the related feature `usage_data<event_name>` should be enabled. - - ```plaintext - POST /usage_data/increment_unique_users - ``` - - | Attribute | Type | Required | Description | - | :-------- | :--- | :------- | :---------- | - | `event` | string | yes | The event name it should be tracked | - - Response - - Return 200 if tracking failed for any reason. - - - `200` if event was tracked or any errors - - `400 Bad request` if event parameter is missing - - `401 Unauthorized` if user is not authenticated - - `403 Forbidden` for invalid CSRF token provided - -1. Track event using base module `Gitlab::UsageDataCounters::HLLRedisCounter.track_event(entity_id, event_name)`. - - Arguments: - - - `entity_id`: value we count. For example: user_id, visitor_id. - - `event_name`: event name. - -1. Get event data using `Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names:, start_date:, end_date)`. - - Arguments: - - - `event_names`: the list of event names. - - `start_date`: start date of the period for which we want to get event data. - - `end_date`: end date of the period for which we want to get event data. - -Recommendations: - -- Key should expire in 29 days for daily and 42 days for weekly. -- If possible, data granularity should be a week. For example a key could be composed from the - metric's name and week of the year, `2020-33-{metric_name}`. -- Use a [feature flag](../../operations/feature_flags.md) to have a control over the impact when - adding new metrics. - -##### Known events in usage data payload - -All events added in [`known_events.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/known_events.yml) are automatically added to usage data generation under the `redis_hll_counters` key. This column is stored in [version-app as a JSON](https://gitlab.com/gitlab-services/version-gitlab-com/-/blob/master/db/schema.rb#L209). -For each event we add metrics for the weekly and monthly time frames, and totals for each where applicable: - -- `#{event_name}_weekly` data for 7 days for daily [aggregation](#adding-new-events) events and data for last complete week for weekly [aggregation](#adding-new-events) events. -- `#{event_name}_monthly` data for 28 days for daily [aggregation](#adding-new-events) events and data for last 4 complete weeks for weekly [aggregation](#adding-new-events) events. -- `#{category}_total_unique_counts_weekly` total unique counts for events in same category for last 7 days or last complete week, if events are in the same Redis slot and if we have more than one metric. -- `#{event_name}_weekly` - Data for 7 days for daily [aggregation](#adding-new-events) events and data for the last complete week for weekly [aggregation](#adding-new-events) events. -- `#{event_name}_monthly` - Data for 28 days for daily [aggregation](#adding-new-events) events and data for the last 4 complete weeks for weekly [aggregation](#adding-new-events) events. -- `#{category}_total_unique_counts_weekly` - Total unique counts for events in the same category for the last 7 days or the last complete week, if events are in the same Redis slot and we have more than one metric. -- `#{event_name}_weekly`: Data for 7 days for daily [aggregation](#adding-new-events) events and data for last complete week for weekly [aggregation](#adding-new-events) events. -- `#{event_name}_monthly`: Data for 28 days for daily [aggregation](#adding-new-events) events and data for last 4 complete weeks for weekly [aggregation](#adding-new-events) events. -- `#{category}_total_unique_counts_weekly` total unique counts for events in same category for last 7 days or last complete week, if events are in the same Redis slot and if we have more than one metric. -- `#{event_name}_weekly`: Data for 7 days for daily [aggregation](#adding-new-events) events and data for the last complete week for weekly [aggregation](#adding-new-events) events. -- `#{event_name}_monthly`: Data for 28 days for daily [aggregation](#adding-new-events) events and data for the last 4 complete weeks for weekly [aggregation](#adding-new-events) events. -- `#{category}_total_unique_counts_weekly`: Total unique counts for events in the same category for the last 7 days or the last complete week, if events are in the same Redis slot and we have more than one metric. -- `#{category}_total_unique_counts_monthly`: Total unique counts for events in same category for the last 28 days or the last 4 complete weeks, if events are in the same Redis slot and we have more than one metric. - -Example of `redis_hll_counters` data: - -```ruby -{:redis_hll_counters=> - {"compliance"=> - {"g_compliance_dashboard_weekly"=>0, - "g_compliance_dashboard_monthly"=>0, - "g_compliance_audit_events_weekly"=>0, - "g_compliance_audit_events_monthly"=>0, - "compliance_total_unique_counts_weekly"=>0, - "compliance_total_unique_counts_monthly"=>0}, - "analytics"=> - {"g_analytics_contribution_weekly"=>0, - "g_analytics_contribution_monthly"=>0, - "g_analytics_insights_weekly"=>0, - "g_analytics_insights_monthly"=>0, - "analytics_total_unique_counts_weekly"=>0, - "analytics_total_unique_counts_monthly"=>0}, - "ide_edit"=> - {"g_edit_by_web_ide_weekly"=>0, - "g_edit_by_web_ide_monthly"=>0, - "g_edit_by_sfe_weekly"=>0, - "g_edit_by_sfe_monthly"=>0, - "ide_edit_total_unique_counts_weekly"=>0, - "ide_edit_total_unique_counts_monthly"=>0}, - "search"=> - {"i_search_total_weekly"=>0, "i_search_total_monthly"=>0, "i_search_advanced_weekly"=>0, "i_search_advanced_monthly"=>0, "i_search_paid_weekly"=>0, "i_search_paid_monthly"=>0, "search_total_unique_counts_weekly"=>0, "search_total_unique_counts_monthly"=>0}, - "source_code"=>{"wiki_action_weekly"=>0, "wiki_action_monthly"=>0} - } -``` - -Example usage: - -```ruby -# Redis Counters -redis_usage_data(Gitlab::UsageDataCounters::WikiPageCounter) -redis_usage_data { ::Gitlab::UsageCounters::PodLogs.usage_totals[:total] } - -# Define events in known_events.yml https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/known_events.yml - -# Tracking events -Gitlab::UsageDataCounters::HLLRedisCounter.track_event(visitor_id, 'expand_vulnerabilities') - -# Get unique events for metric -redis_usage_data { Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: 'expand_vulnerabilities', start_date: 28.days.ago, end_date: Date.current) } -``` - -### Alternative Counters - -Handles `StandardError` and fallbacks into -1 this way not all measures fail if we encounter one exception. -Mainly used for settings and configurations. - -Method: `alt_usage_data(value = nil, fallback: -1, &block)` - -Arguments: - -- `value`: a simple static value in which case the value is simply returned. -- or a `block`: which is evaluated -- `fallback: -1`: the common value used for any metrics that are failing. - -Example of usage: - -```ruby -alt_usage_data { Gitlab::VERSION } -alt_usage_data { Gitlab::CurrentSettings.uuid } -alt_usage_data(999) -``` - -### Prometheus Queries - -In those cases where operational metrics should be part of Usage Ping, a database or Redis query is unlikely -to provide useful data. Instead, Prometheus might be more appropriate, since most of GitLab's architectural -components publish metrics to it that can be queried back, aggregated, and included as usage data. - -NOTE: **Note:** -Prometheus as a data source for Usage Ping is currently only available for single-node Omnibus installations -that are running the [bundled Prometheus](../../administration/monitoring/prometheus/index.md) instance. - -In order to query Prometheus for metrics, a helper method is available that will `yield` a fully configured -`PrometheusClient`, given it is available as per the note above: - -```ruby -with_prometheus_client do |client| - response = client.query('<your query>') - ... -end -``` - -Please refer to [the `PrometheusClient` definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/prometheus_client.rb) -for how to use its API to query for data. - -## Developing and testing Usage Ping - -### 1. Use your Rails console to manually test counters - -```ruby -# count -Gitlab::UsageData.count(User.active) -Gitlab::UsageData.count(::Clusters::Cluster.aws_installed.enabled, :cluster_id) - -# count distinct -Gitlab::UsageData.distinct_count(::Project, :creator_id) -Gitlab::UsageData.distinct_count(::Note.with_suggestions.where(time_period), :author_id, start: ::User.minimum(:id), finish: ::User.maximum(:id)) -``` - -### 2. Generate the SQL query - -Your Rails console will return the generated SQL queries. - -Example: - -```ruby -pry(main)> Gitlab::UsageData.count(User.active) - (2.6ms) SELECT "features"."key" FROM "features" - (15.3ms) SELECT MIN("users"."id") FROM "users" WHERE ("users"."state" IN ('active')) AND ("users"."user_type" IS NULL OR "users"."user_type" IN (6, 4)) - (2.4ms) SELECT MAX("users"."id") FROM "users" WHERE ("users"."state" IN ('active')) AND ("users"."user_type" IS NULL OR "users"."user_type" IN (6, 4)) - (1.9ms) SELECT COUNT("users"."id") FROM "users" WHERE ("users"."state" IN ('active')) AND ("users"."user_type" IS NULL OR "users"."user_type" IN (6, 4)) AND "users"."id" BETWEEN 1 AND 100000 -``` - -### 3. Optimize queries with #database-lab - -Paste the SQL query into `#database-lab` to see how the query performs at scale. - -- `#database-lab` is a Slack channel which uses a production-sized environment to test your queries. -- GitLab.com’s production database has a 15 second timeout. -- Any single query must stay below 1 second execution time with cold caches. -- Add a specialized index on columns involved to reduce the execution time. - -In order to have an understanding of the query's execution we add in the MR description the following information: - -- For counters that have a `time_period` test we add information for both cases: - - `time_period = {}` for all time periods - - `time_period = { created_at: 28.days.ago..Time.current }` for last 28 days period -- Execution plan and query time before and after optimization -- Query generated for the index and time -- Migration output for up and down execution - -We also use `#database-lab` and [explain.depesz.com](https://explain.depesz.com/). For more details, see the [database review guide](../database_review.md#preparation-when-adding-or-modifying-queries). - -#### Optimization recommendations and examples - -- Use specialized indexes [example 1](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/26871), [example 2](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/26445). -- Use defined `start` and `finish`, and simple queries, because these values can be memoized and reused, [example](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/37155). -- Avoid joins and write the queries as simply as possible, [example](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/36316). -- Set a custom `batch_size` for `distinct_count`, [example](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/38000). - -### 4. Add the metric definition - -When adding, changing, or updating metrics, please update the [Event Dictionary's **Usage Ping** table](event_dictionary.md). - -### 5. Add new metric to Versions Application - -Check if new metrics need to be added to the Versions Application. See `usage_data` [schema](https://gitlab.com/gitlab-services/version-gitlab-com/-/blob/master/db/schema.rb#L147) and usage data [parameters accepted](https://gitlab.com/gitlab-services/version-gitlab-com/-/blob/master/app/services/usage_ping.rb). Any metrics added under the `counts` key are saved in the `counts` column. - -### 6. Add the feature label - -Add the `feature` label to the Merge Request for new Usage Ping metrics. These are user-facing changes and are part of expanding the Usage Ping feature. - -### 7. Add a changelog file - -Ensure you comply with the [Changelog entries guide](../changelog.md). - -### 8. Ask for a Telemetry Review - -On GitLab.com, we have DangerBot setup to monitor Telemetry related files and DangerBot will recommend a Telemetry review. Mention `@gitlab-org/growth/telemetry/engineers` in your MR for a review. - -### 9. Verify your metric - -On GitLab.com, the Product Analytics team regularly monitors Usage Ping. They may alert you that your metrics need further optimization to run quicker and with greater success. You may also use the [Usage Ping QA dashboard](https://app.periscopedata.com/app/gitlab/632033/Usage-Ping-QA) to check how well your metric performs. The dashboard allows filtering by GitLab version, by "Self-managed" & "Saas" and shows you how many failures have occurred for each metric. Whenever you notice a high failure rate, you may re-optimize your metric. - -### Optional: Test Prometheus based Usage Ping - -If the data submitted includes metrics [queried from Prometheus](#prometheus-queries) that you would like to inspect and verify, -then you need to ensure that a Prometheus server is running locally, and that furthermore the respective GitLab components -are exporting metrics to it. If you do not need to test data coming from Prometheus, no further action -is necessary, since Usage Ping should degrade gracefully in the absence of a running Prometheus server. - -There are currently three kinds of components that may export data to Prometheus, and which are included in Usage Ping: - -- [`node_exporter`](https://github.com/prometheus/node_exporter) - Exports node metrics from the host machine -- [`gitlab-exporter`](https://gitlab.com/gitlab-org/gitlab-exporter) - Exports process metrics from various GitLab components -- various GitLab services such as Sidekiq and the Rails server that export their own metrics - -#### Test with an Omnibus container - -This is the recommended approach to test Prometheus based Usage Ping. - -The easiest way to verify your changes is to build a new Omnibus image from your code branch via CI, then download the image -and run a local container instance: - -1. From your merge request, click on the `qa` stage, then trigger the `package-and-qa` job. This job will trigger an Omnibus -build in a [downstream pipeline of the `omnibus-gitlab-mirror` project](https://gitlab.com/gitlab-org/build/omnibus-gitlab-mirror/-/pipelines). -1. In the downstream pipeline, wait for the `gitlab-docker` job to finish. -1. Open the job logs and locate the full container name including the version. It will take the following form: `registry.gitlab.com/gitlab-org/build/omnibus-gitlab-mirror/gitlab-ee:<VERSION>`. -1. On your local machine, make sure you are logged in to the GitLab Docker registry. You can find the instructions for this in -[Authenticating to the GitLab Container Registry](../../user/packages/container_registry/index.md#authenticating-to-the-gitlab-container-registry). -1. Once logged in, download the new image via `docker pull registry.gitlab.com/gitlab-org/build/omnibus-gitlab-mirror/gitlab-ee:<VERSION>` -1. For more information about working with and running Omnibus GitLab containers in Docker, please refer to [GitLab Docker images](https://docs.gitlab.com/omnibus/docker/README.html) in the Omnibus documentation. - -#### Test with GitLab development toolkits - -This is the less recommended approach, since it comes with a number of difficulties when emulating a real GitLab deployment. - -The [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit) is not currently set up to run a Prometheus server or `node_exporter` alongside other GitLab components. If you would -like to do so, [Monitoring the GDK with Prometheus](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/master/doc/howto/prometheus/index.md#monitoring-the-gdk-with-prometheus) is a good start. - -The [GCK](https://gitlab.com/gitlab-org/gitlab-compose-kit) has limited support for testing Prometheus based Usage Ping. -By default, it already comes with a fully configured Prometheus service that is set up to scrape a number of components, -but with the following limitations: - -- It does not currently run a `gitlab-exporter` instance, so several `process_*` metrics from services such as Gitaly may be missing. -- While it runs a `node_exporter`, `docker-compose` services emulate hosts, meaning that it would normally report itself to not be associated -with any of the other services that are running. That is not how node metrics are reported in a production setup, where `node_exporter` -always runs as a process alongside other GitLab components on any given node. From Usage Ping's perspective none of the node data would therefore -appear to be associated to any of the services running, since they all appear to be running on different hosts. To alleviate this problem, the `node_exporter` in GCK was arbitrarily "assigned" to the `web` service, meaning only for this service `node_*` metrics will appear in Usage Ping. - -## Example Usage Ping payload - -The following is example content of the Usage Ping payload. - -```json -{ - "uuid": "0000000-0000-0000-0000-000000000000", - "hostname": "example.com", - "version": "12.10.0-pre", - "installation_type": "omnibus-gitlab", - "active_user_count": 999, - "recorded_at": "2020-04-17T07:43:54.162+00:00", - "edition": "EEU", - "license_md5": "00000000000000000000000000000000", - "license_id": null, - "historical_max_users": 999, - "licensee": { - "Name": "ABC, Inc.", - "Email": "email@example.com", - "Company": "ABC, Inc." - }, - "license_user_count": 999, - "license_starts_at": "2020-01-01", - "license_expires_at": "2021-01-01", - "license_plan": "ultimate", - "license_add_ons": { - }, - "license_trial": false, - "counts": { - "assignee_lists": 999, - "boards": 999, - "ci_builds": 999, - ... - }, - "container_registry_enabled": true, - "dependency_proxy_enabled": false, - "gitlab_shared_runners_enabled": true, - "gravatar_enabled": true, - "influxdb_metrics_enabled": true, - "ldap_enabled": false, - "mattermost_enabled": false, - "omniauth_enabled": true, - "prometheus_enabled": false, - "prometheus_metrics_enabled": false, - "reply_by_email_enabled": "incoming+%{key}@incoming.gitlab.com", - "signup_enabled": true, - "web_ide_clientside_preview_enabled": true, - "ingress_modsecurity_enabled": true, - "projects_with_expiration_policy_disabled": 999, - "projects_with_expiration_policy_enabled": 999, - ... - "elasticsearch_enabled": true, - "license_trial_ends_on": null, - "geo_enabled": false, - "git": { - "version": { - "major": 2, - "minor": 26, - "patch": 1 - } - }, - "gitaly": { - "version": "12.10.0-rc1-93-g40980d40", - "servers": 56, - "clusters": 14, - "filesystems": [ - "EXT_2_3_4" - ] - }, - "gitlab_pages": { - "enabled": true, - "version": "1.17.0" - }, - "container_registry_server": { - "vendor": "gitlab", - "version": "2.9.1-gitlab" - }, - "database": { - "adapter": "postgresql", - "version": "9.6.15" - }, - "avg_cycle_analytics": { - "issue": { - "average": 999, - "sd": 999, - "missing": 999 - }, - "plan": { - "average": null, - "sd": 999, - "missing": 999 - }, - "code": { - "average": null, - "sd": 999, - "missing": 999 - }, - "test": { - "average": null, - "sd": 999, - "missing": 999 - }, - "review": { - "average": null, - "sd": 999, - "missing": 999 - }, - "staging": { - "average": null, - "sd": 999, - "missing": 999 - }, - "production": { - "average": null, - "sd": 999, - "missing": 999 - }, - "total": 999 - }, - "analytics_unique_visits": { - "g_analytics_contribution": 999, - ... - }, - "usage_activity_by_stage": { - "configure": { - "project_clusters_enabled": 999, - ... - }, - "create": { - "merge_requests": 999, - ... - }, - "manage": { - "events": 999, - ... - }, - "monitor": { - "clusters": 999, - ... - }, - "package": { - "projects_with_packages": 999 - }, - "plan": { - "issues": 999, - ... - }, - "release": { - "deployments": 999, - ... - }, - "secure": { - "user_container_scanning_jobs": 999, - ... - }, - "verify": { - "ci_builds": 999, - ... - } - }, - "usage_activity_by_stage_monthly": { - "configure": { - "project_clusters_enabled": 999, - ... - }, - "create": { - "merge_requests": 999, - ... - }, - "manage": { - "events": 999, - ... - }, - "monitor": { - "clusters": 999, - ... - }, - "package": { - "projects_with_packages": 999 - }, - "plan": { - "issues": 999, - ... - }, - "release": { - "deployments": 999, - ... - }, - "secure": { - "user_container_scanning_jobs": 999, - ... - }, - "verify": { - "ci_builds": 999, - ... - } - }, - "topology": { - "duration_s": 0.013836685999194742, - "application_requests_per_hour": 4224, - "query_apdex_weekly_average": 0.996, - "failures": [], - "nodes": [ - { - "node_memory_total_bytes": 33269903360, - "node_memory_utilization": 0.35, - "node_cpus": 16, - "node_cpu_utilization": 0.2, - "node_uname_info": { - "machine": "x86_64", - "sysname": "Linux", - "release": "4.19.76-linuxkit" - }, - "node_services": [ - { - "name": "web", - "process_count": 16, - "process_memory_pss": 233349888, - "process_memory_rss": 788220927, - "process_memory_uss": 195295487, - "server": "puma" - }, - { - "name": "sidekiq", - "process_count": 1, - "process_memory_pss": 734080000, - "process_memory_rss": 750051328, - "process_memory_uss": 731533312 - }, - ... - ], - ... - }, - ... - ] - } -} -``` - -## Exporting Usage Ping SQL queries and definitions - -Two Rake tasks exist to export Usage Ping definitions. - -- The Rake tasks export the raw SQL queries for `count`, `distinct_count`, `sum`. -- The Rake tasks export the Redis counter class or the line of the Redis block for `redis_usage_data`. -- The Rake tasks calculate the `alt_usage_data` metrics. - -In the home directory of your local GitLab installation run the following Rake tasks for the YAML and JSON versions respectively: - -```shell -# for YAML export -bin/rake gitlab:usage_data:dump_sql_in_yaml - -# for JSON export -bin/rake gitlab:usage_data:dump_sql_in_json - -# You may pipe the output into a file -bin/rake gitlab:usage_data:dump_sql_in_yaml > ~/Desktop/usage-metrics-2020-09-02.yaml -``` +This document was moved to [another location](../product_analytics/usage_ping.md). diff --git a/doc/development/testing_guide/best_practices.md b/doc/development/testing_guide/best_practices.md index 6ef9be381b4..2c1d70a005e 100644 --- a/doc/development/testing_guide/best_practices.md +++ b/doc/development/testing_guide/best_practices.md @@ -81,7 +81,7 @@ browser is much slower than parsing the HTML response from the app. A common cause of slow tests is excessive creation of objects, and thus computation and DB time. Factories are essential to development, but they can -make inserting data into the DB so easy that we may be able to optimize. +make inserting data into the DB so easy that we may be able to optimize. The two basic techniques to bear in mind here are: @@ -151,11 +151,14 @@ In order to reuse a single object for all calls to a named factory in implicit p can be used: ```ruby +RSpec.describe API::Search, factory_default: :keep do let_it_be(:namespace) { create_default(:namespace) } ``` Then every project we create will use this `namespace`, without us having to pass -it as `namespace: namespace`. +it as `namespace: namespace`. In order to make it work along with `let_it_be`, `factory_default: :keep` +must be explicitly specified. That will keep the default factory for every example in a suite instead of +recreating it for each example. Maybe we don't need to create 208 different projects - we can create one and reuse it. In addition, we can see that only about 1/3 of the @@ -237,7 +240,7 @@ end it 'schedules a background job' do expect(BackgroundJob).to receive(:perform_async) - + subject.execute end ``` @@ -249,7 +252,7 @@ combining the examples: ```ruby it 'performs the expected side-effects' do expect(BackgroundJob).to receive(:perform_async) - + expect { subject.execute } .to change(Event, :count).by(1) .and change { arg_0.frobulance }.to('wibble') @@ -481,26 +484,30 @@ This will result in only one `Project`, `User`, and `ProjectMember` created for is handled automatically using a transaction rollback. Note that if you modify an object defined inside a `let_it_be` block, -then you will need to reload the object as needed, or specify the `reload` -option to reload for every example. +then you must do one of the following: + +- Reload the object as needed. +- Use the `let_it_be_with_reload` alias. +- Specify the `reload` option to reload for every example. ```ruby +let_it_be_with_reload(:project) { create(:project) } let_it_be(:project, reload: true) { create(:project) } ``` -You can also specify the `refind` option as well to completely load a -new object. +You can also use the `let_it_be_with_refind` alias, or specify the `refind` +option as well to completely load a new object. ```ruby +let_it_be_with_refind(:project) { create(:project) } let_it_be(:project, refind: true) { create(:project) } ``` ### Time-sensitive tests -[Timecop](https://github.com/travisjeffery/timecop) is available in our -Ruby-based tests for verifying things that are time-sensitive. Any test that -exercises or verifies something time-sensitive should make use of Timecop to -prevent transient test failures. +[`ActiveSupport::Testing::TimeHelpers`](https://api.rubyonrails.org/v6.0.3.1/classes/ActiveSupport/Testing/TimeHelpers.html) +can be used to verify things that are time-sensitive. Any test that exercises or verifies something time-sensitive +should make use of these helpers to prevent transient test failures. Example: @@ -508,7 +515,7 @@ Example: it 'is overdue' do issue = build(:issue, due_date: Date.tomorrow) - Timecop.freeze(3.days.from_now) do + travel_to(3.days.from_now) do expect(issue).to be_overdue end end @@ -888,6 +895,10 @@ GitLab uses [factory_bot](https://github.com/thoughtbot/factory_bot) as a test f resulting record to pass validation. - When instantiating from a factory, don't supply attributes that aren't required by the test. +- Prefer [implicit](https://github.com/thoughtbot/factory_bot/blob/master/GETTING_STARTED.md#implicit-definition) + or [explicit](https://github.com/thoughtbot/factory_bot/blob/master/GETTING_STARTED.md#explicit-definition) + association definitions instead of using `create` / `build` for association setup. + See [issue #262624](https://gitlab.com/gitlab-org/gitlab/-/issues/262624) for further context. - Factories don't have to be limited to `ActiveRecord` objects. [See example](https://gitlab.com/gitlab-org/gitlab-foss/commit/0b8cefd3b2385a21cfed779bd659978c0402766d). diff --git a/doc/development/testing_guide/ci.md b/doc/development/testing_guide/ci.md index 6917639454c..8091142410c 100644 --- a/doc/development/testing_guide/ci.md +++ b/doc/development/testing_guide/ci.md @@ -28,9 +28,6 @@ After that, the next pipeline will use the up-to-date `knapsack/report-master.js The GitLab test suite is [monitored](../performance.md#rspec-profiling) for the `master` branch, and any branch that includes `rspec-profile` in their name. -A [public dashboard](https://redash.gitlab.com/public/dashboards/l1WhHXaxrCWM5Ai9D7YDqHKehq6OU3bx5gssaiWe?org_slug=default) is available for everyone to see. Feel free to look at the -slowest test files and try to improve them. - ## CI setup - Rails logging to `log/test.log` is disabled by default in CI [for diff --git a/doc/development/testing_guide/end_to_end/beginners_guide.md b/doc/development/testing_guide/end_to_end/beginners_guide.md index c552c44c864..a1883f44170 100644 --- a/doc/development/testing_guide/end_to_end/beginners_guide.md +++ b/doc/development/testing_guide/end_to_end/beginners_guide.md @@ -196,8 +196,8 @@ end **What do we test?** -1. Can we log in? -1. Can we log out? +1. Can we sign in? +1. Can we sign out? **How do we test?** diff --git a/doc/development/testing_guide/end_to_end/best_practices.md b/doc/development/testing_guide/end_to_end/best_practices.md index 36cb49256a6..58bae749dc5 100644 --- a/doc/development/testing_guide/end_to_end/best_practices.md +++ b/doc/development/testing_guide/end_to_end/best_practices.md @@ -241,7 +241,11 @@ All tests expect to be able to log in at the start of the test. For an example see: <https://gitlab.com/gitlab-org/gitlab/-/issues/34736> -Ideally, any actions performed in an `after(:context)` (or [`before(:context)`](#limit-the-use-of-the-ui-in-beforecontext-and-after-hooks)) block would be performed via the API. But if it's necessary to do so via the UI (e.g., if API functionality doesn't exist), make sure to log out at the end of the block. +Ideally, actions performed in an `after(:context)` (or +[`before(:context)`](#limit-the-use-of-the-ui-in-beforecontext-and-after-hooks)) +block are performed using the API. If it's necessary to do so with the user +interface (for example, if API functionality doesn't exist), be sure to sign +out at the end of the block. ```ruby after(:all) do @@ -310,3 +314,56 @@ end # Using native mouse click events in the case of a mask/overlay click_element_coordinates(:title) ``` + +## Ensure `expect` statements wait efficiently + +In general, we use an `expect` statement to check that something _is_ as we expect it. For example: + +```ruby +Page::Project::Pipeline::Show.perform do |pipeline| + expect(pipeline).to have_job("a_job") +end +``` + +### Ensure `expect` checks for negation efficiently + +However, sometimes we want to check that something is _not_ as we _don't_ want it to be. In other +words, we want to make sure something is absent. In such a case we should use an appropriate +predicate method that returns quickly, rather than waiting for a state that won't appear. + +It's most efficient to use a predicate method that returns immediately when there is no job, or waits +until it disappears: + +```ruby +# Good +Page::Project::Pipeline::Show.perform do |pipeline| + expect(pipeline).to have_no_job("a_job") +end +``` + +### Problematic alternatives + +Alternatively, if we want to check that a job doesn't exist it might be tempting to use `not_to`: + +```ruby +# Bad +Page::Project::Pipeline::Show.perform do |pipeline| + expect(pipeline).not_to have_job("a_job") +end +``` + +For this statement to pass, `have_job("a_job")` has to return `false` so that `not_to` can negate it. +The problem is that `have_job("a_job")` waits up to ten seconds for `"a job"` to appear before +returning `false`. Under the expected condition this test will take ten seconds longer than it needs to. + +Instead, we could force no wait: + +```ruby +# Not as bad but potentially flaky +Page::Project::Pipeline::Show.perform do |pipeline| + expect(pipeline).not_to have_job("a_job", wait: 0) +end +``` + +The problem is that if `"a_job"` is present and we're waiting for it to disappear, this statement +will fail. diff --git a/doc/development/testing_guide/end_to_end/environment_selection.md b/doc/development/testing_guide/end_to_end/environment_selection.md index 9eb7db72a51..325f251b280 100644 --- a/doc/development/testing_guide/end_to_end/environment_selection.md +++ b/doc/development/testing_guide/end_to_end/environment_selection.md @@ -1,7 +1,7 @@ # Environment selection -Some tests are designed to be run against specific environments. We can specify -what environments to run tests against using the `only` metadata. +Some tests are designed to be run against specific environments or [pipelines](https://about.gitlab.com/handbook/engineering/quality/guidelines/debugging-qa-test-failures/#scheduled-qa-test-pipelines). +We can specify what environments or pipelines to run tests against using the `only` metadata. ## Available switches @@ -11,15 +11,16 @@ what environments to run tests against using the `only` metadata. | `subdomain` | Set the subdomain matcher | `Array` or `String` | | `domain` | Set the domain matcher | `String` | | `production` | Match against production | `Static` | +| `pipeline` | Match against a pipeline | `Array` or `Static`| CAUTION: **Caution:** -You cannot specify `:production` and `{ <switch>: 'value' }` simultaneously. +You cannot specify `:production` and `{ <switch>: 'value' }` simultaneously. These options are mutually exclusive. If you want to specify production, you can control the `tld` and `domain` independently. ## Examples -| Environment | Key | Matches (regex) | +| Environment or pipeline | Key | Matches (regex for environments, string matching for pipelines) | | ---------------- | --- | --------------- | | `any` | `` | `.+.com` | | `gitlab.com` | `only: :production` | `gitlab.com` | @@ -27,18 +28,24 @@ can control the `tld` and `domain` independently. | `gitlab.com and staging.gitlab.com` | `only: { subdomain: /(staging.)?/, domain: 'gitlab' }` | `(staging.)?gitlab.com` | | `dev.gitlab.org` | `only: { tld: '.org', domain: 'gitlab', subdomain: 'dev' }` | `(dev).gitlab.org` | | `staging.gitlab.com & domain.gitlab.com` | `only: { subdomain: %i[staging domain] }` | `(staging|domain).+.com` | +| `nightly` | `only: { pipeline: :nightly }` | "nightly" | +| `nightly`, `canary` | `only_run_in_pipeline: [:nightly, :canary]` | ["nightly"](https://gitlab.com/gitlab-org/quality/nightly) and ["canary"](https://gitlab.com/gitlab-org/quality/canary) | ```ruby RSpec.describe 'Area' do - it 'runs in any environment' do; end + it 'runs in any environment or pipeline' do; end - it 'runs only in production', only: :production do; end + it 'runs only in production environment', only: :production do; end - it 'runs only in staging', only: { subdomain: :staging } do; end + it 'runs only in staging environment', only: { subdomain: :staging } do; end - it 'runs in dev', only: { tld: '.org', domain: 'gitlab', subdomain: 'dev' } do; end + it 'runs in dev environment', only: { tld: '.org', domain: 'gitlab', subdomain: 'dev' } do; end - it 'runs in prod and staging', only: { subdomain: /(staging.)?/, domain: 'gitlab' } {} + it 'runs in prod and staging environments', only: { subdomain: /(staging.)?/, domain: 'gitlab' } {} + + it 'runs only in nightly pipeline', only: { pipeline: :nightly } do; end + + it 'runs in nightly and canary pipelines', only: { pipeline: [:nightly, :canary] } do; end end ``` @@ -46,6 +53,8 @@ NOTE: **Note:** If the test has a `before` or `after`, you must add the `only` metadata to the outer `RSpec.describe`. +If you want to run an `only: { :pipeline }` tagged test on your local GDK make sure either the `CI_PROJECT_NAME` environment variable is unset, or that the `CI_PROJECT_NAME` environment variable matches the specified pipeline in the `only: { :pipeline }` tag, or just delete the `only: { :pipeline }` tag. + ## Quarantining a test for a specific environment Similarly to specifying that a test should only run against a specific environment, it's also possible to quarantine a diff --git a/doc/development/testing_guide/end_to_end/feature_flags.md b/doc/development/testing_guide/end_to_end/feature_flags.md index 87a9738b313..e571774167d 100644 --- a/doc/development/testing_guide/end_to_end/feature_flags.md +++ b/doc/development/testing_guide/end_to_end/feature_flags.md @@ -1,29 +1,70 @@ # Testing with feature flags -To run a specific test with a feature flag enabled you can use the `QA::Runtime::Feature` class to enable and disable feature flags ([via the API](../../../api/features.md)). +To run a specific test with a feature flag enabled you can use the `QA::Runtime::Feature` class to +enable and disable feature flags ([via the API](../../../api/features.md)). -Note that administrator authorization is required to change feature flags. `QA::Runtime::Feature` will automatically authenticate as an administrator as long as you provide an appropriate access token via `GITLAB_QA_ADMIN_ACCESS_TOKEN` (recommended), or provide `GITLAB_ADMIN_USERNAME` and `GITLAB_ADMIN_PASSWORD`. +Note that administrator authorization is required to change feature flags. `QA::Runtime::Feature` +will automatically authenticate as an administrator as long as you provide an appropriate access +token via `GITLAB_QA_ADMIN_ACCESS_TOKEN` (recommended), or provide `GITLAB_ADMIN_USERNAME` +and `GITLAB_ADMIN_PASSWORD`. -Please be sure to include the tag `:requires_admin` so that the test can be skipped in environments where admin access is not available. +Please be sure to include the tag `:requires_admin` so that the test can be skipped in environments +where admin access is not available. + +CAUTION: **Caution:** +You are strongly advised to [enable feature flags only for a group, project, user](../../feature_flags/development.md#feature-actors), +or [feature group](../../feature_flags/development.md#feature-groups). This makes it possible to +test a feature in a shared environment without affecting other users. + +For example, the code below would enable a feature flag named `:feature_flag_name` for the project +created by the test: ```ruby RSpec.describe "with feature flag enabled", :requires_admin do + let(:project) { Resource::Project.fabricate_via_api! } + before do - Runtime::Feature.enable('feature_flag_name') + Runtime::Feature.enable(:feature_flag_name, project: project) end it "feature flag test" do - # Execute a test with a feature flag enabled + # Execute the test with the feature flag enabled. + # It will only affect the project created in this test. end after do - Runtime::Feature.disable('feature_flag_name') + Runtime::Feature.disable(:feature_flag_name, project: project) end end ``` +Note that the `enable` and `disable` methods first set the flag and then check that the updated +value is returned by the API. + +Similarly, you can enable a feature for a group, user, or feature group: + +```ruby +group = Resource::Group.fabricate_via_api! +Runtime::Feature.enable(:feature_flag_name, group: group) + +user = Resource::User.fabricate_via_api! +Runtime::Feature.enable(:feature_flag_name, user: user) + +feature_group = "a_feature_group" +Runtime::Feature.enable(:feature_flag_name, feature_group: feature_group) +``` + +If no scope is provided, the feature flag will be set instance-wide: + +```ruby +# This will affect all users! +Runtime::Feature.enable(:feature_flag_name) +``` + ## Running a scenario with a feature flag enabled -It's also possible to run an entire scenario with a feature flag enabled, without having to edit existing tests or write new ones. +It's also possible to run an entire scenario with a feature flag enabled, without having to edit +existing tests or write new ones. -Please see the [QA README](https://gitlab.com/gitlab-org/gitlab/tree/master/qa#running-tests-with-a-feature-flag-enabled) for details. +Please see the [QA README](https://gitlab.com/gitlab-org/gitlab/tree/master/qa#running-tests-with-a-feature-flag-enabled) +for details. diff --git a/doc/development/testing_guide/end_to_end/rspec_metadata_tests.md b/doc/development/testing_guide/end_to_end/rspec_metadata_tests.md index a9f54b53e5a..3a1303d9c0c 100644 --- a/doc/development/testing_guide/end_to_end/rspec_metadata_tests.md +++ b/doc/development/testing_guide/end_to_end/rspec_metadata_tests.md @@ -11,7 +11,7 @@ This is a partial list of the [RSpec metadata](https://relishapp.com/rspec/rspec | `:gitaly_cluster` | The test will run against a GitLab instance where repositories are stored on redundant Gitaly nodes behind a Praefect node. All nodes are [separate containers](../../../administration/gitaly/praefect.md#requirements-for-configuring-a-gitaly-cluster). Tests that use this tag have a longer setup time since there are three additional containers that need to be started. | | `:jira` | The test requires a Jira Server. [GitLab-QA](https://gitlab.com/gitlab-org/gitlab-qa) will provision the Jira Server in a Docker container when the `Test::Integration::Jira` test scenario is run. | `:kubernetes` | The test includes a GitLab instance that is configured to be run behind an SSH tunnel, allowing a TLS-accessible GitLab. This test will also include provisioning of at least one Kubernetes cluster to test against. *This tag is often be paired with `:orchestrated`.* | -| `:only` | The test is only to be run against specific environments. See [Environment selection](environment_selection.md) for more information. | +| `:only` | The test is only to be run against specific environments or pipelines. See [Environment selection](environment_selection.md) for more information. | | `:orchestrated` | The GitLab instance under test may be [configured by `gitlab-qa`](https://gitlab.com/gitlab-org/gitlab-qa/-/blob/master/docs/what_tests_can_be_run.md#orchestrated-tests) to be different to the default GitLab configuration, or `gitlab-qa` may launch additional services in separate Docker containers, or both. Tests tagged with `:orchestrated` are excluded when testing environments where we can't dynamically modify GitLab's configuration (for example, Staging). | | `:quarantine` | The test has been [quarantined](https://about.gitlab.com/handbook/engineering/quality/guidelines/debugging-qa-test-failures/#quarantining-tests), will run in a separate job that only includes quarantined tests, and is allowed to fail. The test will be skipped in its regular job so that if it fails it will not hold up the pipeline. Note that you can also [quarantine a test only when it runs against specific environment](environment_selection.md#quarantining-a-test-for-a-specific-environment). | | `:reliable` | The test has been [promoted to a reliable test](https://about.gitlab.com/handbook/engineering/quality/guidelines/reliable-tests/#promoting-an-existing-test-to-reliable) meaning it passes consistently in all pipelines, including merge requests. | diff --git a/doc/development/testing_guide/end_to_end/running_tests_that_require_special_setup.md b/doc/development/testing_guide/end_to_end/running_tests_that_require_special_setup.md index 7ac0a00fcff..658839fcf96 100644 --- a/doc/development/testing_guide/end_to_end/running_tests_that_require_special_setup.md +++ b/doc/development/testing_guide/end_to_end/running_tests_that_require_special_setup.md @@ -274,7 +274,7 @@ CHROME_HEADLESS=false bundle exec bin/qa QA::EE::Scenario::Test::Geo --primary-a ### Using Geo in Docker -You can use [GitLab-QA Orchestrator](https://gitlab.com/gitlab-org/gitlab-qa) to orchestrate two GitLab containers and configure them as a Geo setup. +You can use [GitLab-QA Orchestrator](https://gitlab.com/gitlab-org/gitlab-qa) to orchestrate two GitLab containers and configure them as a Geo setup. Geo requires an EE license. To visit the Geo sites in your browser, you will need a reverse proxy server (for example, [NGINX](https://www.nginx.com/)). @@ -319,7 +319,7 @@ Geo requires an EE license. To visit the Geo sites in your browser, you will nee _Map the hostnames to the local IP in `/etc/hosts` file on your machine:_ ```plaintext - 127.0.0.1 gitlab-primary.geo gitlab-secondary.geo + 127.0.0.1 gitlab-primary.geo gitlab-secondary.geo ``` _Note the assigned ports:_ diff --git a/doc/development/testing_guide/end_to_end/style_guide.md b/doc/development/testing_guide/end_to_end/style_guide.md index 7e9f097f624..645c2633831 100644 --- a/doc/development/testing_guide/end_to_end/style_guide.md +++ b/doc/development/testing_guide/end_to_end/style_guide.md @@ -141,7 +141,7 @@ end ``` ```ruby -Page::Project::New.peform do |new_page| +Page::Project::New.perform do |new_page| new_page.do_something end ``` @@ -155,7 +155,7 @@ end ``` ```ruby -Page::Project::New.peform do |page| +Page::Project::New.perform do |page| page.do_something end ``` diff --git a/doc/development/testing_guide/frontend_testing.md b/doc/development/testing_guide/frontend_testing.md index 30e78766dde..730f8d5ad7d 100644 --- a/doc/development/testing_guide/frontend_testing.md +++ b/doc/development/testing_guide/frontend_testing.md @@ -229,7 +229,7 @@ beforeEach(() => { it('exists', () => { // Best - // NOTE: both mount and shallowMount work as long as a DOM element is available + // NOTE: both mount and shallowMount work as long as a DOM element is available // Finds a properly formatted link with an accessible name of "Click Me" getByRole(el, 'link', { name: /Click Me/i }) getByRole(el, 'link', { name: 'Click Me' }) diff --git a/doc/development/what_requires_downtime.md b/doc/development/what_requires_downtime.md index 7c99bcde413..9063fb867e2 100644 --- a/doc/development/what_requires_downtime.md +++ b/doc/development/what_requires_downtime.md @@ -176,7 +176,7 @@ class ChangeUsersUsernameStringToText < ActiveRecord::Migration[4.2] end def down - cleanup_concurrent_column_type_change :users, :username + undo_change_column_type_concurrently :users, :username end end ``` @@ -197,7 +197,7 @@ class ChangeUsersUsernameStringToTextCleanup < ActiveRecord::Migration[4.2] end def down - change_column_type_concurrently :users, :username, :string + undo_cleanup_concurrent_column_type_change :users, :username, :string end end ``` diff --git a/doc/development/wikis.md b/doc/development/wikis.md new file mode 100644 index 00000000000..9a436762645 --- /dev/null +++ b/doc/development/wikis.md @@ -0,0 +1,94 @@ +--- +type: reference, dev +stage: none +group: Development +info: "See the Technical Writers assigned to Development Guidelines: https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments-to-development-guidelines" +description: "GitLab's development guidelines for Wikis" +--- + +# Wikis development guide + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/227027) in GitLab 13.5. + +## Overview + +The wiki functionality in GitLab is based on [Gollum 4.x](https://github.com/gollum/gollum/), +which is used in [Gitaly's](gitaly.md) Ruby service and accessed from the Rails app through Gitaly RPC calls. + +Wikis use Git repositories as storage backend, and can be accessed through: + +- The [Web UI](../user/project/wiki/index.md) +- The [REST API](../api/wikis.md) +- [Git itself](../user/project/wiki/#adding-and-editing-wiki-pages-locally) + +[Introduced](https://gitlab.com/groups/gitlab-org/-/epics/2214) in GitLab 13.5, wikis are also available +for groups, in addition to projects. + +## Involved Gems + +Some notable gems that are used for wikis are: + +| Component | Description | Gem name | GitLab project | Upstream project | +|:--------------|:-----------------------------------------------|:-------------------------------|:--------------------------------------------------------------------------------------------------------|:--------------------------------------------------------------------| +| `gitlab` | Markup renderer, depends on various other gems | `gitlab-markup` | [`gitlab-org/gitlab-markup`](https://gitlab.com/gitlab-org/gitlab-markup) | [`github/markup`](https://github.com/github/markup) | +| `gitaly-ruby` | Main Gollum library | `gitlab-gollum-lib` | [`gitlab-org/gollum-lib`](https://gitlab.com/gitlab-org/gollum-lib) | [`gollum/gollum-lib`](https://github.com/gollum/gollum-lib) | +| | Gollum Git adapter for Rugged | `gitlab-gollum-rugged_adapter` | [`gitlab-org/gitlab-gollum-rugged_adapter`](https://gitlab.com/gitlab-org/gitlab-gollum-rugged_adapter) | [`gollum/rugged_adapter`](https://github.com/gollum/rugged_adapter) | +| | Rugged (also used in Gitaly itself) | `rugged` | - | [`libgit2/rugged`](https://github.com/libgit2/rugged) | + +### Notes on Gollum + +We only use Gollum as a storage abstraction layer, to handle the mapping between wiki page slugs and files in the repository. + +When rendering wiki pages, we don't use Gollum at all and instead go through a +[custom Banzai pipeline](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/banzai/pipeline/wiki_pipeline.rb). +This adds some [wiki-specific markup](../user/markdown.md#wiki-specific-markdown), such as Gollum's `[[link]]` syntax. + +Since we do not make use of most of Gollum's features, we plan to move away from it entirely at some point. +[See this epic](https://gitlab.com/groups/gitlab-org/-/epics/2381) for reference. + +## Model classes + +The `Wiki` class is the main abstraction around a wiki repository, it needs to be initialized +with a container which can be either a `Project` or `Group`: + +```mermaid +classDiagram + Wiki --> ProjectWiki + Wiki --> GroupWiki + + class Wiki { + #container + #repository + } + + class ProjectWiki { + #project → #container + } + + class GroupWiki { + #group → #container + } +``` + +Some models wrap similar classes from Gitaly and Gollum: + +| Rails Model | Gitaly Class | Gollum | +|:------------|:--------------------------------------------------------|:---------------| +| `Wiki` | `Gitlab::Git::Wiki` | `Gollum::Wiki` | +| `WikiPage` | `Gitlab::Git::WikiPage`, `Gitlab::Git::WikiPageVersion` | `Gollum::Page` | +| | `Gitlab::Git::WikiFile` | `Gollum::File` | + +Only some data is persisted in the database: + +| Model | Description | +|:----------------------|:-----------------------------------------| +| `WikiPage::Meta` | Metadata for wiki pages | +| `WikiPage::Slug` | Current and previous slugs of wiki pages | +| `ProjectRepository` | Gitaly storage data for project wikis | +| `GroupWikiRepository` | Gitaly storage data for group wikis | + +## Attachments + +The web UI uploads attachments through the REST API, which stores the files as commits in the wiki repository. + +Prior to GitLab 11.3 attachments were stored outside of the repository, [see this issue](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/33475). diff --git a/doc/getting-started/subscription.md b/doc/getting-started/subscription.md deleted file mode 100644 index 8bcd11c20c8..00000000000 --- a/doc/getting-started/subscription.md +++ /dev/null @@ -1,3 +0,0 @@ ---- -redirect_to: '../subscriptions/index.md' ---- diff --git a/doc/git_hooks/git_hooks.md b/doc/git_hooks/git_hooks.md deleted file mode 100644 index b251e58410a..00000000000 --- a/doc/git_hooks/git_hooks.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../push_rules/push_rules.md' ---- - -This document was moved to [another location](../push_rules/push_rules.md) diff --git a/doc/gitlab-basics/README.md b/doc/gitlab-basics/README.md index a7f18e13f74..3850655aaed 100644 --- a/doc/gitlab-basics/README.md +++ b/doc/gitlab-basics/README.md @@ -31,7 +31,7 @@ The following are guides to basic GitLab functionality: - [Add a file](add-file.md), to add new files to a project's repository. - [Create an issue](../user/project/issues/managing_issues.md#create-a-new-issue), to start collaborating within a project. -- [Create a merge request](add-merge-request.md), to request changes made in a branch +- [Create a merge request](../user/project/merge_requests/creating_merge_requests.md), to request changes made in a branch be merged into a project's repository. - See how these features come together in the [GitLab Flow introduction video](https://youtu.be/InKNIvky2KE) and [GitLab Flow page](../topics/gitlab_flow.md). diff --git a/doc/gitlab-basics/add-file.md b/doc/gitlab-basics/add-file.md index c5b57d4623d..0b400d7fdb4 100644 --- a/doc/gitlab-basics/add-file.md +++ b/doc/gitlab-basics/add-file.md @@ -29,16 +29,17 @@ to the desired destination: cd <destination folder> ``` -[Create a branch](create-branch.md) to add your file to, before it is added to the master -(main) branch of the project. It is not strictly necessary, but working directly in -the `master` branch is not recommended unless your project is very small, and you are -the only person working on it. You can [switch to an existing branch](start-using-git.md#work-on-an-existing-branch), +[Create a new branch](create-branch.md) to add your file into. Submitting changes directly +to the default branch should be avoided unless your project is very small and you're the +only person working on it. + +You can also [switch to an existing branch](start-using-git.md#work-on-an-existing-branch) if you have one already. Using your standard tool for copying files (for example, Finder in macOS, or File Explorer -in Windows), put the file into a directory within the GitLab project. +on Windows), put the file into a directory within the GitLab project. -Check if your file is actually present in the directory (if you are in Windows, +Check if your file is actually present in the directory (if you're on Windows, use `dir` instead): ```shell @@ -79,7 +80,7 @@ Now you can push (send) your changes (in the branch `<branch-name>`) to GitLab git push origin <branch-name> ``` -Your image will be added to your branch in your repository in GitLab. +Your image is added to your branch in your repository in GitLab. <!-- ## Troubleshooting diff --git a/doc/gitlab-basics/create-branch.md b/doc/gitlab-basics/create-branch.md index 0ad5cb53e97..9e51a2749a6 100644 --- a/doc/gitlab-basics/create-branch.md +++ b/doc/gitlab-basics/create-branch.md @@ -9,11 +9,11 @@ type: howto A branch is an independent line of development in a [project](../user/project/index.md). -When you create a new branch (in your [terminal](basic-git-commands.md) or with +When you create a new branch (in your [terminal](start-using-git.md) or with [the web interface](../user/project/repository/web_editor.md#create-a-new-branch)), you are creating a snapshot of a certain branch, usually the main `master` branch, -at it's current state. From there, you can start to make your own changes without +at its current state. From there, you can start to make your own changes without affecting the main codebase. The history of your changes will be tracked in your branch. When your changes are ready, you then merge them into the rest of the codebase with a -[merge request](add-merge-request.md). +[merge request](../user/project/merge_requests/creating_merge_requests.md). diff --git a/doc/gitlab-basics/create-project.md b/doc/gitlab-basics/create-project.md index f411ac769c0..616bb752694 100644 --- a/doc/gitlab-basics/create-project.md +++ b/doc/gitlab-basics/create-project.md @@ -18,13 +18,13 @@ To create a project in GitLab: icon in the navigation bar. This opens the **New project** page. 1. On the **New project** page, choose if you want to: - Create a [blank project](#blank-projects). - - Create a project using with one of the available [project templates](#project-templates). + - Create a project using one of the available [project templates](#project-templates). - [Import a project](../user/project/import/index.md) from a different repository, - if enabled on your GitLab instance. Contact your GitLab admin if this is unavailable. + if enabled on your GitLab instance. Contact your GitLab administrator if this is unavailable. - Run [CI/CD pipelines for external repositories](../ci/ci_cd_for_external_repos/index.md). **(PREMIUM)** NOTE: **Note:** -For a list of words that cannot be used as project names see +For a list of words that can't be used as project names see [Reserved project and group names](../user/reserved_names.md). ### Blank projects @@ -33,17 +33,17 @@ To create a new blank project on the **New project** page: 1. On the **Blank project** tab, provide the following information: - The name of your project in the **Project name** field. You can't use - special characters, but you can use spaces, hyphens, underscores or even - emoji. When adding the name, the **Project slug** will auto populate. - The slug is what the GitLab instance will use as the URL path to the project. + special characters, but you can use spaces, hyphens, underscores, or even + emoji. When adding the name, the **Project slug** auto populates. + The slug is what the GitLab instance uses as the URL path to the project. If you want a different slug, input the project name first, then change the slug after. - The path to your project in the **Project slug** field. This is the URL - path for your project that the GitLab instance will use. If the - **Project name** is blank, it will auto populate when you fill in + path for your project that the GitLab instance uses. If the + **Project name** is blank, it auto populates when you fill in the **Project slug**. - The **Project description (optional)** field enables you to enter a - description for your project's dashboard, which will help others + description for your project's dashboard, which helps others understand what your project is about. Though it's not required, it's a good idea to fill this in. - Changing the **Visibility Level** modifies the project's @@ -58,7 +58,7 @@ To create a new blank project on the **New project** page: Project templates can pre-populate a new project with the necessary files to get you started quickly. -There are two types of project templates: +There are two main types of project templates: - [Built-in templates](#built-in-templates), sourced from the following groups: - [`project-templates`](https://gitlab.com/gitlab-org/project-templates) @@ -128,11 +128,11 @@ To use a custom project template on the **New project** page: > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/26388) in GitLab 10.5. -When you create a new repository locally, instead of going to GitLab to manually -create a new project and then [clone the repo](start-using-git.md#clone-a-repository) +When you create a new repository locally, instead of manually creating a new project in GitLab +and then [cloning the repository](start-using-git.md#clone-a-repository) locally, you can directly push it to GitLab to create the new project, all without leaving -your terminal. If you have access rights to the associated namespace, GitLab will -automatically create a new project under that GitLab namespace with its visibility +your terminal. If you have access rights to the associated namespace, GitLab +automatically creates a new project under that GitLab namespace with its visibility set to Private by default (you can later change it in the [project's settings](../public_access/public_access.md#how-to-change-project-visibility)). This can be done by using either SSH or HTTPS: @@ -145,7 +145,7 @@ git push --set-upstream git@gitlab.example.com:namespace/nonexistent-project.git git push --set-upstream https://gitlab.example.com/namespace/nonexistent-project.git master ``` -Once the push finishes successfully, a remote message will indicate +Once the push finishes successfully, a remote message indicates the command to set the remote and the URL to the new project: ```plaintext diff --git a/doc/gitlab-basics/start-using-git.md b/doc/gitlab-basics/start-using-git.md index 3812fd3b92a..b6192700d29 100644 --- a/doc/gitlab-basics/start-using-git.md +++ b/doc/gitlab-basics/start-using-git.md @@ -323,19 +323,25 @@ to work on a different **branch**. When you create a branch in a Git repository, you make a copy of its files at the time of branching. You're free to do whatever you want with the code in your branch without impacting the main branch or other branches. And when -you're ready to bring your changes to the main codebase, you can merge your branch into the main one +you're ready to bring your changes to the main codebase, you can merge your branch into the default branch used in your project (such as `master`). +A new branch is often called **feature branch** to differentiate from the +**default branch**. + ### Create a branch -To create a new branch, to work from without affecting the `master` branch, type -the following (spaces won't be recognized in the branch name, so you will need to -use a hyphen or underscore): +To create a new feature branch and work from without affecting the `master` +branch: ```shell git checkout -b <name-of-branch> ``` +Note that Git does **not** accept empty spaces and special characters in branch +names, so use only lowercase letters, numbers, hyphens (`-`), and underscores +(`_`). Do not use capital letters, as it may cause duplications. + ### Switch to the master branch You are always in a branch when working with Git. The main branch is the master @@ -411,6 +417,9 @@ For example, to push your local commits to the _`master`_ branch of the _`origin git push origin master ``` +On certain occasions, Git won't allow you to push to your repository, and then +you'll need to [force an update](../topics/git/git_rebase.md#force-push). + NOTE: **Note:** To create a merge request from a fork to an upstream repository, see the [forking workflow](../user/project/repository/forking_workflow.md). @@ -459,6 +468,10 @@ git checkout <name-of-branch> git merge master ``` +## Advanced use of Git through the command line + +For an introduction of more advanced Git techniques, see [Git rebase, force-push, and merge conflicts](../topics/git/git_rebase.md). + ## Synchronize changes in a forked repository with the upstream [Forking a repository](../user/project/repository/forking_workflow.md) lets you create diff --git a/doc/gitlab-geo/README.md b/doc/gitlab-geo/README.md deleted file mode 100644 index 67e919fc136..00000000000 --- a/doc/gitlab-geo/README.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../administration/geo/index.md' ---- - -This document was moved to [another location](../administration/geo/index.md). diff --git a/doc/gitlab-geo/after_setup.md b/doc/gitlab-geo/after_setup.md deleted file mode 100644 index c8a7b9d1096..00000000000 --- a/doc/gitlab-geo/after_setup.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../administration/geo/replication/using_a_geo_server.md' ---- - -This document was moved to [another location](../administration/geo/replication/using_a_geo_server.md). diff --git a/doc/gitlab-geo/bring-primary-back.md b/doc/gitlab-geo/bring-primary-back.md deleted file mode 100644 index 8c43f4d805f..00000000000 --- a/doc/gitlab-geo/bring-primary-back.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../administration/geo/disaster_recovery/bring_primary_back.md' ---- - -This document was moved to [another location](../administration/geo/disaster_recovery/bring_primary_back.md). diff --git a/doc/gitlab-geo/configuration.md b/doc/gitlab-geo/configuration.md deleted file mode 100644 index b46a2caea4a..00000000000 --- a/doc/gitlab-geo/configuration.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../administration/geo/replication/configuration.md' ---- - -This document was moved to [another location](../administration/geo/replication/configuration.md). diff --git a/doc/gitlab-geo/configuration_source.md b/doc/gitlab-geo/configuration_source.md deleted file mode 100644 index b46a2caea4a..00000000000 --- a/doc/gitlab-geo/configuration_source.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../administration/geo/replication/configuration.md' ---- - -This document was moved to [another location](../administration/geo/replication/configuration.md). diff --git a/doc/gitlab-geo/database.md b/doc/gitlab-geo/database.md deleted file mode 100644 index 2f68068d95b..00000000000 --- a/doc/gitlab-geo/database.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../administration/geo/setup/database.md' ---- - -This document was moved to [another location](../administration/geo/setup/database.md). diff --git a/doc/gitlab-geo/database_source.md b/doc/gitlab-geo/database_source.md deleted file mode 100644 index 2f68068d95b..00000000000 --- a/doc/gitlab-geo/database_source.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../administration/geo/setup/database.md' ---- - -This document was moved to [another location](../administration/geo/setup/database.md). diff --git a/doc/gitlab-geo/disaster-recovery.md b/doc/gitlab-geo/disaster-recovery.md deleted file mode 100644 index d42e815a879..00000000000 --- a/doc/gitlab-geo/disaster-recovery.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../administration/geo/disaster_recovery/index.md' ---- - -This document was moved to [another location](../administration/geo/disaster_recovery/index.md). diff --git a/doc/gitlab-geo/docker_registry.md b/doc/gitlab-geo/docker_registry.md deleted file mode 100644 index 26a708f6845..00000000000 --- a/doc/gitlab-geo/docker_registry.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../administration/geo/replication/docker_registry.md' ---- - -This document was moved to [another location](../administration/geo/replication/docker_registry.md). diff --git a/doc/gitlab-geo/faq.md b/doc/gitlab-geo/faq.md deleted file mode 100644 index f1952bc7e4c..00000000000 --- a/doc/gitlab-geo/faq.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../administration/geo/replication/faq.md' ---- - -This document was moved to [another location](../administration/geo/replication/faq.md). diff --git a/doc/gitlab-geo/ha.md b/doc/gitlab-geo/ha.md deleted file mode 100644 index 0be70791d45..00000000000 --- a/doc/gitlab-geo/ha.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../administration/geo/replication/multiple_servers.md' ---- - -This document was moved to [another location](../administration/geo/replication/multiple_servers.md). diff --git a/doc/gitlab-geo/object_storage.md b/doc/gitlab-geo/object_storage.md deleted file mode 100644 index 1f29b7b7e8c..00000000000 --- a/doc/gitlab-geo/object_storage.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../administration/geo/replication/object_storage.md' ---- - -This document was moved to [another location](../administration/geo/replication/object_storage.md). diff --git a/doc/gitlab-geo/planned-failover.md b/doc/gitlab-geo/planned-failover.md deleted file mode 100644 index 720b6bc9424..00000000000 --- a/doc/gitlab-geo/planned-failover.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../administration/geo/disaster_recovery/planned_failover.md' ---- - -This document was moved to [another location](../administration/geo/disaster_recovery/planned_failover.md). diff --git a/doc/gitlab-geo/security-review.md b/doc/gitlab-geo/security-review.md deleted file mode 100644 index a0a5b0e536c..00000000000 --- a/doc/gitlab-geo/security-review.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../administration/geo/replication/security_review.md' ---- - -This document was moved to [another location](../administration/geo/replication/security_review.md). diff --git a/doc/gitlab-geo/ssh.md b/doc/gitlab-geo/ssh.md deleted file mode 100644 index 4f8db687850..00000000000 --- a/doc/gitlab-geo/ssh.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../administration/operations/fast_ssh_key_lookup.md' ---- - -This document was moved to [another location](../administration/operations/fast_ssh_key_lookup.md). diff --git a/doc/gitlab-geo/troubleshooting.md b/doc/gitlab-geo/troubleshooting.md deleted file mode 100644 index 25fe1372c69..00000000000 --- a/doc/gitlab-geo/troubleshooting.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../administration/geo/replication/troubleshooting.md' ---- - -This document was moved to [another location](../administration/geo/replication/troubleshooting.md). diff --git a/doc/gitlab-geo/tuning.md b/doc/gitlab-geo/tuning.md deleted file mode 100644 index 84ac40f99db..00000000000 --- a/doc/gitlab-geo/tuning.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../administration/geo/replication/tuning.md' ---- - -This document was moved to [another location](../administration/geo/replication/tuning.md). diff --git a/doc/gitlab-geo/updating_the_geo_nodes.md b/doc/gitlab-geo/updating_the_geo_nodes.md deleted file mode 100644 index 28234ec02ed..00000000000 --- a/doc/gitlab-geo/updating_the_geo_nodes.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../administration/geo/replication/updating_the_geo_nodes.md' ---- - -This document was moved to [another location](../administration/geo/replication/updating_the_geo_nodes.md). diff --git a/doc/gitlab-geo/using_a_geo_server.md b/doc/gitlab-geo/using_a_geo_server.md deleted file mode 100644 index c8a7b9d1096..00000000000 --- a/doc/gitlab-geo/using_a_geo_server.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../administration/geo/replication/using_a_geo_server.md' ---- - -This document was moved to [another location](../administration/geo/replication/using_a_geo_server.md). diff --git a/doc/hooks/custom_hooks.md b/doc/hooks/custom_hooks.md deleted file mode 100644 index c6d44bb03e9..00000000000 --- a/doc/hooks/custom_hooks.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -redirect_to: '../administration/server_hooks.md' ---- - -# Custom Git Hooks - -This document was moved to [administration/server_hooks.md](../administration/server_hooks.md). diff --git a/doc/incoming_email/README.md b/doc/incoming_email/README.md deleted file mode 100644 index 9544983974f..00000000000 --- a/doc/incoming_email/README.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../administration/reply_by_email.md' ---- - -This document was moved to [administration/reply_by_email](../administration/reply_by_email.md). diff --git a/doc/incoming_email/postfix.md b/doc/incoming_email/postfix.md deleted file mode 100644 index a7192325229..00000000000 --- a/doc/incoming_email/postfix.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../administration/reply_by_email_postfix_setup.md' ---- - -This document was moved to [administration/reply_by_email_postfix_setup](../administration/reply_by_email_postfix_setup.md). diff --git a/doc/install/README.md b/doc/install/README.md index f1ef368685e..6b08bb28bbb 100644 --- a/doc/install/README.md +++ b/doc/install/README.md @@ -1,4 +1,7 @@ --- +stage: Enablement +group: Distribution +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers comments: false description: Read through the GitLab installation methods. type: index @@ -6,33 +9,42 @@ type: index # Installation **(CORE ONLY)** -GitLab can be installed in most GNU/Linux distributions and in a number -of cloud providers. To get the best experience from GitLab, you need to balance -performance, reliability, ease of administration (backups, upgrades and troubleshooting), -and cost of hosting. - -There are many ways you can install GitLab depending on your platform: - -1. **Omnibus GitLab**: The official deb/rpm packages that contain a bundle of GitLab - and the various components it depends on, like PostgreSQL, Redis, Sidekiq, etc. -1. **GitLab Helm chart**: The cloud native Helm chart for installing GitLab and all - its components on Kubernetes. -1. **Docker**: The Omnibus GitLab packages dockerized. -1. **Source**: Install GitLab and all its components from scratch. - -TIP: **If in doubt, choose Omnibus:** -The Omnibus GitLab packages are mature, -[scalable](../administration/reference_architectures/index.md) and are used +GitLab can be installed in most GNU/Linux distributions and with several +cloud providers. To get the best experience from GitLab, you must balance +performance, reliability, ease of administration (backups, upgrades, and +troubleshooting), and the cost of hosting. + +Depending on your platform, select from the following available methods to +install GitLab: + +- [_Omnibus GitLab_](#installing-gitlab-using-the-omnibus-gitlab-package-recommended): + The official deb/rpm packages that contain a bundle of GitLab and the + components it depends on, including PostgreSQL, Redis, and Sidekiq. +- [_GitLab Helm chart_](#installing-gitlab-on-kubernetes-via-the-gitlab-helm-charts): + The cloud native Helm chart for installing GitLab and all of its components + on Kubernetes. +- [_Docker_](#installing-gitlab-with-docker): The Omnibus GitLab packages, + dockerized. +- [_Source_](#installing-gitlab-from-source): Install GitLab and all of its + components from scratch. +- [_Cloud provider_](#installing-gitlab-on-cloud-providers): Install directly + from platforms like AWS, Azure, and GCP. + +If you're not sure which installation method to use, we recommend you use +Omnibus GitLab. The Omnibus GitLab packages are mature, +[scalable](../administration/reference_architectures/index.md), and are used today on GitLab.com. The Helm charts are recommended for those who are familiar with Kubernetes. ## Requirements -Before installing GitLab, it is of critical importance to review the system [requirements](requirements.md). The system requirements include details on the minimum hardware, software, database, and additional requirements to support GitLab. +Before you install GitLab, be sure to review the [system requirements](requirements.md). +The system requirements include details about the minimum hardware, software, +database, and additional requirements to support GitLab. ## Installing GitLab using the Omnibus GitLab package (recommended) -The Omnibus GitLab package uses our official deb/rpm repositories. This is +The Omnibus GitLab package uses our official deb/rpm repositories, and is recommended for most users. If you need additional flexibility and resilience, we recommend deploying @@ -42,11 +54,6 @@ GitLab as described in our [reference architecture documentation](../administrat ## Installing GitLab on Kubernetes via the GitLab Helm charts -NOTE: **Kubernetes experience required:** -We recommend being familiar with Kubernetes before using it to deploy GitLab in -production. The methods for management, observability, and some concepts are -different than traditional deployments. - When installing GitLab on Kubernetes, there are some trade-offs that you need to be aware of: @@ -56,11 +63,17 @@ need to be aware of: are deployed in a redundant fashion. - There are some feature [limitations to be aware of](https://docs.gitlab.com/charts/#limitations). +Due to these trade-offs, having Kubernetes experience is a requirement for +using this method. We recommend being familiar with Kubernetes before using it +to deploy GitLab in production. The methods for management, observability, and +some concepts are different than traditional deployments. + [**> Install GitLab on Kubernetes using the GitLab Helm charts.**](https://docs.gitlab.com/charts/) ## Installing GitLab with Docker -GitLab maintains a set of official Docker images based on the Omnibus GitLab package. +GitLab maintains a set of official Docker images based on the Omnibus GitLab +package. [**> Install GitLab using the official GitLab Docker images.**](docker.md) @@ -68,7 +81,7 @@ GitLab maintains a set of official Docker images based on the Omnibus GitLab pac If the Omnibus GitLab package is not available in your distribution, you can install GitLab from source: Useful for unsupported systems like \*BSD. For an -overview of the directory structure, read the [structure documentation](structure.md). +overview of the directory structure, read the [structure documentation](installation.md#gitlab-directory-structure). [**> Install GitLab from source.**](installation.md) @@ -86,6 +99,37 @@ the above methods, provided the cloud provider supports it. - _Testing only!_ [DigitalOcean and Docker Machine](digitaloceandocker.md): Quickly test any version of GitLab on DigitalOcean using Docker Machine. -## Securing your GitLab installation - -After completing your installation, check out our [recommended practices to secure your GitLab instance](../security/README.md#securing-your-gitlab-installation). +## Next steps + +Here are a few resources you might want to check out after completing the +installation: + +- [Upload a license](../user/admin_area/license.md) or [start a free trial](https://about.gitlab.com/free-trial/): + Activate all GitLab Enterprise Edition functionality with a license. +- [Set up runners](https://docs.gitlab.com/runner/): Set up one or more GitLab + Runners, the agents that are responsible for all of GitLab's CI/CD features. +- [GitLab Pages](../administration/pages/index.md): Configure GitLab Pages to + allow hosting of static sites. +- [GitLab Registry](../administration/packages/container_registry.md): With the + GitLab Container Registry, every project can have its own space to store Docker + images. +- [Secure GitLab](../security/README.md#securing-your-gitlab-installation): + Recommended practices to secure your GitLab instance. +- [SMTP](https://docs.gitlab.com/omnibus/settings/smtp.html): Configure SMTP + for proper email notifications support. +- [LDAP](../administration/auth/ldap/index.md): Configure LDAP to be used as + an authentication mechanism for GitLab. +- [Back up and restore GitLab](../raketasks/backup_restore.md): Learn the different + ways you can back up or restore GitLab. +- [Upgrade GitLab](../update/README.md): Every 22nd of the month, a new feature-rich GitLab version + is released. Learn how to upgrade to it, or to an interim release that contains a security fix. +- [Scaling GitLab](../administration/reference_architectures/index.md): + GitLab supports several different types of clustering. +- [Advanced Search](../integration/elasticsearch.md): Leverage Elasticsearch for + faster, more advanced code search across your entire GitLab instance. +- [Geo replication](../administration/geo/index.md): + Geo is the solution for widely distributed development teams. +- [Release and maintenance policy](../policy/maintenance.md): Learn about GitLab's + policies governing version naming, as well as release pace for major, minor, patch, + and security releases. +- [Pricing](https://about.gitlab.com/pricing/): Pricing for the different tiers. diff --git a/doc/install/aws/index.md b/doc/install/aws/index.md index aba76ecf50e..93ac17d823b 100644 --- a/doc/install/aws/index.md +++ b/doc/install/aws/index.md @@ -1,4 +1,7 @@ --- +stage: Enablement +group: Distribution +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: howto --- @@ -463,7 +466,7 @@ Connect to your GitLab instance via **Bastion Host A** using [SSH Agent Forwardi #### Disable Let's Encrypt -Since we're adding our SSL certificate at the load balancer, we do not need GitLab's built-in support for Let's Encrypt. Let's Encrypt [is enabled by default](https://docs.gitlab.com/omnibus/settings/ssl.html#lets-encrypt-integration) when using an `https` domain since GitLab 10.7, so we need to explicitly disable it: +Since we're adding our SSL certificate at the load balancer, we do not need GitLab's built-in support for Let's Encrypt. Let's Encrypt [is enabled by default](https://docs.gitlab.com/omnibus/settings/ssl.html#lets-encrypt-integration) when using an `https` domain in GitLab 10.7 and later, so we need to explicitly disable it: 1. Open `/etc/gitlab/gitlab.rb` and disable it: diff --git a/doc/install/azure/index.md b/doc/install/azure/index.md index b6e3025a0e0..b5e72a9b84a 100644 --- a/doc/install/azure/index.md +++ b/doc/install/azure/index.md @@ -422,7 +422,7 @@ on any cloud service you choose. ## Where to next? -Check out our other [Technical Articles](../../articles/index.md) or browse the [GitLab Documentation](../../README.md) to learn more about GitLab. +Check out our other [Technical Articles](../../topics/index.md) or browse the [GitLab Documentation](../../README.md) to learn more about GitLab. ### Useful links diff --git a/doc/install/docker.md b/doc/install/docker.md index c2d7655d526..ca780caa563 100644 --- a/doc/install/docker.md +++ b/doc/install/docker.md @@ -1,4 +1,7 @@ --- +stage: Enablement +group: Distribution +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: index --- diff --git a/doc/install/google_cloud_platform/index.md b/doc/install/google_cloud_platform/index.md index 8ca5c5c266a..da2b30df476 100644 --- a/doc/install/google_cloud_platform/index.md +++ b/doc/install/google_cloud_platform/index.md @@ -1,4 +1,7 @@ --- +stage: Enablement +group: Distribution +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers description: 'Learn how to install a GitLab instance on Google Cloud Platform.' type: howto --- diff --git a/doc/install/installation.md b/doc/install/installation.md index e2c77073983..0adf09595e4 100644 --- a/doc/install/installation.md +++ b/doc/install/installation.md @@ -1,4 +1,7 @@ --- +stage: Enablement +group: Distribution +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: howto --- @@ -66,7 +69,6 @@ of this page: maintained for all projects. **This area contains critical data for projects. [Keep a backup](../raketasks/backup_restore.md).** -NOTE: **Note:** The default locations for repositories can be configured in `config/gitlab.yml` of GitLab and `config.yml` of GitLab Shell. @@ -98,8 +100,9 @@ apt-get upgrade -y apt-get install sudo -y ``` -NOTE: **Note:** -During this installation, some files need to be edited manually. If you are familiar with vim, set it as default editor with the commands below. If you are not familiar with vim, skip this and keep using the default editor. +During this installation, some files need to be edited manually. If you are familiar +with vim, set it as default editor with the commands below. If you are not familiar +with vim, skip this and keep using the default editor: ```shell # Install vim and set as default editor @@ -119,15 +122,13 @@ sudo apt-get install -y build-essential zlib1g-dev libyaml-dev libssl-dev libgdb Ubuntu 14.04 (Trusty Tahr) doesn't have the `libre2-dev` package available, but you can [install re2 manually](https://github.com/google/re2/wiki/Install). -If you want to use Kerberos for user authentication, install `libkrb5-dev`: +If you want to use Kerberos for user authentication, install `libkrb5-dev` +(if you don't know what Kerberos is, you can assume you don't need it): ```shell sudo apt-get install libkrb5-dev ``` -NOTE: **Note:** -If you don't know what Kerberos is, you can assume you don't need it. - Make sure you have the right version of Git installed: ```shell @@ -199,8 +200,11 @@ needs to be installed. sudo apt-get install -y graphicsmagick ``` -NOTE: **Note:** -In order to receive mail notifications, make sure to install a mail server. By default, Debian is shipped with exim4 but this [has problems](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/12754) while Ubuntu does not ship with one. The recommended mail server is postfix and you can install it with: +In order to receive mail notifications, make sure to install a mail server. +By default, Debian is shipped with `exim4` but this +[has problems](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/12754) while +Ubuntu does not ship with one. The recommended mail server is `postfix` and you +can install it with: ```shell sudo apt-get install -y postfix @@ -218,10 +222,8 @@ sudo apt-get install -y libimage-exiftool-perl ## 2. Ruby The Ruby interpreter is required to run GitLab. - -NOTE: **Note:** -The current supported Ruby (MRI) version is 2.6.x. GitLab 12.2 -dropped support for Ruby 2.5.x. +See the [requirements page](requirements.md#ruby-versions) for the minimum +Ruby requirements. The use of Ruby version managers such as [RVM](https://rvm.io/), [rbenv](https://github.com/rbenv/rbenv) or [chruby](https://github.com/postmodern/chruby) with GitLab in production, frequently leads to hard to diagnose problems. Version managers @@ -258,7 +260,7 @@ sudo gem install bundler --no-document --version '< 2' ## 3. Go -Since GitLab 8.0, GitLab has several daemons written in Go. To install +In GitLab 8.0 and later, GitLab has several daemons written in Go. To install GitLab we need a Go compiler. The instructions below assume you use 64-bit Linux. You can find downloads for other platforms at the [Go download page](https://golang.org/dl). @@ -276,7 +278,7 @@ rm go1.13.5.linux-amd64.tar.gz ## 4. Node -Since GitLab 8.17, GitLab requires the use of Node to compile JavaScript +In GitLab 8.17 and later, GitLab requires the use of Node to compile JavaScript assets, and Yarn to manage JavaScript dependencies. The current minimum requirements for these are: @@ -311,14 +313,14 @@ sudo adduser --disabled-login --gecos 'GitLab' git ## 6. Database NOTE: **Note:** -Starting from GitLab 12.1, only PostgreSQL is supported. Since GitLab 13.0, we [require PostgreSQL 11+](requirements.md#postgresql-requirements). +In GitLab 12.1 and later, only PostgreSQL is supported. In GitLab 13.0 and later, we [require PostgreSQL 11+](requirements.md#postgresql-requirements). 1. Install the database packages: ```shell sudo apt-get install -y postgresql postgresql-client libpq-dev postgresql-contrib ``` - + 1. Verify the PostgreSQL version you have is supported by the version of GitLab you're installing: @@ -407,7 +409,6 @@ Starting from GitLab 12.1, only PostgreSQL is supported. Since GitLab 13.0, we [ ## 7. Redis -NOTE: **Note:** See the [requirements page](requirements.md#redis-versions) for the minimum Redis requirements. @@ -552,11 +553,9 @@ sudo -u git -H cp config/resque.yml.example config/resque.yml sudo -u git -H editor config/resque.yml ``` -CAUTION: **Caution:** Make sure to edit both `gitlab.yml` and `puma.rb` to match your setup. If you want to use the Unicorn web server, see [Using Unicorn](#using-unicorn) for the additional steps. -NOTE: **Note:** If you want to use HTTPS, see [Using HTTPS](#using-https) for the additional steps. ### Configure GitLab DB Settings @@ -596,13 +595,13 @@ Make sure you have `bundle` (run `bundle -v`): - `>= 1.5.2`, because some [issues](https://devcenter.heroku.com/changelog-items/411) were [fixed](https://github.com/rubygems/bundler/pull/2817) in 1.5.2. - `< 2.x`. +Install the gems (if you want to use Kerberos for user authentication, omit +`kerberos` in the `--without` option below): + ```shell sudo -u git -H bundle install --deployment --without development test mysql aws kerberos ``` -NOTE: **Note:** -If you want to use Kerberos for user authentication, omit `kerberos` in the `--without` option above. - ### Install GitLab Shell GitLab Shell is an SSH access and repository management software developed specially for GitLab. @@ -616,10 +615,8 @@ sudo -u git -H bundle exec rake gitlab:shell:install RAILS_ENV=production sudo -u git -H editor /home/git/gitlab-shell/config.yml ``` -NOTE: **Note:** If you want to use HTTPS, see [Using HTTPS](#using-https) for the additional steps. -NOTE: **Note:** Make sure your hostname can be resolved on the machine itself by either a proper DNS record or an additional line in `/etc/hosts` ("127.0.0.1 hostname"). This might be necessary, for example, if you set up GitLab behind a reverse proxy. If the hostname cannot be resolved, the final installation check fails with `Check GitLab API access: FAILED. code: 401` and pushing commits are rejected with `[remote rejected] master -> master (hook declined)`. ### Install GitLab Workhorse @@ -638,7 +635,7 @@ You can specify a different Git repository by providing it as an extra parameter sudo -u git -H bundle exec rake "gitlab:workhorse:install[/home/git/gitlab-workhorse,https://example.com/gitlab-workhorse.git]" RAILS_ENV=production ``` -### Install GitLab-Elasticsearch-indexer on Enterprise Edition +### Install GitLab-Elasticsearch-indexer on Enterprise Edition **(STARTER ONLY)** GitLab-Elasticsearch-Indexer uses [GNU Make](https://www.gnu.org/software/make/). The following command-line installs GitLab-Elasticsearch-Indexer in `/home/git/gitlab-elasticsearch-indexer` @@ -657,9 +654,6 @@ sudo -u git -H bundle exec rake "gitlab:indexer:install[/home/git/gitlab-elastic The source code first is fetched to the path specified by the first parameter. Then a binary is built under its `bin` directory. You then need to update `gitlab.yml`'s `production -> elasticsearch -> indexer_path` setting to point to that binary. -NOTE: **Note:** -Elasticsearch is a feature of GitLab Enterprise Edition and isn't included in GitLab Community Edition. - ### Install GitLab Pages GitLab Pages uses [GNU Make](https://www.gnu.org/software/make/). This step is optional and only needed if you wish to host static sites from within GitLab. The following commands install GitLab Pages in `/home/git/gitlab-pages`. For additional setup steps, consult the [administration guide](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/administration/pages/source.md) for your version of GitLab as the GitLab Pages daemon can be run several different ways. @@ -726,8 +720,7 @@ sudo -u git -H bundle exec rake gitlab:setup RAILS_ENV=production force=yes # When done, you see 'Administrator account created:' ``` -NOTE: **Note:** -You can set the Administrator/root password and e-mail by supplying them in environmental variables, `GITLAB_ROOT_PASSWORD` and `GITLAB_ROOT_EMAIL` respectively, as seen below. If you don't set the password (and it is set to the default one), wait to expose GitLab to the public internet until the installation is done and you've logged into the server the first time. During the first login, you'll be forced to change the default password. An Enterprise Edition license may also be installed at this time by supplying a full path in the `GITLAB_LICENSE_FILE` environment variable. +You can set the Administrator/root password and email by supplying them in environmental variables, `GITLAB_ROOT_PASSWORD` and `GITLAB_ROOT_EMAIL` respectively, as seen below. If you don't set the password (and it is set to the default one), wait to expose GitLab to the public internet until the installation is done and you've logged into the server the first time. During the first login, you'll be forced to change the default password. An Enterprise Edition license may also be installed at this time by supplying a full path in the `GITLAB_LICENSE_FILE` environment variable. ```shell sudo -u git -H bundle exec rake gitlab:setup RAILS_ENV=production GITLAB_ROOT_PASSWORD=yourpassword GITLAB_ROOT_EMAIL=youremail GITLAB_LICENSE_FILE="/path/to/license" @@ -804,7 +797,6 @@ sudo /etc/init.d/gitlab restart ## 9. NGINX -NOTE: **Note:** NGINX is the officially supported web server for GitLab. If you cannot or do not want to use NGINX as your web server, see [GitLab recipes](https://gitlab.com/gitlab-org/gitlab-recipes/). ### Installation @@ -841,7 +833,6 @@ If you intend to enable GitLab Pages, there is a separate NGINX config you need to use. Read all about the needed configuration at the [GitLab Pages administration guide](../administration/pages/index.md). -NOTE: **Note:** If you want to use HTTPS, replace the `gitlab` NGINX config with `gitlab-ssl`. See [Using HTTPS](#using-https) for HTTPS configuration details. ### Test Configuration @@ -854,10 +845,18 @@ sudo nginx -t You should receive `syntax is okay` and `test is successful` messages. If you receive errors check your `gitlab` or `gitlab-ssl` NGINX config file for typos, etc. as indicated in the error message given. -NOTE: **Note:** -Verify that the installed version is greater than 1.12.1 by running `nginx -v`. If it's lower, you may receive the error below: -`nginx: [emerg] unknown "start$temp=[filtered]$rest" variable -nginx: configuration file /etc/nginx/nginx.conf test failed` +Verify that the installed version is greater than 1.12.1: + +```shell +nginx -v +``` + +If it's lower, you may receive the error below: + +```plaintext +nginx: [emerg] unknown "start$temp=[filtered]$rest" variable +nginx: configuration file /etc/nginx/nginx.conf test failed +``` ### Restart @@ -877,7 +876,8 @@ sudo -u git -H bundle exec rake gitlab:check RAILS_ENV=production If all items are green, congratulations on successfully installing GitLab! -NOTE: Supply `SANITIZE=true` environment variable to `gitlab:check` to omit project names from the output of the check command. +TIP: **Tip:** +Supply the `SANITIZE=true` environment variable to `gitlab:check` to omit project names from the output of the check command. ### Initial Login diff --git a/doc/install/openshift_and_gitlab/index.md b/doc/install/openshift_and_gitlab/index.md index 022e2095c68..7ac5aa6667a 100644 --- a/doc/install/openshift_and_gitlab/index.md +++ b/doc/install/openshift_and_gitlab/index.md @@ -1,6 +1,8 @@ --- +stage: Enablement +group: Distribution +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: howto -date: 2016-06-28 --- # How to install GitLab on OpenShift Origin 3 @@ -372,7 +374,7 @@ running scaled to 2. Upping the GitLab pods is actually like adding new application servers to your cluster. You can see how that would work if you didn't use GitLab with -OpenShift by following the [HA documentation](../../administration/high_availability/gitlab.md) for the application servers. +OpenShift by following the [HA documentation](../../administration/reference_architectures/index.md) for the application servers. Bare in mind that you may need more resources (CPU, RAM, disk space) when you scale up. If a pod is in pending state for too long, you can navigate to diff --git a/doc/install/relative_url.md b/doc/install/relative_url.md index 09069a2d1fd..82cf134f968 100644 --- a/doc/install/relative_url.md +++ b/doc/install/relative_url.md @@ -1,4 +1,7 @@ --- +stage: Enablement +group: Distribution +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference --- diff --git a/doc/install/requirements.md b/doc/install/requirements.md index da0128fecc3..bc320bcd335 100644 --- a/doc/install/requirements.md +++ b/doc/install/requirements.md @@ -1,4 +1,7 @@ --- +stage: Enablement +group: Distribution +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference --- diff --git a/doc/install/structure.md b/doc/install/structure.md index 87ef11c60fe..ca90a3de1b8 100644 --- a/doc/install/structure.md +++ b/doc/install/structure.md @@ -2,4 +2,4 @@ redirect_to: installation.md#gitlab-directory-structure --- -This page was moved to [another location](#installation.md#gitlab-directory-structure). +This page was moved to [another location](installation.md#gitlab-directory-structure). diff --git a/doc/integration/README.md b/doc/integration/README.md index c5c21644d1c..c8ce367e99f 100644 --- a/doc/integration/README.md +++ b/doc/integration/README.md @@ -8,15 +8,8 @@ GitLab can be integrated with external services for enhanced functionality. ## Issue trackers -You can use an [external issue tracker](external-issue-tracker.md) at the same time as the GitLab issue tracker, or use only the external issue tracker. - -GitLab can be integrated with the following external issue trackers: - -- Jira -- Redmine -- Bugzilla -- EWM -- YouTrack +You can use an [external issue tracker](external-issue-tracker.md) at the same time as the GitLab +issue tracker, or use only the external issue tracker. ## Authentication sources @@ -26,10 +19,10 @@ GitLab can be configured to authenticate access requests with the following auth - Enable sign in with [Bitbucket](bitbucket.md) accounts. - Configure GitLab to sign in using [CAS](cas.md). - Integrate with [Kerberos](kerberos.md). -- Enable sign in via [LDAP](ldap.md). +- Enable sign in via [LDAP](../administration/auth/ldap/index.md). - Enable [OAuth2 provider](oauth_provider.md) application creation. - Use [OmniAuth](omniauth.md) to enable sign in via Twitter, GitHub, GitLab.com, Google, -Bitbucket, Facebook, Shibboleth, SAML, Crowd, Azure or Authentiq ID. + Bitbucket, Facebook, Shibboleth, SAML, Crowd, Azure or Authentiq ID. - Use GitLab as an [OpenID Connect](openid_connect_provider.md) identity provider. - Authenticate to [Vault](vault.md) through GitLab OpenID Connect. - Configure GitLab as a [SAML](saml.md) 2.0 Service Provider. diff --git a/doc/integration/cas.md b/doc/integration/cas.md index cf4e501e772..eee801350eb 100644 --- a/doc/integration/cas.md +++ b/doc/integration/cas.md @@ -45,10 +45,10 @@ To enable the CAS OmniAuth provider you must register your application with your - { name: 'cas3', label: 'cas', args: { - url: 'CAS_SERVER', - login_url: '/CAS_PATH/login', - service_validate_url: '/CAS_PATH/p3/serviceValidate', - logout_url: '/CAS_PATH/logout'} } + url: 'CAS_SERVER', + login_url: '/CAS_PATH/login', + service_validate_url: '/CAS_PATH/p3/serviceValidate', + logout_url: '/CAS_PATH/logout' } } ``` 1. Change 'CAS_PATH' to the root of your CAS instance (ie. `cas`). diff --git a/doc/integration/elasticsearch.md b/doc/integration/elasticsearch.md index f40955ad8ff..a88f2db5c26 100644 --- a/doc/integration/elasticsearch.md +++ b/doc/integration/elasticsearch.md @@ -10,8 +10,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/109 "Elasticsearch Merge Request") in GitLab [Starter](https://about.gitlab.com/pricing/) 8.4. > - Support for [Amazon Elasticsearch](https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/es-gsg.html) was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/1305) in GitLab [Starter](https://about.gitlab.com/pricing/) 9.0. -This document describes how to set up Elasticsearch with GitLab. After -Elasticsearch is enabled, you'll have the benefit of fast search response times +This document describes how to enable Advanced Search. After +Advanced Search is enabled, you'll have the benefit of fast search response times and the advantage of the following special searches: - [Advanced Search](../user/search/advanced_global_search.md) @@ -66,21 +66,19 @@ source. You must [install it separately](https://www.elastic.co/guide/en/elastic Be sure to select your version. Providing detailed information on installing Elasticsearch is out of the scope of this document. -NOTE: **Note:** Elasticsearch should be installed on a separate server, whether you install it yourself or use a cloud hosted offering like Elastic's [Elasticsearch Service](https://www.elastic.co/elasticsearch/service) (available on AWS, GCP, or Azure) or the [Amazon Elasticsearch](https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/es-gsg.html) service. Running Elasticsearch on the same server as GitLab is not recommended and can cause a degradation in GitLab instance performance. -NOTE: **Note:** **For a single node Elasticsearch cluster the functional cluster health status will be yellow** (will never be green) because the primary shard is allocated but replicas can not be as there is no other node to which Elasticsearch can assign a replica. After the data is added to the database or repository and [Elasticsearch is -enabled in the Admin Area](#enabling-elasticsearch) the search index will be +enabled in the Admin Area](#enabling-advanced-search) the search index will be updated automatically. ## Elasticsearch repository indexer @@ -155,15 +153,19 @@ Example: PREFIX=/usr sudo -E make install ``` -After installation, be sure to [enable Elasticsearch](#enabling-elasticsearch). - -## Enabling Elasticsearch +After installation, be sure to [enable Elasticsearch](#enabling-advanced-search). NOTE: **Note:** +If you see an error such as `Permission denied - /home/git/gitlab-elasticsearch-indexer/` while indexing, you +may need to set the `production -> elasticsearch -> indexer_path` setting in your `gitlab.yml` file to +`/usr/local/bin/gitlab-elasticsearch-indexer`, which is where the binary is installed. + +## Enabling Advanced Search + For GitLab instances with more than 50GB repository data you can follow the instructions for [Indexing large instances](#indexing-large-instances) below. -To enable Elasticsearch, you need to have admin access to GitLab: +To enable Advanced Search, you need to have admin access to GitLab: 1. Navigate to **Admin Area** (wrench icon), then **Settings > General** and expand the **Advanced Search** section. @@ -172,23 +174,12 @@ To enable Elasticsearch, you need to have admin access to GitLab: To see the Advanced Search section, you need an active Starter [license](../user/admin_area/license.md). -1. Configure the [Elasticsearch settings](#elasticsearch-configuration) for - your Elasticsearch cluster. Do not enable **Elasticsearch indexing** or - **Search with Elasticsearch enabled** yet. -1. Click **Save changes** for the changes to take effect. -1. Before enabling **Elasticsearch indexing** you need to create an index by - running the Rake task: - - ```shell - # Omnibus installations - sudo gitlab-rake gitlab:elastic:create_empty_index - - # Installations from source - bundle exec rake gitlab:elastic:create_empty_index RAILS_ENV=production - ``` - -1. Now enable `Elasticsearch indexing` in **Admin Area > Settings > - General > Advanced Search** and click **Save changes**. +1. Configure the [Advanced Search settings](#advanced-search-configuration) for + your Elasticsearch cluster. Do not enable **Search with Elasticsearch enabled** + yet. +1. Now enable **Elasticsearch indexing** in **Admin Area > Settings > + General > Advanced Search** and click **Save changes**. This will create + an empty index if one does not already exist. 1. Click **Index all projects**. 1. Click **Check progress** in the confirmation message to see the status of the background jobs. @@ -206,13 +197,13 @@ To enable Elasticsearch, you need to have admin access to GitLab: **Admin Area > Settings > General > Advanced Search** and click **Save changes**. -### Elasticsearch configuration +### Advanced Search configuration The following Elasticsearch settings are available: | Parameter | Description | |-------------------------------------------------------|-------------| -| `Elasticsearch indexing` | Enables or disables Elasticsearch indexing. You may want to enable indexing but disable search in order to give the index time to be fully completed, for example. Also, keep in mind that this option doesn't have any impact on existing data, this only enables/disables the background indexer which tracks data changes and ensures new data is indexed. | +| `Elasticsearch indexing` | Enables or disables Elasticsearch indexing and creates an empty index if one does not already exist. You may want to enable indexing but disable search in order to give the index time to be fully completed, for example. Also, keep in mind that this option doesn't have any impact on existing data, this only enables/disables the background indexer which tracks data changes and ensures new data is indexed. | | `Pause Elasticsearch indexing` | Enables or disables temporary indexing pause. This is useful for cluster migration/reindexing. All changes are still tracked, but they are not committed to the Elasticsearch index until unpaused. | | `Search with Elasticsearch enabled` | Enables or disables using Elasticsearch in search. | | `URL` | The URL to use for connecting to Elasticsearch. Use a comma-separated list to support clustering (e.g., `http://host1, https://host2:9200`). If your Elasticsearch instance is password protected, pass the `username:password` in the URL (e.g., `http://<username>:<password>@<elastic_host>:9200/`). | @@ -238,14 +229,14 @@ If you select `Limit namespaces and projects that can be indexed`, more options You can select namespaces and projects to index exclusively. Note that if the namespace is a group it will include any sub-groups and projects belonging to those sub-groups to be indexed as well. -Elasticsearch only provides cross-group code/commit search (global) if all name-spaces are indexed. In this particular scenario where only a subset of namespaces are indexed, a global search will not provide a code or commit scope. This will be possible only in the scope of an indexed namespace. Currently there is no way to code/commit search in multiple indexed namespaces (when only a subset of namespaces has been indexed). For example if two groups are indexed, there is no way to run a single code search on both. You can only run a code search on the first group and then on the second. +Advanced Search only provides cross-group code/commit search (global) if all name-spaces are indexed. In this particular scenario where only a subset of namespaces are indexed, a global search will not provide a code or commit scope. This will be possible only in the scope of an indexed namespace. Currently there is no way to code/commit search in multiple indexed namespaces (when only a subset of namespaces has been indexed). For example if two groups are indexed, there is no way to run a single code search on both. You can only run a code search on the first group and then on the second. You can filter the selection dropdown by writing part of the namespace or project name you're interested in.  NOTE: **Note:** -If no namespaces or projects are selected, no Elasticsearch indexing will take place. +If no namespaces or projects are selected, no Advanced Search indexing will take place. CAUTION: **Warning:** If you have already indexed your instance, you will have to regenerate the index in order to delete all existing data @@ -253,7 +244,7 @@ for filtering to work correctly. To do this run the Rake tasks `gitlab:elastic:r `gitlab:elastic:clear_index_status`. Afterwards, removing a namespace or a project from the list will delete the data from the Elasticsearch index as expected. -## Disabling Elasticsearch +## Disabling Advanced Search To disable the Elasticsearch integration: @@ -283,7 +274,7 @@ alias such as we can change the underlying index at will. NOTE: **Note:** Any index attached to the production alias is deemed a `primary` and will be -used by the GitLab Elasticsearch integration. +used by the GitLab Advanced Search integration. ### Pause the indexing @@ -315,7 +306,6 @@ CAUTION: **Caution:** It is highly recommended that you take a snapshot of your cluster to ensure there is a recovery path if anything goes wrong. -NOTE: **Note:** Due to a technical limitation, there will be a slight downtime because of the fact that we need to reclaim the current `primary` index to be used as the alias. @@ -413,7 +403,6 @@ To trigger the re-index from `primary` index: Under **Admin Area > Settings > General > Advanced Search > Elasticsearch zero-downtime reindexing**, click on **Trigger cluster reindexing**. -NOTE: **Note:** Reindexing can be a lengthy process depending on the size of your Elasticsearch cluster. CAUTION: **Caution:** @@ -421,12 +410,12 @@ After the reindexing is completed, the original index will be scheduled to be de While the reindexing is running, you will be able to follow its progress under that same section. -## GitLab Elasticsearch Rake tasks +## GitLab Advanced Search Rake tasks Rake tasks are available to: - [Build and install](#building-and-installing) the indexer. -- Delete indexes when [disabling Elasticsearch](#disabling-elasticsearch). +- Delete indexes when [disabling Elasticsearch](#disabling-advanced-search). - Add GitLab data to an index. The following are some available Rake tasks: @@ -436,7 +425,7 @@ The following are some available Rake tasks: | [`sudo gitlab-rake gitlab:elastic:index`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Enables Elasticsearch indexing and run `gitlab:elastic:create_empty_index`, `gitlab:elastic:clear_index_status`, `gitlab:elastic:index_projects`, and `gitlab:elastic:index_snippets`. | | [`sudo gitlab-rake gitlab:elastic:index_projects`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Iterates over all projects and queues Sidekiq jobs to index them in the background. | | [`sudo gitlab-rake gitlab:elastic:index_projects_status`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Determines the overall status of the indexing. It is done by counting the total number of indexed projects, dividing by a count of the total number of projects, then multiplying by 100. | -| [`sudo gitlab-rake gitlab:elastic:clear_index_status`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Deletes all instances of IndexStatus for all projects. | +| [`sudo gitlab-rake gitlab:elastic:clear_index_status`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Deletes all instances of IndexStatus for all projects. Note that this command will result in a complete wipe of the index, and it should be used with caution. | | [`sudo gitlab-rake gitlab:elastic:create_empty_index[<TARGET_NAME>]`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Generates an empty index and assigns an alias for it on the Elasticsearch side only if it doesn't already exist. | | [`sudo gitlab-rake gitlab:elastic:delete_index[<TARGET_NAME>]`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Removes the GitLab index and alias (if exists) on the Elasticsearch instance. | | [`sudo gitlab-rake gitlab:elastic:recreate_index[<TARGET_NAME>]`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Wrapper task for `gitlab:elastic:delete_index[<TARGET_NAME>]` and `gitlab:elastic:create_empty_index[<TARGET_NAME>]`. | @@ -467,7 +456,7 @@ Indexing project repositories...I, [2019-03-04T21:27:03.083410 #3384] INFO -- : I, [2019-03-04T21:27:05.215266 #3384] INFO -- : Indexing GitLab User / test (ID=33) is done! ``` -## Elasticsearch index scopes +## Advanced Search index scopes When performing a search, the GitLab index will use the following scopes: @@ -499,7 +488,7 @@ For basic guidance on choosing a cluster configuration you may refer to [Elastic - `refresh_interval` is a per index setting. You may want to adjust that from default `1s` to a bigger value if you don't need data in realtime. This will change how soon you will see fresh results. If that's important for you, you should leave it as close as possible to the default value. - You might want to raise [`indices.memory.index_buffer_size`](https://www.elastic.co/guide/en/elasticsearch/reference/current/indexing-buffer.html) to 30% or 40% if you have a lot of heavy indexing operations. -### Elasticsearch integration settings guidance +### Advanced Search integration settings guidance - The `Number of Elasticsearch shards` setting usually corresponds with the number of CPUs available in your cluster. For example, if you have a 3-node cluster with 4 cores each, this means you will benefit from having at least 3*4=12 shards in the cluster. Please note, it's only possible to change the shards number by using [Split index API](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-split-index.html) or by reindexing to a different index with a changed number of shards. - The `Number of Elasticsearch replicas` setting should most of the time be equal to `1` (each shard will have 1 replica). Using `0` is not recommended, because losing one node will corrupt the index. @@ -507,7 +496,7 @@ For basic guidance on choosing a cluster configuration you may refer to [Elastic ### Indexing large instances This section may be helpful in the event that the other -[basic instructions](#enabling-elasticsearch) cause problems +[basic instructions](#enabling-advanced-search) cause problems due to large volumes of data being indexed. CAUTION: **Warning:** @@ -516,7 +505,7 @@ Make sure to prepare for this task by having a [Scalable and Highly Available Setup](../administration/reference_architectures/index.md) or creating [extra Sidekiq processes](../administration/operations/extra_sidekiq_processes.md). -1. [Configure your Elasticsearch host and port](#enabling-elasticsearch). +1. [Configure your Elasticsearch host and port](#enabling-advanced-search). 1. Create empty indexes: ```shell @@ -537,7 +526,7 @@ Sidekiq processes](../administration/operations/extra_sidekiq_processes.md). bundle exec rake gitlab:elastic:clear_index_status RAILS_ENV=production ``` -1. [Enable **Elasticsearch indexing**](#enabling-elasticsearch). +1. [Enable **Elasticsearch indexing**](#enabling-advanced-search). 1. Indexing large Git repositories can take a while. To speed up the process, you can [tune for indexing speed](https://www.elastic.co/guide/en/elasticsearch/reference/current/tune-for-indexing-speed.html#tune-for-indexing-speed): - You can temporarily disable [`refresh`](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-refresh.html), the operation responsible for making changes to an index available to search. @@ -664,7 +653,7 @@ Sidekiq processes](../administration/operations/extra_sidekiq_processes.md). } }' ``` -1. After the indexing has completed, enable [**Search with Elasticsearch enabled**](#enabling-elasticsearch). +1. After the indexing has completed, enable [**Search with Elasticsearch enabled**](#enabling-advanced-search). ### Deleted documents @@ -698,96 +687,114 @@ However, some larger installations may wish to tune the merge policy settings: ## Troubleshooting -### Common issues +Here are some common pitfalls and how to overcome them. + +### How can I verify that my GitLab instance is using Elasticsearch? -Here are some common pitfalls and how to overcome them: +There are a couple of ways to achieve that: -- **How can I verify my GitLab instance is using Elasticsearch?** +- Whenever you perform a search there will be a link on the search results page + in the top right hand corner saying "Advanced search functionality is enabled". + This is always correctly identifying whether the current project/namespace + being searched is using Elasticsearch. - The easiest method is via the rails console (`sudo gitlab-rails console`) by running the following: +- From the admin area under **Settings > General > Elasticsearch** check that the + Advanced Search settings are checked. + + Those same settings there can be obtained from the Rails console if necessary: ```ruby - u = User.find_by_username('your-username') - s = SearchService.new(u, {:search => 'search_term'}) - pp s.search_objects.class.name + ::Gitlab::CurrentSettings.elasticsearch_search? # Whether or not searches will use Elasticsearch + ::Gitlab::CurrentSettings.elasticsearch_indexing? # Whether or not content will be indexed in Elasticsearch + ::Gitlab::CurrentSettings.elasticsearch_limit_indexing? # Whether or not Elasticsearch is limited only to certain projects/namespaces ``` - If you see `"ActiveRecord::Relation"`, you are **not** using Elasticsearch. - - If you see `"Kaminari::PaginatableArray"` you are using Elasticsearch. +- If Elasticsearch is limited to specific namespaces and you need to know if + Elasticsearch is being used for a specific project or namespace, you can use + the Rails console: - NOTE: **Note:** - The above instructions are used to verify that GitLab is using Elasticsearch only when indexing all namespaces. This is not to be used for scenarios that only index a [subset of namespaces](#limiting-namespaces-and-projects). + ```ruby + ::Gitlab::CurrentSettings.search_using_elasticsearch?(scope: Namespace.find_by_full_path("/my-namespace")) + ::Gitlab::CurrentSettings.search_using_elasticsearch?(scope: Project.find_by_full_path("/my-namespace/my-project")) + ``` -- **I updated GitLab and now I can't find anything** +### I updated GitLab and now I can't find anything - We continuously make updates to our indexing strategies and aim to support - newer versions of Elasticsearch. When indexing changes are made, it may - be necessary for you to [reindex](#zero-downtime-reindexing) after updating GitLab. +We continuously make updates to our indexing strategies and aim to support +newer versions of Elasticsearch. When indexing changes are made, it may +be necessary for you to [reindex](#zero-downtime-reindexing) after updating GitLab. -- **I indexed all the repositories but I can't find anything** +### I indexed all the repositories but I can't get any hits for my search term in the UI - Make sure you indexed all the database data [as stated above](#enabling-elasticsearch). +Make sure you indexed all the database data [as stated above](#enabling-advanced-search). - Beyond that, check via the [Elasticsearch Search API](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-search.html) to see if the data shows up on the Elasticsearch side. +If there aren't any results (hits) in the UI search, check if you are seeing the same results via the rails console (`sudo gitlab-rails console`): - If it shows up via the [Elasticsearch Search API](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-search.html), check that it shows up via the rails console (`sudo gitlab-rails console`): +```ruby +u = User.find_by_username('your-username') +s = SearchService.new(u, {:search => 'search_term', :scope => 'blobs'}) +pp s.search_objects.to_a +``` - ```ruby - u = User.find_by_username('your-username') - s = SearchService.new(u, {:search => 'search_term', :scope => 'blobs'}) - pp s.search_objects.to_a - ``` +Beyond that, check via the [Elasticsearch Search API](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-search.html) to see if the data shows up on the Elasticsearch side: - NOTE: **Note:** - The above instructions are not to be used for scenarios that only index a [subset of namespaces](#limiting-namespaces-and-projects). +```shell +curl --request GET <elasticsearch_server_ip>:9200/gitlab-production/_search?q=<search_term> +``` - See [Elasticsearch Index Scopes](#elasticsearch-index-scopes) for more information on searching for specific types of data. +More [complex Elasticsearch API calls](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-filter-context.html) are also possible. -- **I indexed all the repositories but then switched Elasticsearch servers and now I can't find anything** +It is important to understand at which level the problem is manifesting (UI, Rails code, Elasticsearch side) to be able to [troubleshoot further](../administration/troubleshooting/elasticsearch.md#search-results-workflow). - You will need to re-run all the Rake tasks to reindex the database, repositories, and wikis. +NOTE: **Note:** +The above instructions are not to be used for scenarios that only index a [subset of namespaces](#limiting-namespaces-and-projects). -- **The indexing process is taking a very long time** +See [Elasticsearch Index Scopes](#advanced-search-index-scopes) for more information on searching for specific types of data. - The more data present in your GitLab instance, the longer the indexing process takes. +### I indexed all the repositories but then switched Elasticsearch servers and now I can't find anything -- **There are some projects that weren't indexed, but we don't know which ones** +You will need to re-run all the Rake tasks to reindex the database, repositories, and wikis. - You can run `sudo gitlab-rake gitlab:elastic:projects_not_indexed` to display projects that aren't indexed. +### The indexing process is taking a very long time -- **No new data is added to the Elasticsearch index when I push code** +The more data present in your GitLab instance, the longer the indexing process takes. - NOTE: **Note:** - This was [fixed in GitLab 13.2](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/35936) and the Rake task is not available for versions greater than that. +### There are some projects that weren't indexed, but I don't know which ones - When performing the initial indexing of blobs, we lock all projects until the project finishes indexing. It could happen that an error during the process causes one or multiple projects to remain locked. In order to unlock them, run: +You can run `sudo gitlab-rake gitlab:elastic:projects_not_indexed` to display projects that aren't indexed. - ```shell - sudo gitlab-rake gitlab:elastic:clear_locked_projects - ``` +### No new data is added to the Elasticsearch index when I push code -- **"Can't specify parent if no parent field has been configured"** +NOTE: **Note:** +This was [fixed in GitLab 13.2](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/35936) and the Rake task is not available for versions greater than that. - If you enabled Elasticsearch before GitLab 8.12 and have not rebuilt indexes you will get - exception in lots of different cases: +When performing the initial indexing of blobs, we lock all projects until the project finishes indexing. It could happen that an error during the process causes one or multiple projects to remain locked. In order to unlock them, run: - ```plaintext - Elasticsearch::Transport::Transport::Errors::BadRequest([400] { - "error": { - "root_cause": [{ - "type": "illegal_argument_exception", - "reason": "Can't specify parent if no parent field has been configured" - }], - "type": "illegal_argument_exception", - "reason": "Can't specify parent if no parent field has been configured" - }, - "status": 400 - }): - ``` +```shell +sudo gitlab-rake gitlab:elastic:clear_locked_projects +``` - This is because we changed the index mapping in GitLab 8.12 and the old indexes should be removed and built from scratch again, - see details in the [update guide](../update/upgrading_from_source.md). +### `Can't specify parent if no parent field has been configured` error + +If you enabled Elasticsearch before GitLab 8.12 and have not rebuilt indexes you will get +exception in lots of different cases: + +```plaintext +Elasticsearch::Transport::Transport::Errors::BadRequest([400] { + "error": { + "root_cause": [{ + "type": "illegal_argument_exception", + "reason": "Can't specify parent if no parent field has been configured" + }], + "type": "illegal_argument_exception", + "reason": "Can't specify parent if no parent field has been configured" + }, + "status": 400 +}): +``` + +This is because we changed the index mapping in GitLab 8.12 and the old indexes should be removed and built from scratch again, +see details in the [update guide](../update/upgrading_from_source.md). - Exception `Elasticsearch::Transport::Transport::Errors::BadRequest` @@ -809,50 +816,51 @@ Here are some common pitfalls and how to overcome them: for this setting ("Maximum Size of HTTP Request Payloads"), based on the size of the underlying instance. -- **My single node Elasticsearch cluster status never goes from `yellow` to `green` even though everything seems to be running properly** +### My single node Elasticsearch cluster status never goes from `yellow` to `green` even though everything seems to be running properly - **For a single node Elasticsearch cluster the functional cluster health status will be yellow** (will never be green) because the primary shard is allocated but replicas can not be as there is no other node to which Elasticsearch can assign a replica. This also applies if you are using the -[Amazon Elasticsearch](https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/aes-handling-errors.html#aes-handling-errors-yellow-cluster-status) service. +**For a single node Elasticsearch cluster the functional cluster health status will be yellow** (never green) because the primary shard is allocated but replicas cannot be as there is no other node to which Elasticsearch can assign a replica. This also applies if you are using the [Amazon Elasticsearch](https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/aes-handling-errors.html#aes-handling-errors-yellow-cluster-status) service. - CAUTION: **Warning:** - Setting the number of replicas to `0` is not something that we recommend (this is not allowed in the GitLab Elasticsearch Integration menu). If you are planning to add more Elasticsearch nodes (for a total of more than 1 Elasticsearch) the number of replicas will need to be set to an integer value larger than `0`. Failure to do so will result in lack of redundancy (losing one node will corrupt the index). +CAUTION: **Warning:** +Setting the number of replicas to `0` is discouraged (this is not allowed in the GitLab Elasticsearch Integration menu). If you are planning to add more Elasticsearch nodes (for a total of more than 1 Elasticsearch) the number of replicas will need to be set to an integer value larger than `0`. Failure to do so will result in lack of redundancy (losing one node will corrupt the index). - If you have a **hard requirement to have a green status for your single node Elasticsearch cluster**, please make sure you understand the risks outlined in the previous paragraph and then simply run the following query to set the number of replicas to `0`(the cluster will no longer try to create any shard replicas): +If you have a **hard requirement to have a green status for your single node Elasticsearch cluster**, please make sure you understand the risks outlined in the previous paragraph and then run the following query to set the number of replicas to `0`(the cluster will no longer try to create any shard replicas): - ```shell - curl --request PUT localhost:9200/gitlab-production/_settings --header 'Content-Type: application/json' --data '{ - "index" : { - "number_of_replicas" : 0 - } - }' - ``` +```shell +curl --request PUT localhost:9200/gitlab-production/_settings --header 'Content-Type: application/json' --data '{ +"index" : { + "number_of_replicas" : 0 + } +}' +``` -- **I'm getting a `health check timeout: no Elasticsearch node available` error in Sidekiq during the indexing process** +### `health check timeout: no Elasticsearch node available` error in Sidekiq - ```plaintext - Gitlab::Elastic::Indexer::Error: time="2020-01-23T09:13:00Z" level=fatal msg="health check timeout: no Elasticsearch node available" - ``` +If you're getting a `health check timeout: no Elasticsearch node available` error in Sidekiq during the indexing process: + +```plaintext +Gitlab::Elastic::Indexer::Error: time="2020-01-23T09:13:00Z" level=fatal msg="health check timeout: no Elasticsearch node available" +``` - You probably have not used either `http://` or `https://` as part of your value in the **"URL"** field of the Elasticsearch Integration Menu. Please make sure you are using either `http://` or `https://` in this field as the [Elasticsearch client for Go](https://github.com/olivere/elastic) that we are using [needs the prefix for the URL to be accepted as valid](https://github.com/olivere/elastic/commit/a80af35aa41856dc2c986204e2b64eab81ccac3a). - Once you have corrected the formatting of the URL, delete the index (via the [dedicated Rake task](#gitlab-elasticsearch-rake-tasks)) and [reindex the content of your instance](#enabling-elasticsearch). +You probably have not used either `http://` or `https://` as part of your value in the **"URL"** field of the Elasticsearch Integration Menu. Please make sure you are using either `http://` or `https://` in this field as the [Elasticsearch client for Go](https://github.com/olivere/elastic) that we are using [needs the prefix for the URL to be accepted as valid](https://github.com/olivere/elastic/commit/a80af35aa41856dc2c986204e2b64eab81ccac3a). +Once you have corrected the formatting of the URL, delete the index (via the [dedicated Rake task](#gitlab-advanced-search-rake-tasks)) and [reindex the content of your instance](#enabling-advanced-search). ### Low-level troubleshooting There is a [more structured, lower-level troubleshooting document](../administration/troubleshooting/elasticsearch.md) for when you experience other issues, including poor performance. -### Known Issues +### Known issues -- **[Elasticsearch `code_analyzer` doesn't account for all code cases](https://gitlab.com/groups/gitlab-org/-/epics/3621)** +[Elasticsearch `code_analyzer` doesn't account for all code cases](https://gitlab.com/groups/gitlab-org/-/epics/3621). - The `code_analyzer` pattern and filter configuration is being evaluated for improvement. We have fixed [most edge cases](https://gitlab.com/groups/gitlab-org/-/epics/3621#note_363429094) that were not returning expected search results due to our pattern and filter configuration. +The `code_analyzer` pattern and filter configuration is being evaluated for improvement. We have fixed [most edge cases](https://gitlab.com/groups/gitlab-org/-/epics/3621#note_363429094) that were not returning expected search results due to our pattern and filter configuration. - Improvements to the `code_analyzer` pattern and filters is being discussed in [epic 3621](https://gitlab.com/groups/gitlab-org/-/epics/3621). +Improvements to the `code_analyzer` pattern and filters are being discussed in [epic 3621](https://gitlab.com/groups/gitlab-org/-/epics/3621). ### Reverting to Basic Search Sometimes there may be issues with your Elasticsearch index data and as such GitLab will allow you to revert to "basic search" when there are no search results and assuming that basic search is supported in that scope. This "basic -search" will behave as though you don't have Elasticsearch enabled at all for +search" will behave as though you don't have Advanced Search enabled at all for your instance and search using other data sources (ie. PostgreSQL data and Git data). diff --git a/doc/integration/external-issue-tracker.md b/doc/integration/external-issue-tracker.md index cde0093f53e..96c9b9d7f62 100644 --- a/doc/integration/external-issue-tracker.md +++ b/doc/integration/external-issue-tracker.md @@ -1,7 +1,7 @@ # External issue tracker -GitLab has a great [issue tracker](../user/project/issues/index.md) but you can also use an external one -such as Jira, Redmine, YouTrack, Bugzilla, or EWM. External issue trackers are configurable per GitLab project. +GitLab has a great [issue tracker](../user/project/issues/index.md) but you can also use an external +one. External issue trackers are configurable per GitLab project. Once configured, you can reference external issues using the format `CODE-123`, where: @@ -15,24 +15,23 @@ GitLab menu always opens the internal issue tracker. When disabled, the link is ## Configuration -The configuration is done via a project's **Integrations**. +The configuration is done via a project's **Settings > Integrations**. ### Integration To enable an external issue tracker you must configure the appropriate **Integration**. Visit the links below for details: -- [Redmine](../user/project/integrations/redmine.md) -- [YouTrack](../user/project/integrations/youtrack.md) -- [Jira](../user/project/integrations/jira.md) - [Bugzilla](../user/project/integrations/bugzilla.md) -- [EWM](../user/project/integrations/ewm.md) - [Custom Issue Tracker](../user/project/integrations/custom_issue_tracker.md) +- [Engineering Workflow Management](../user/project/integrations/ewm.md) +- [Jira](../user/project/integrations/jira.md) +- [Redmine](../user/project/integrations/redmine.md) +- [YouTrack](../user/project/integrations/youtrack.md) ### Service Template -To save you the hassle from configuring each project's service individually, -GitLab provides the ability to set Service Templates which can then be -overridden in each project's settings. +To avoid configuring each project's service individually, GitLab provides the ability to set +Service Templates. These can then be overridden in each project's settings. Read more on [Services Templates](../user/project/integrations/services_templates.md). diff --git a/doc/integration/facebook.md b/doc/integration/facebook.md index fc65191994d..dbefb560fe7 100644 --- a/doc/integration/facebook.md +++ b/doc/integration/facebook.md @@ -83,7 +83,7 @@ To enable the Facebook OmniAuth provider you must register your application with ```yaml - { name: 'facebook', app_id: 'YOUR_APP_ID', - app_secret: 'YOUR_APP_SECRET' } + app_secret: 'YOUR_APP_SECRET' } ``` 1. Change 'YOUR_APP_ID' to the API key from Facebook page in step 10. diff --git a/doc/integration/github.md b/doc/integration/github.md index ccce4672cbf..ce2b50acc54 100644 --- a/doc/integration/github.md +++ b/doc/integration/github.md @@ -71,17 +71,18 @@ Follow these steps to incorporate the GitHub OAuth 2 app in your GitLab server: ```yaml - { name: 'github', app_id: 'YOUR_APP_ID', - app_secret: 'YOUR_APP_SECRET', - args: { scope: 'user:email' } } + app_secret: 'YOUR_APP_SECRET', + args: { scope: 'user:email' } } ``` For GitHub Enterprise: ```yaml - - { name: 'github', app_id: 'YOUR_APP_ID', - app_secret: 'YOUR_APP_SECRET', - url: "https://github.example.com/", - args: { scope: 'user:email' } } + - { name: 'github', + app_id: 'YOUR_APP_ID', + app_secret: 'YOUR_APP_SECRET', + url: "https://github.example.com/", + args: { scope: 'user:email' } } ``` **Replace `https://github.example.com/` with your GitHub URL.** @@ -125,11 +126,12 @@ omnibus_gitconfig['system'] = { "http" => ["sslVerify = false"] } For installation from source: ```yaml -- { name: 'github', app_id: 'YOUR_APP_ID', - app_secret: 'YOUR_APP_SECRET', - url: "https://github.example.com/", - verify_ssl: false, - args: { scope: 'user:email' } } +- { name: 'github', + app_id: 'YOUR_APP_ID', + app_secret: 'YOUR_APP_SECRET', + url: "https://github.example.com/", + verify_ssl: false, + args: { scope: 'user:email' } } ``` You will also need to disable Git SSL verification on the server hosting GitLab. @@ -148,7 +150,7 @@ via Omnibus, or [restart GitLab](../administration/restart_gitlab.md#installatio Check the [`production.log`](../administration/logs.md#productionlog) on your GitLab server to obtain further details. If you are getting the error like `Faraday::ConnectionFailed (execution expired)` in the log, there may be a connectivity issue -between your GitLab instance and GitHub Enterprise. To verify it, [start the rails console](../administration/troubleshooting/debug.md#starting-a-rails-console-session) +between your GitLab instance and GitHub Enterprise. To verify it, [start the rails console](../administration/operations/rails_console.md#starting-a-rails-console-session) and run the commands below replacing `<github_url>` with the URL of your GitHub Enterprise instance: ```ruby diff --git a/doc/integration/gitlab.md b/doc/integration/gitlab.md index f423863ce8e..a200f6b6470 100644 --- a/doc/integration/gitlab.md +++ b/doc/integration/gitlab.md @@ -63,9 +63,10 @@ GitLab.com will generate an application ID and secret key for you to use. For installations from source: ```yaml - - { name: 'gitlab', app_id: 'YOUR_APP_ID', - app_secret: 'YOUR_APP_SECRET', - args: { scope: 'api' } } + - { name: 'gitlab', + app_id: 'YOUR_APP_ID', + app_secret: 'YOUR_APP_SECRET', + args: { scope: 'api' } } ``` 1. Change 'YOUR_APP_ID' to the Application ID from the GitLab.com application page. diff --git a/doc/integration/gitpod.md b/doc/integration/gitpod.md index f26483e3b5e..b4d12b90be0 100644 --- a/doc/integration/gitpod.md +++ b/doc/integration/gitpod.md @@ -8,7 +8,8 @@ info: "To determine the technical writer assigned to the Stage/Group associated # Gitpod Integration > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/228893) in GitLab 13.4. -> - It's [deployed behind a feature flag](#enable-or-disable-the-gitpod-integration), disabled by default. +> - It was [deployed behind a feature flag](#enable-or-disable-the-gitpod-integration), disabled by default. +> - [Became enabled by default](https://gitlab.com/gitlab-org/gitlab/-/issues/258206) in GitLab 13.5. > - It's enabled on GitLab.com. > - It's recommended for production use. > - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#configure-your-gitlab-instance-with-gitpod). **(CORE ONLY)** @@ -50,25 +51,25 @@ can follow the same steps once the integration has been enabled and configured b If you are new to Gitpod, head over to the [Gitpod documentation](https://www.gitpod.io/docs/self-hosted/latest/self-hosted/) and get your instance up and running. -1. In GitLab, go to **Admin Area > Settings > Integrations**. +1. In GitLab, go to **Admin Area > Settings > General**. 1. Expand the **Gitpod** configuration section. 1. Check **Enable Gitpod**. 1. Add your Gitpod instance URL (for example, `https://gitpod.example.com`). ## Enable or disable the Gitpod integration **(CORE ONLY)** -The Gitpod integration is under development and not ready for production use. It is deployed behind a -feature flag that is **disabled by default**. +The Gitpod integration is deployed behind a feature flag that is **enabled by default**. [GitLab administrators with access to the GitLab Rails console](../administration/feature_flags.md) -can enable it. +can enable or disable it. -To enable it: +To disable it: ```ruby -Feature.enable(:gitpod) +Feature.disable(:gitpod) ``` -To disable it: +To enable it: ```ruby -Feature.disable(:gitpod) +Feature.enable(:gitpod) +``` diff --git a/doc/integration/google.md b/doc/integration/google.md index 0f848bbc7aa..4cf589c1da8 100644 --- a/doc/integration/google.md +++ b/doc/integration/google.md @@ -84,9 +84,10 @@ On your GitLab server: For installations from source: ```yaml - - { name: 'google_oauth2', app_id: 'YOUR_APP_ID', - app_secret: 'YOUR_APP_SECRET', - args: { access_type: 'offline', approval_prompt: '' } } + - { name: 'google_oauth2', + app_id: 'YOUR_APP_ID', + app_secret: 'YOUR_APP_SECRET', + args: { access_type: 'offline', approval_prompt: '' } } ``` 1. Change `YOUR_APP_ID` to the client ID from the Google Developer page diff --git a/doc/integration/jira_development_panel.md b/doc/integration/jira_development_panel.md index 9b7aa5829c1..b86eb1c38b6 100644 --- a/doc/integration/jira_development_panel.md +++ b/doc/integration/jira_development_panel.md @@ -148,7 +148,7 @@ GitLab. No other error messages appear in any logs. If there was an issue with SSL/TLS, this error message will be generated. -- The [GitLab Jira integration](jira.md) requires GitLab to connect to Jira. Any +- The [GitLab Jira integration](../user/project/integrations/jira.md) requires GitLab to connect to Jira. Any TLS issues that arise from a private certificate authority or self-signed certificate [are resolved on the GitLab server](https://docs.gitlab.com/omnibus/settings/ssl.html#other-certificate-authorities), as GitLab is the TLS client. @@ -184,6 +184,16 @@ The message `Successfully connected` indicates a successful TLS handshake. If there are problems, the Java TLS library generates errors that you can look up for more detail. +##### Scope error when connecting Jira via DVCS + +```plaintext +The requested scope is invalid, unknown, or malformed. +``` + +Potential resolutions: + +- Verify the URL includes `scope=api` on the end of the URL. + ##### Jira error adding account and no repositories listed ```plaintext @@ -253,13 +263,18 @@ The GitLab for Jira App uses an iframe to add namespaces on the settings page. S > "You need to sign in or sign up before continuing." -In this case, enable cross-site cookies in your browser. +In this case, use [Firefox](https://www.mozilla.org/en-US/firefox/), [Google Chrome](https://www.google.com/chrome/index.html) or enable cross-site cookies in your browser. ## Usage -Once the integration is set up on GitLab and Jira you may refer any Jira issue by its ID in branch names, commit messages and merge request titles on GitLab's side, -and you will be able to see the linked `branches`, `commits`, and `merge requests` when entering a Jira issue -(inside the Jira issue, merge requests will be called "pull requests"). +After the integration is set up on GitLab and Jira, you can: + +- Refer to any Jira issue by its ID in GitLab branch names, commit messages, and merge request + titles. +- See the linked branches, commits, and merge requests in Jira issues (merge requests are + called "pull requests" in Jira issues). + +Jira issue IDs must be formatted in uppercase for the integration to work.  diff --git a/doc/integration/kerberos.md b/doc/integration/kerberos.md index 1b14b5a986f..1a193deca18 100644 --- a/doc/integration/kerberos.md +++ b/doc/integration/kerberos.md @@ -1,3 +1,10 @@ +--- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference, how-to +--- + # Kerberos integration **(STARTER ONLY)** GitLab can integrate with [Kerberos](https://web.mit.edu/kerberos/) as an authentication mechanism. @@ -114,6 +121,40 @@ Taken together, these rules mean that linking will only work if your users' Kerberos usernames are of the form `foo@AD.EXAMPLE.COM` and their LDAP Distinguished Names look like `sAMAccountName=foo,dc=ad,dc=example,dc=com`. +### Custom allowed realms + +[Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/9962) in GitLab 13.5. + +You can configure custom allowed realms when +the user's Kerberos realm doesn't match the domain from the user's LDAP DN. The +configuration value must specify all domains that users may be expected to +have. Any other domains will be ignored and an LDAP identity will not be linked. + +**For Omnibus installations** + +1. Edit `/etc/gitlab/gitlab.rb`: + + ```ruby + gitlab_rails['kerberos_simple_ldap_linking_allowed_realms'] = ['example.com','kerberos.example.com'] + ``` + +1. Save the file and [reconfigure](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) + GitLab for the changes to take effect. + +--- + +**For installations from source** + +1. Edit `config/gitlab.yml`: + + ```yaml + kerberos: + simple_ldap_linking_allowed_realms: ['example.com','kerberos.example.com'] + ``` + +1. Save the file and [restart](../administration/restart_gitlab.md#installations-from-source) + GitLab for the changes to take effect. + ## HTTP Git access A linked Kerberos account enables you to `git pull` and `git push` using your @@ -123,6 +164,13 @@ GitLab users with a linked Kerberos account can also `git pull` and `git push` using Kerberos tokens, i.e., without having to send their password with each operation. +DANGER: **Danger:** +There is a [known issue](https://github.com/curl/curl/issues/1261) with `libcurl` +older than version 7.64.1 wherein it won't reuse connections when negotiating. +This leads to authorization issues when push is larger than `http.postBuffer` +config. Ensure that Git is using at least `libcurl` 7.64.1 to avoid this. To +know the `libcurl` version installed, run `curl-config --version`. + ### HTTP Git access with Kerberos token (passwordless authentication) #### Support for Git before 2.4 @@ -207,9 +255,10 @@ remove the OmniAuth provider named `kerberos` from your `gitlab.yml` / ```yaml omniauth: + # Rest of configuration omitted # ... providers: - - { name: 'kerberos' } # <-- remove this line + - { name: 'kerberos' } # <-- remove this line ``` 1. [Restart GitLab](../administration/restart_gitlab.md#installations-from-source) for the changes to take effect. diff --git a/doc/integration/omniauth.md b/doc/integration/omniauth.md index dd183ad9eb0..cf09c2f2803 100644 --- a/doc/integration/omniauth.md +++ b/doc/integration/omniauth.md @@ -50,7 +50,7 @@ earlier version, you'll need to explicitly enable it. - `allow_single_sign_on` allows you to specify the providers you want to allow to automatically create an account. It defaults to `false`. If `false` users must be created manually or they will not be able to sign in via OmniAuth. -- `auto_link_ldap_user` can be used if you have [LDAP / ActiveDirectory](ldap.md) +- `auto_link_ldap_user` can be used if you have [LDAP / ActiveDirectory](../administration/auth/ldap/index.md) integration enabled. It defaults to `false`. When enabled, users automatically created through an OmniAuth provider will have their LDAP identity created in GitLab as well. - `block_auto_created_users` defaults to `true`. If `true` auto created users will @@ -104,21 +104,21 @@ To change these settings: ```yaml ## OmniAuth settings - omniauth: - # Allow login via Twitter, Google, etc. using OmniAuth providers - # Versions prior to 11.4 require this to be set to true - # enabled: true + omniauth: + # Allow login via Twitter, Google, etc. using OmniAuth providers + # Versions prior to 11.4 require this to be set to true + # enabled: true - # CAUTION! - # This allows users to login without having a user account first. Define the allowed providers - # using an array, e.g. ["saml", "twitter"], or as true/false to allow all providers or none. - # User accounts will be created automatically when authentication was successful. - allow_single_sign_on: ["saml", "twitter"] + # CAUTION! + # This allows users to login without having a user account first. Define the allowed providers + # using an array, e.g. ["saml", "twitter"], or as true/false to allow all providers or none. + # User accounts will be created automatically when authentication was successful. + allow_single_sign_on: ["saml", "twitter"] - auto_link_ldap_user: true + auto_link_ldap_user: true - # Locks down those users until they have been cleared by the admin (default: true). - block_auto_created_users: true + # Locks down those users until they have been cleared by the admin (default: true). + block_auto_created_users: true ``` Now we can choose one or more of the [Supported Providers](#supported-providers) @@ -142,7 +142,7 @@ The chosen OmniAuth provider is now active and can be used to sign in to GitLab ## Automatically Link Existing Users to OmniAuth Users -> [Introduced in GitLab 13.4.](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/36664) +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/36664) in GitLab 13.4. You can automatically link OmniAuth users with existing GitLab users if their email addresses match. For example, the following setting is used to enable the auto link feature for both a SAML provider and the Twitter OAuth provider: diff --git a/doc/integration/salesforce.md b/doc/integration/salesforce.md index 7e0b2518e76..dbd0a03e3cf 100644 --- a/doc/integration/salesforce.md +++ b/doc/integration/salesforce.md @@ -64,7 +64,7 @@ To get the credentials (a pair of Client ID and Client Secret), you must [create - { name: 'salesforce', app_id: 'SALESFORCE_CLIENT_ID', app_secret: 'SALESFORCE_CLIENT_SECRET' - } + } ``` 1. Change `SALESFORCE_CLIENT_ID` to the Consumer Key from the Salesforce connected application page. diff --git a/doc/integration/saml.md b/doc/integration/saml.md index e7e94b21683..ee08a0026cd 100644 --- a/doc/integration/saml.md +++ b/doc/integration/saml.md @@ -113,16 +113,16 @@ in your SAML IdP: omniauth: providers: - { - name: 'saml', - args: { - assertion_consumer_service_url: 'https://gitlab.example.com/users/auth/saml/callback', - idp_cert_fingerprint: '43:51:43:a1:b5:fc:8b:b7:0a:3a:a9:b1:0f:66:73:a8', - idp_sso_target_url: 'https://login.example.com/idp', - issuer: 'https://gitlab.example.com', - name_identifier_format: 'urn:oasis:names:tc:SAML:2.0:nameid-format:persistent' - }, - label: 'Company Login' # optional label for SAML login button, defaults to "Saml" - } + name: 'saml', + args: { + assertion_consumer_service_url: 'https://gitlab.example.com/users/auth/saml/callback', + idp_cert_fingerprint: '43:51:43:a1:b5:fc:8b:b7:0a:3a:a9:b1:0f:66:73:a8', + idp_sso_target_url: 'https://login.example.com/idp', + issuer: 'https://gitlab.example.com', + name_identifier_format: 'urn:oasis:names:tc:SAML:2.0:nameid-format:persistent' + }, + label: 'Company Login' # optional label for SAML login button, defaults to "Saml" + } ``` 1. Change the value for `assertion_consumer_service_url` to match the HTTPS endpoint @@ -210,7 +210,7 @@ Example: idp_sso_target_url: 'https://login.example.com/idp', issuer: 'https://gitlab.example.com', name_identifier_format: 'urn:oasis:names:tc:SAML:2.0:nameid-format:transient' - } } + } } ``` ### External Groups **(STARTER ONLY)** @@ -228,7 +228,7 @@ SAML login supports automatic identification on whether a user should be conside idp_sso_target_url: 'https://login.example.com/idp', issuer: 'https://gitlab.example.com', name_identifier_format: 'urn:oasis:names:tc:SAML:2.0:nameid-format:persistent' - } } + } } ``` ### Admin Groups **(STARTER ONLY)** @@ -248,7 +248,7 @@ considered admin users. idp_sso_target_url: 'https://login.example.com/idp', issuer: 'https://gitlab.example.com', name_identifier_format: 'urn:oasis:names:tc:SAML:2.0:nameid-format:transient' - } } + } } ``` ### Auditor Groups **(STARTER ONLY)** @@ -270,7 +270,7 @@ considered auditor users. idp_sso_target_url: 'https://login.example.com/idp', issuer: 'https://gitlab.example.com', name_identifier_format: 'urn:oasis:names:tc:SAML:2.0:nameid-format:transient' - } } + } } ``` ## Bypass two factor authentication @@ -328,22 +328,22 @@ In addition to the changes in GitLab, make sure that your IdP is returning the omniauth: providers: - { - name: 'saml', - args: { - assertion_consumer_service_url: 'https://gitlab.example.com/users/auth/saml/callback', - idp_cert_fingerprint: '43:51:43:a1:b5:fc:8b:b7:0a:3a:a9:b1:0f:66:73:a8', - idp_sso_target_url: 'https://login.example.com/idp', - issuer: 'https://gitlab.example.com', - name_identifier_format: 'urn:oasis:names:tc:SAML:2.0:nameid-format:persistent', - upstream_two_factor_authn_contexts: - [ - 'urn:oasis:names:tc:SAML:2.0:ac:classes:CertificateProtectedTransport', - 'urn:oasis:names:tc:SAML:2.0:ac:classes:SecondFactorOTPSMS', - 'urn:oasis:names:tc:SAML:2.0:ac:classes:SecondFactorIGTOKEN' - ] - }, - label: 'Company Login' # optional label for SAML login button, defaults to "Saml" - } + name: 'saml', + args: { + assertion_consumer_service_url: 'https://gitlab.example.com/users/auth/saml/callback', + idp_cert_fingerprint: '43:51:43:a1:b5:fc:8b:b7:0a:3a:a9:b1:0f:66:73:a8', + idp_sso_target_url: 'https://login.example.com/idp', + issuer: 'https://gitlab.example.com', + name_identifier_format: 'urn:oasis:names:tc:SAML:2.0:nameid-format:persistent', + upstream_two_factor_authn_contexts: + [ + 'urn:oasis:names:tc:SAML:2.0:ac:classes:CertificateProtectedTransport', + 'urn:oasis:names:tc:SAML:2.0:ac:classes:SecondFactorOTPSMS', + 'urn:oasis:names:tc:SAML:2.0:ac:classes:SecondFactorIGTOKEN' + ] + }, + label: 'Company Login' # optional label for SAML login button, defaults to "Saml" + } ``` 1. Save the file and [restart GitLab](../administration/restart_gitlab.md#installations-from-source) for the changes to take effect @@ -436,7 +436,7 @@ args: { issuer: 'https://gitlab.example.com', name_identifier_format: 'urn:oasis:names:tc:SAML:2.0:nameid-format:persistent', attribute_statements: { email: ['EmailAddress'] }, - allowed_clock_drift: 1 # for one second clock drift + allowed_clock_drift: 1 # for one second clock drift } ``` @@ -561,10 +561,10 @@ args: { <redacted> -----END PRIVATE KEY-----', security: { - authn_requests_signed: true, # enable signature on AuthNRequest - want_assertions_signed: true, # enable the requirement of signed assertion - embed_sign: true, # embedded signature or HTTP GET parameter signature - metadata_signed: false, # enable signature on Metadata + authn_requests_signed: true, # enable signature on AuthNRequest + want_assertions_signed: true, # enable the requirement of signed assertion + embed_sign: true, # embedded signature or HTTP GET parameter signature + metadata_signed: false, # enable signature on Metadata signature_method: 'http://www.w3.org/2001/04/xmldsig-more#rsa-sha256', digest_method: 'http://www.w3.org/2001/04/xmlenc#sha256', } @@ -588,6 +588,52 @@ Refer to the documentation for your SAML Identity Provider for information on ho The [Generated passwords for users created through integrated authentication](../security/passwords_for_integrated_authentication_methods.md) guide provides an overview of how GitLab generates and sets passwords for users created via SAML. +## Configuring Group SAML on a self-managed GitLab instance **(PREMIUM ONLY)** + +For information on the GitLab.com implementation, please see the [SAML SSO for GitLab.com groups page](../user/group/saml_sso). + +Group SAML SSO helps if you need to allow access via multiple SAML identity providers, but as a multi-tenant solution is less suited to cases where you administer your own GitLab instance. + +To proceed with configuring Group SAML SSO instead, you'll need to enable the `group_saml` OmniAuth provider. This can be done from: + +- `gitlab.rb` for [Omnibus GitLab installations](#omnibus-installations). +- `gitlab/config/gitlab.yml` for [source installations](#source-installations). + +### Limitations + +Group SAML on a self-managed instance is limited when compared to the recommended +[instance-wide SAML](../user/group/saml_sso/index.md). The recommended solution allows you to take advantage of: + +- [LDAP compatibility](../administration/auth/ldap/index.md). +- [LDAP Group Sync](../user/group/index.md#manage-group-memberships-via-ldap). +- [Required groups](#required-groups). +- [Admin groups](#admin-groups). +- [Auditor groups](#auditor-groups). + +### Omnibus installations + +1. Make sure GitLab is + [configured with HTTPS](../install/installation.md#using-https). +1. Enable OmniAuth and the `group_saml` provider in `gitlab.rb`: + + ```ruby + gitlab_rails['omniauth_enabled'] = true + gitlab_rails['omniauth_providers'] = [{ name: 'group_saml' }] + ``` + +### Source installations + +1. Make sure GitLab is + [configured with HTTPS](../install/installation.md#using-https). +1. Enable OmniAuth and the `group_saml` provider in `gitlab/config/gitlab.yml`: + + ```yaml + omniauth: + enabled: true + providers: + - { name: 'group_saml' } + ``` + ## Troubleshooting You can find the base64-encoded SAML Response in the [`production_json.log`](../administration/logs.md#production_jsonlog). diff --git a/doc/integration/sourcegraph.md b/doc/integration/sourcegraph.md index c366dab49b1..47c84643a7d 100644 --- a/doc/integration/sourcegraph.md +++ b/doc/integration/sourcegraph.md @@ -74,7 +74,7 @@ You can skip this step if you already have your GitLab repositories searchable i ### Configure your GitLab instance with Sourcegraph -1. In GitLab, go to **Admin Area > Settings > Integrations**. +1. In GitLab, go to **Admin Area > Settings > General**. 1. Expand the **Sourcegraph** configuration section. 1. Check **Enable Sourcegraph**. 1. Set the Sourcegraph URL to your Sourcegraph instance, e.g., `https://sourcegraph.example.com`. diff --git a/doc/integration/twitter.md b/doc/integration/twitter.md index 95de56f24d8..e501eac0c5f 100644 --- a/doc/integration/twitter.md +++ b/doc/integration/twitter.md @@ -65,7 +65,8 @@ To enable the Twitter OmniAuth provider you must register your application with For installations from source: ```yaml - - { name: 'twitter', app_id: 'YOUR_APP_ID', + - { name: 'twitter', + app_id: 'YOUR_APP_ID', app_secret: 'YOUR_APP_SECRET' } ``` diff --git a/doc/intro/README.md b/doc/intro/README.md index 4bf9c766fc4..02429f387ee 100644 --- a/doc/intro/README.md +++ b/doc/intro/README.md @@ -28,7 +28,7 @@ Create issues, labels, milestones, cast your vote, and review issues. Create merge requests and review code. - [Fork a project and contribute to it](../user/project/repository/forking_workflow.md) -- [Create a new merge request](../gitlab-basics/add-merge-request.md) +- [Create a new merge request](../user/project/merge_requests/creating_merge_requests.md) - [Automatically close issues from merge requests](../user/project/issues/managing_issues.md#closing-issues-automatically) - [Automatically merge when pipeline succeeds](../user/project/merge_requests/merge_when_pipeline_succeeds.md) - [Revert any commit](../user/project/merge_requests/revert_changes.md) diff --git a/doc/logs/logs.md b/doc/logs/logs.md deleted file mode 100644 index 0cb092c85fd..00000000000 --- a/doc/logs/logs.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../administration/logs.md' ---- - -This document was moved to [administration/logs.md](../administration/logs.md). diff --git a/doc/operations/error_tracking.md b/doc/operations/error_tracking.md index c5699ee3d22..150264eddcb 100644 --- a/doc/operations/error_tracking.md +++ b/doc/operations/error_tracking.md @@ -20,10 +20,8 @@ You can sign up to the cloud hosted <https://sentry.io>, deploy your own [on-pre ### Enabling Sentry -NOTE: **Note:** -You will need at least Maintainer [permissions](../user/permissions.md) to enable the Sentry integration. - -GitLab provides an easy way to connect Sentry to your project: +GitLab provides an easy way to connect Sentry to your project. You will need at +least Maintainer [permissions](../user/permissions.md) to enable the Sentry integration. 1. Sign up to Sentry.io or [deploy your own](#deploying-sentry) Sentry instance. 1. [Create](https://docs.sentry.io/product/sentry-basics/guides/integrate-frontend/create-new-project/) a new Sentry project. For each GitLab project that you want to integrate, we recommend that you create a new Sentry project. @@ -46,10 +44,8 @@ You may also want to enable Sentry's GitLab integration by following the steps i ## Error Tracking List -NOTE: **Note:** -You will need at least Reporter [permissions](../user/permissions.md) to view the Error Tracking list. - -You can find the Error Tracking list at **Operations > Error Tracking** in your project's sidebar. +Users with at least Reporter [permissions](../user/permissions.md) +can find the Error Tracking list at **Operations > Error Tracking** in your project's sidebar. Here, you can filter errors by title or by status (one of Ignored , Resolved, or Unresolved) and sort in descending order by Frequency, First Seen, or Last Seen. By default, the error list is ordered by Last Seen and filtered to Unresolved errors.  diff --git a/doc/operations/feature_flags.md b/doc/operations/feature_flags.md index fe7be48270a..00ebfe5ccf8 100644 --- a/doc/operations/feature_flags.md +++ b/doc/operations/feature_flags.md @@ -4,10 +4,11 @@ group: Progressive Delivery info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# Feature Flags **(STARTER)** +# Feature Flags **(CORE)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/7433) in GitLab 11.4. -> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/212318) to [GitLab Starter](https://about.gitlab.com/pricing/) in 13.4 +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/212318) to [GitLab Starter](https://about.gitlab.com/pricing/) in 13.4. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/212318) to [GitLab Core](https://about.gitlab.com/pricing/) in 13.5. With Feature Flags, you can deploy your application's new features to production in smaller batches. You can toggle a feature on and off to subsets of users, helping you achieve Continuous Delivery. @@ -55,6 +56,20 @@ To create and enable a feature flag: You can change these settings by clicking the **{pencil}** (edit) button next to any feature flag in the list. +## Maximum number of feature flags + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/254379) in GitLab 13.5. + +The maximum number of feature flags per project on self-managed GitLab instances +is 200. On GitLab.com, the maximum number is determined by [GitLab.com tier](https://about.gitlab.com/pricing/): + +| Tier | Number of feature flags per project | +|----------|-------------------------------------| +| Free | 50 | +| Bronze | 100 | +| Silver | 150 | +| Gold | 200 | + ## Feature flag strategies > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/35555) in GitLab 13.0. @@ -86,12 +101,49 @@ and clicking **{pencil}** (edit). Enables the feature for all users. It uses the [`default`](https://unleash.github.io/docs/activation_strategy#default) Unleash activation strategy. +### Percent Rollout + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/43340) in GitLab 13.5. + +Enables the feature for a percentage of page views, with configurable consistency +of behavior. This consistency is also known as stickiness. It uses the +[`flexibleRollout`](https://unleash.github.io/docs/activation_strategy#flexiblerollout) +Unleash activation strategy. + +You can configure the consistency to be based on: + +- **User IDs**: Each user ID has a consistent behavior, ignoring session IDs. +- **Session IDs**: Each session ID has a consistent behavior, ignoring user IDs. +- **Random**: Consistent behavior is not guaranteed. The feature is enabled for the + selected percentage of page views randomly. User IDs and session IDs are ignored. +- **Available ID**: Consistent behavior is attempted based on the status of the user: + - If the user is logged in, make behavior consistent based on user ID. + - If the user is anonymous, make the behavior consistent based on the session ID. + - If there is no user ID or session ID, then the feature is enabled for the selected + percentage of page view randomly. + +For example, set a value of 15% based on **Available ID** to enable the feature for 15% of page views. For +authenticated users this is based on their user ID. For anonymous users with a session ID it would be based on their +session ID instead as they do not have a user ID. Then if no session ID is provided, it falls back to random. + +The rollout percentage can be from 0% to 100%. + +Selecting a consistency based on User IDs functions the same as the [percent of Users](#percent-of-users) rollout. + +CAUTION: **Caution:** +Selecting **Random** provides inconsistent application behavior for individual users. + ### Percent of Users Enables the feature for a percentage of authenticated users. It uses the [`gradualRolloutUserId`](https://unleash.github.io/docs/activation_strategy#gradualrolloutuserid) Unleash activation strategy. +NOTE: **Note:** +[Percent rollout](#percent-rollout) with a consistency based on **User IDs** has the same +behavior. It is recommended to use percent rollout instead of percent of users as +it is more flexible. + For example, set a value of 15% to enable the feature for 15% of authenticated users. The rollout percentage can be from 0% to 100%. @@ -105,7 +157,8 @@ ID for the feature to be enabled. See the [Ruby example](#ruby-application-examp ### User IDs -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/8240) in GitLab 12.2. [Updated](https://gitlab.com/gitlab-org/gitlab/-/issues/34363) to be defined per environment in GitLab 12.6. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/8240) in GitLab 12.2. +> - [Updated](https://gitlab.com/gitlab-org/gitlab/-/issues/34363) to be defined per environment in GitLab 12.6. Enables the feature for a list of target users. It is implemented using the Unleash [`userWithId`](https://unleash.github.io/docs/activation_strategy#userwithid) @@ -353,31 +406,9 @@ end ## Feature Flag Related Issues **(PREMIUM)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/36617) in GitLab 13.2. -> - It's deployed behind a feature flag, enabled by default. -> - It's enabled on GitLab.com. -> - It can't be enabled or disabled per-project -> - It's recommended for production use. -> - For GitLab self-managed instances, GitLab administrators can opt to disable it. +> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/251234) in GitLab 13.5. -You can link related issues to a feature flag. In the **Linked issues** section, click the `+` button and input the issue reference number or the full URL of the issue. +You can link related issues to a feature flag. In the **Linked issues** section, +click the `+` button and input the issue reference number or the full URL of the issue. This feature is similar to the [related issues](../user/project/issues/related_issues.md) feature. - -### Enable or disable Feature Flag Related Issues **(CORE ONLY)** - -Feature Flag Related Issues is under development but ready for production use. -It is deployed behind a feature flag that is **enabled by default**. -[GitLab administrators with access to the GitLab Rails console](../administration/feature_flags.md) -can opt to disable it for your instance. - -To disable it: - -```ruby -Feature.disable(:feature_flags_issue_links) -``` - -To enable it: - -```ruby -Feature.enable(:feature_flags_issue_links) -``` diff --git a/doc/operations/incident_management/alert_details.md b/doc/operations/incident_management/alert_details.md index 860e6d32ae4..459331ea0a5 100644 --- a/doc/operations/incident_management/alert_details.md +++ b/doc/operations/incident_management/alert_details.md @@ -1,200 +1,5 @@ --- -stage: Monitor -group: Health -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +redirect_to: alerts.md --- -# Alert details page - -Navigate to the Alert details view by visiting the -[Alert list](./alerts.md) and selecting an alert from the -list. You need least Developer [permissions](../../user/permissions.md) to access -alerts. - -TIP: **Tip:** -To review live examples of GitLab alerts, visit the -[alert list](https://gitlab.com/gitlab-examples/ops/incident-setup/everyone/tanuki-inc/-/alert_management) -for this demo project. Click any alert in the list to examine its alert details -page. - -Alerts provide **Overview** and **Alert details** tabs to give you the right -amount of information you need. - -## Alert overview tab - -The **Overview** tab provides basic information about the alert: - - - -## Alert details tab - - - -### Update an alert's status - -The Alert detail view enables you to update the Alert Status. -See [Create and manage alerts in GitLab](./alerts.md) for more details. - -### Create an issue from an alert - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/217745) in GitLab 13.1. - -The Alert detail view enables you to create an issue with a -description automatically populated from an alert. To create the issue, -click the **Create Issue** button. You can then view the issue from the -alert by clicking the **View Issue** button. - -Closing a GitLab issue associated with an alert changes the alert's status to Resolved. -See [Create and manage alerts in GitLab](alerts.md) for more details about alert statuses. - -### Update an alert's assignee - -> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3066) in GitLab 13.1. - -The Alert detail view allows users to update the Alert assignee. - -In large teams, where there is shared ownership of an alert, it can be difficult -to track who is investigating and working on it. The Alert detail view -enables you to update the Alert assignee: - -NOTE: **Note:** -GitLab currently only supports a single assignee per alert. - -1. To display the list of current alerts, click - **{cloud-gear}** **Operations > Alerts**: - -  - -1. Select your desired alert to display its **Alert Details View**: - -  - -1. If the right sidebar is not expanded, click - **{angle-double-right}** **Expand sidebar** to expand it. -1. In the right sidebar, locate the **Assignee** and click **Edit**. From the - dropdown menu, select each user you want to assign to the alert. GitLab creates - a [to-do list item](../../user/todos.md) for each user. - -  - -To remove an assignee, click **Edit** next to the **Assignee** dropdown menu and -deselect the user from the list of assignees, or click **Unassigned**. - -### Alert system notes - -> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3066) in GitLab 13.1. - -When you take action on an alert, this is logged as a system note, -which is visible in the Alert Details view. This gives you a linear -timeline of the alert's investigation and assignment history. - -The following actions will result in a system note: - -- [Updating the status of an alert](#update-an-alerts-status) -- [Creating an issue based on an alert](#create-an-issue-from-an-alert) -- [Assignment of an alert to a user](#update-an-alerts-assignee) - - - -### Create a to-do from an alert - -> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3066) in GitLab 13.1. - -You can manually create [To-Do list items](../../user/todos.md) for yourself from the -Alert details screen, and view them later on your **To-Do List**. To add a to-do: - -1. To display the list of current alerts, click - **{cloud-gear}** **Operations > Alerts**. -1. Select your desired alert to display its **Alert Management Details View**. -1. Click the **Add a To-Do** button in the right sidebar: - -  - -Click the **To-Do** **{todo-done}** in the navigation bar to view your current to-do list. - - - -### View an alert's metrics data - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/217768) in GitLab 13.2. - -To view the metrics for an alert: - - 1. Sign in as a user with Developer or higher [permissions](../../user/permissions.md). - 1. Navigate to **{cloud-gear}** **Operations > Alerts**. - 1. Click the alert you want to view. - 1. Below the title of the alert, click the **Metrics** tab. - - - -For GitLab-managed Prometheus instances, metrics data is automatically available -for the alert, making it easy to see surrounding behavior. See -[Managed Prometheus instances](../metrics/alerts.md#managed-prometheus-instances) -for information on setting up alerts. - -For externally-managed Prometheus instances, you can configure your alerting rules to -display a chart in the alert. See -[Embedding metrics based on alerts in incident issues](../metrics/embed.md#embedding-metrics-based-on-alerts-in-incident-issues) -for information on how to appropriately configure your alerting rules. See -[External Prometheus instances](../metrics/alerts.md#external-prometheus-instances) -for information on setting up alerts for your self-managed Prometheus instance. - -## Use cases for assigning alerts - -Consider a team formed by different sections of monitoring, collaborating on a -single application. After an alert surfaces, it's extremely important to -route the alert to the team members who can address and resolve the alert. - -Assigning Alerts eases collaboration and delegation. All -assignees are shown in your team's work-flows, and all assignees receive -notifications, simplifying communication and ownership of the alert. - -After completing their portion of investigating or fixing the alert, users can -unassign their account from the alert when their role is complete. -The alert status can be updated on the [Alert list](./alerts.md) to -reflect if the alert has been resolved. - -## View an alert's logs - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/217768) in GitLab 13.3. - -To view the logs for an alert: - - 1. Sign in as a user with Developer or higher [permissions](../../user/permissions.md). - 1. Navigate to **{cloud-gear}** **Operations > Alerts**. - 1. Click the alert you want to view. - 1. Below the title of the alert, click the **Metrics** tab. - 1. Click the [menu](../metrics/dashboards/index.md#chart-context-menu) of the metric chart to view options. - 1. Click **View logs**. - -Read [View logs from metrics panel](#view-logs-from-metrics-panel) for additional information. - -## Embed metrics in incidents and issues - -You can embed metrics anywhere [GitLab Markdown](../../user/markdown.md) is used, such as descriptions, -comments on issues, and merge requests. Embedding metrics helps you share them -when discussing incidents or performance issues. You can output the dashboard directly -into any issue, merge request, epic, or any other Markdown text field in GitLab -by [copying and pasting the link to the metrics dashboard](../metrics/embed.md#embedding-gitlab-managed-kubernetes-metrics). - -You can embed both -[GitLab-hosted metrics](../metrics/embed.md) and -[Grafana metrics](../metrics/embed_grafana.md) -in incidents and issue templates. - -### Context menu - -You can view more details about an embedded metrics panel from the context menu. -To access the context menu, click the **{ellipsis_v}** **More actions** dropdown box -above the upper right corner of the panel. For a list of options, see -[Chart context menu](../metrics/dashboards/index.md#chart-context-menu). - -#### View logs from metrics panel - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/201846) in GitLab Ultimate 12.8. -> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/25455) to [GitLab Core](https://about.gitlab.com/pricing/) 12.9. - -Viewing logs from a metrics panel can be useful if you're triaging an application -incident and need to [explore logs](../metrics/dashboards/index.md#chart-context-menu) -from across your application. These logs help you understand what is affecting -your application's performance and resolve any problems. +This document was moved to [another location](alerts.md). diff --git a/doc/operations/incident_management/alert_integrations.md b/doc/operations/incident_management/alert_integrations.md new file mode 100644 index 00000000000..58c1e1eae76 --- /dev/null +++ b/doc/operations/incident_management/alert_integrations.md @@ -0,0 +1,163 @@ +--- +stage: Monitor +group: Health +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Alert integrations + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/13203) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.4. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/42640) to [GitLab Core](https://about.gitlab.com/pricing/) in 12.8. + +GitLab can accept alerts from any source via a webhook receiver. This can be configured generically or, in GitLab versions 13.1 and greater, you can configure +[External Prometheus instances](../metrics/alerts.md#external-prometheus-instances) +to use this endpoint. + +## Integrations list + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/245331) in [GitLab Core](https://about.gitlab.com/pricing/) 13.5. + +With Maintainer or higher [permissions](../../user/permissions.md), you can view +the list of configured alerts integrations by navigating to +**Settings > Operations** in your project's sidebar menu, and expanding **Alerts** section. +The list displays the integration name, type, and status (enabled or disabled): + + + +## Configuration + +You can either configure alerts to integrate with an [external Prometheus server](#external-prometheus-integration), +or provide a [generic HTTP endpoint](#generic-http-endpoint) to receive alerts +from other services. + +### Generic HTTP Endpoint + +Enabling the Generic HTTP Endpoint creates a unique HTTP endpoint that can receive alert payloads in JSON format. You can always +[customize the payload](#customizing-the-payload) to your liking. + +You will need to activate the endpoint and obtain credentials to set up this integration: + +1. Sign in to GitLab as a user with maintainer [permissions](../../user/permissions.md) + for a project. +1. Navigate to **Settings > Operations** in your project. +1. Expand the **Alerts** section, and in the **Integration** dropdown menu, select **Generic**. +1. Toggle the **Active** alert setting to display the **URL** and **Authorization Key** + for the webhook configuration. + +### External Prometheus integration + +For GitLab versions 13.1 and greater, please see [External Prometheus Instances](../metrics/alerts.md#external-prometheus-instances) to configure alerts for this integration. + +## Customizing the payload + +You can customize the payload by sending the following parameters. This applies to all types of integrations. All fields +other than `title` are optional: + +| Property | Type | Description | +| ------------------------- | --------------- | ----------- | +| `title` | String | The title of the incident. Required. | +| `description` | String | A high-level summary of the problem. | +| `start_time` | DateTime | The time of the incident. If none is provided, a timestamp of the issue will be used. | +| `end_time` | DateTime | For existing alerts only. When provided, the alert is resolved and the associated incident is closed. | +| `service` | String | The affected service. | +| `monitoring_tool` | String | The name of the associated monitoring tool. | +| `hosts` | String or Array | One or more hosts, as to where this incident occurred. | +| `severity` | String | The severity of the alert. Must be one of `critical`, `high`, `medium`, `low`, `info`, `unknown`. Default is `critical`. | +| `fingerprint` | String or Array | The unique identifier of the alert. This can be used to group occurrences of the same alert. | +| `gitlab_environment_name` | String | The name of the associated GitLab [environment](../../ci/environments/index.md). This can be used to associate your alert to your environment. | + +You can also add custom fields to the alert's payload. The values of extra +parameters aren't limited to primitive types (such as strings or numbers), but +can be a nested JSON object. For example: + +```json +{ "foo": { "bar": { "baz": 42 } } } +``` + +TIP: **Payload size:** +Ensure your requests are smaller than the [payload application limits](../../administration/instance_limits.md#generic-alert-json-payloads). + +Example request: + +```shell +curl --request POST \ + --data '{"title": "Incident title"}' \ + --header "Authorization: Bearer <authorization_key>" \ + --header "Content-Type: application/json" \ + <url> +``` + +The `<authorization_key>` and `<url>` values can be found when configuring an alert integration. + +Example payload: + +```json +{ + "title": "Incident title", + "description": "Short description of the incident", + "start_time": "2019-09-12T06:00:55Z", + "service": "service affected", + "monitoring_tool": "value", + "hosts": "value", + "severity": "high", + "fingerprint": "d19381d4e8ebca87b55cda6e8eee7385", + "foo": { + "bar": { + "baz": 42 + } + } +} +``` + +## Triggering test alerts + +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3066) in GitLab Core in 13.2. + +After a [project maintainer or owner](../../user/permissions.md) +configures an integration, you can trigger a test +alert to confirm your integration works properly. + +1. Sign in as a user with Developer or greater [permissions](../../user/permissions.md). +1. Navigate to **Settings > Operations** in your project. +1. Click **Alerts endpoint** to expand the section. +1. Enter a sample payload in **Alert test payload** (valid JSON is required). +1. Click **Test alert payload**. + +GitLab displays an error or success message, depending on the outcome of your test. + +## Automatic grouping of identical alerts **(PREMIUM)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/214557) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.2. + +In GitLab versions 13.2 and greater, GitLab groups alerts based on their +payload. When an incoming alert contains the same payload as another alert +(excluding the `start_time` and `hosts` attributes), GitLab groups these alerts +together and displays a counter on the [Alert Management List](./incidents.md) +and details pages. + +If the existing alert is already `resolved`, GitLab creates a new alert instead. + + + +## Link to your Opsgenie Alerts + +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3066) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.2. + +You can monitor alerts using a GitLab integration with [Opsgenie](https://www.atlassian.com/software/opsgenie). + +If you enable the Opsgenie integration, you can't have other GitLab alert +services, such as [Generic Alerts](generic_alerts.md) or Prometheus alerts, +active at the same time. + +To enable Opsgenie integration: + +1. Sign in as a user with Maintainer or Owner [permissions](../../user/permissions.md). +1. Navigate to **Operations > Alerts**. +1. In the **Integrations** select box, select **Opsgenie**. +1. Select the **Active** toggle. +1. In the **API URL** field, enter the base URL for your Opsgenie integration, + such as `https://app.opsgenie.com/alert/list`. +1. Select **Save changes**. + +After you enable the integration, navigate to the Alerts list page at +**Operations > Alerts**, and then select **View alerts in Opsgenie**. diff --git a/doc/operations/incident_management/alert_notifications.md b/doc/operations/incident_management/alert_notifications.md new file mode 100644 index 00000000000..130c4e82088 --- /dev/null +++ b/doc/operations/incident_management/alert_notifications.md @@ -0,0 +1,36 @@ +--- +stage: Monitor +group: Health +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Paging and notifications + +When there is a new alert or incident, it is important for a responder to be notified +immediately so they can triage and respond to the problem. Responders can receive +notifications using the methods described on this page. + +## Slack notifications + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/216326) in GitLab 13.1. + +Responders can be paged via Slack using the +[Slack Notifications Service](../../user/project/integrations/slack.md), which you +can configure for new alerts and new incidents. After configuring, responders +receive a **single** page via Slack. To set up Slack notifications on your mobile +device, make sure to enable notifications for the Slack app on your phone so +you never miss a page. + +## Email notifications + +Email notifications are available in projects that have been +[configured to create incidents automatically](incidents.md#create-incidents-automatically) +for triggered alerts. Project members with the **Owner** or **Maintainer** roles are +sent an email notification automatically. (This is not configurable.) To optionally +send additional email notifications to project members with the **Developer** role: + +1. Navigate to **Settings > Operations**. +1. Expand the **Incidents** section. +1. In the **Alert Integration** tab, select the **Send a separate email notification to Developers** + check box. +1. Select **Save changes**. diff --git a/doc/operations/incident_management/alerts.md b/doc/operations/incident_management/alerts.md index d908af63000..a6168386024 100644 --- a/doc/operations/incident_management/alerts.md +++ b/doc/operations/incident_management/alerts.md @@ -4,119 +4,261 @@ group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# Create and manage alerts in GitLab +# Alerts -Users with at least Developer [permissions](../../user/permissions.md) can access -the Alert Management list at **{cloud-gear}** **Operations > Alerts** in your -project's sidebar. The Alert Management list displays alerts sorted by start time, -but you can change the sort order by clicking the headers in the Alert Management list. +Alerts are a critical entity in your incident managment workflow. They represent a notable event that might indicate a service outage or disruption. GitLab provides a list view for triage and detail view for deeper investigation of what happened. + +## Alert List + +Users with at least Developer [permissions](../../user/permissions.md) can +access the Alert list at **Operations > Alerts** in your project's +sidebar. The Alert list displays alerts sorted by start time, but +you can change the sort order by clicking the headers in the Alert list. ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/217745) in GitLab 13.1.) The alert list displays the following information: - + -- **Search** - The alert list supports a simple free text search on the title, +- **Search**: The alert list supports a simple free text search on the title, description, monitoring tool, and service fields. ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/213884) in GitLab 13.1.) -- **Severity** - The current importance of a alert and how much attention it should - receive. For a listing of all statuses, read [Alert Management severity](#alert-severity). -- **Start time** - How long ago the alert fired. This field uses the standard - GitLab pattern of `X time ago`, but is supported by a granular date/time tooltip - depending on the user's locale. -- **Alert description** - The description of the alert, which attempts to capture the most meaningful data. -- **Event count** - The number of times that an alert has fired. -- **Issue** - A link to the incident issue that has been created for the alert. -- **Status** - The current status of the alert: +- **Severity**: The current importance of a alert and how much attention it + should receive. For a listing of all statuses, read [Alert Management severity](#alert-severity). +- **Start time**: How long ago the alert fired. This field uses the standard + GitLab pattern of `X time ago`, but is supported by a granular date/time + tooltip depending on the user's locale. +- **Alert description**: The description of the alert, which attempts to + capture the most meaningful data. +- **Event count**: The number of times that an alert has fired. +- **Issue**: A link to the incident issue that has been created for the alert. +- **Status**: The current status of the alert: - **Triggered**: No one has begun investigation. - **Acknowledged**: Someone is actively investigating the problem. - **Resolved**: No further work is required. - + TIP: **Tip:** -Check out a [live example](https://gitlab.com/gitlab-examples/ops/incident-setup/everyone/tanuki-inc/-/alert_management) +Check out a live example available from the +[`tanuki-inc` project page](https://gitlab-examples-ops-incident-setup-everyone-tanuki-inc.34.69.64.147.nip.io/) in GitLab to examine alerts in action. -## Enable Alerts +## Alert severity + +Each level of alert contains a uniquely shaped and color-coded icon to help +you identify the severity of a particular alert. These severity icons help you +immediately identify which alerts you should prioritize investigating: + + -NOTE: **Note:** -You need at least Maintainer [permissions](../../user/permissions.md) to enable -the Alerts feature. +Alerts contain one of the following icons: -There are several ways to accept alerts into your GitLab project. -Enabling any of these methods enables the Alert list. After configuring -alerts, visit **{cloud-gear}** **Operations > Alerts** in your project's sidebar -to view the list of alerts. +| Severity | Icon | Color (hexadecimal) | +|----------|-------------------------|---------------------| +| Critical | **{severity-critical}** | `#8b2615` | +| High | **{severity-high}** | `#c0341d` | +| Medium | **{severity-medium}** | `#fca429` | +| Low | **{severity-low}** | `#fdbc60` | +| Info | **{severity-info}** | `#418cd8` | +| Unknown | **{severity-unknown}** | `#bababa` | -### Enable GitLab-managed Prometheus alerts +## Alert details page -You can install the GitLab-managed Prometheus application on your Kubernetes -cluster. For more information, read -[Managed Prometheus on Kubernetes](../../user/project/integrations/prometheus.md#managed-prometheus-on-kubernetes). -When GitLab-managed Prometheus is installed, the [Alerts list](alerts.md) -is also enabled. +Navigate to the Alert details view by visiting the [Alert list](./alerts.md) +and selecting an alert from the list. You need least Developer [permissions](../../user/permissions.md) +to access alerts. -To populate the alerts with data, read -[GitLab-Managed Prometheus instances](../metrics/alerts.md#managed-prometheus-instances). +TIP: **Tip:** +To review live examples of GitLab alerts, visit the +[alert list](https://gitlab.com/gitlab-examples/ops/incident-setup/everyone/tanuki-inc/-/alert_management) +for this demo project. Select any alert in the list to examine its alert details +page. -### Enable external Prometheus alerts +Alerts provide **Overview** and **Alert details** tabs to give you the right +amount of information you need. -You can configure an externally-managed Prometheus instance to send alerts -to GitLab. To set up this configuration, read the [configuring Prometheus](../metrics/alerts.md#external-prometheus-instances) documentation. Activating the external Prometheus -configuration also enables the [Alerts list](./alerts.md). +### Alert details tab -To populate the alerts with data, read -[External Prometheus instances](../metrics/alerts.md#external-prometheus-instances). +The **Alert details** tab has two sections. The top section provides a short list of critical details such as the severity, start time, number of events, and originating monitorting tool. The second section displays the full alert payload. -### Enable a Generic Alerts endpoint +### Metrics tab -GitLab provides the Generic Alerts endpoint so you can accept alerts from a third-party -alerts service. Read the -[instructions for toggling generic alerts](generic_alerts.md#setting-up-generic-alerts) -to add this option. After configuring the endpoint, the -[Alerts list](./alerts.md) is enabled. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/217768) in GitLab 13.2. -To populate the alerts with data, read [Customizing the payload](./generic_alerts.md#customizing-the-payload) for requests to the alerts endpoint. +The **Metrics** tab will display a metrics chart for alerts coming from Prometheus. If the alert originated from any other tool, the **Metrics** tab will be empty. To set up alerts for GitLab-managed Prometheus instances, see [Managed Prometheus instances](../metrics/alerts.md#managed-prometheus-instances). For externally-managed Prometheus instances, you will need to configure your alerting +rules to display a chart in the alert. For information about how to configure +your alerting rules, see [Embedding metrics based on alerts in incident issues](../metrics/embed.md#embedding-metrics-based-on-alerts-in-incident-issues). See +[External Prometheus instances](../metrics/alerts.md#external-prometheus-instances) +for information about setting up alerts for your self-managed Prometheus +instance. -### Opsgenie integration **(PREMIUM)** +To view the metrics for an alert: -> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3066) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.2. +1. Sign in as a user with Developer or higher [permissions](../../user/permissions.md). +1. Navigate to **Operations > Alerts**. +1. Select the alert you want to view. +1. Below the title of the alert, select the **Metrics** tab. -A new way of monitoring Alerts via a GitLab integration is with -[Opsgenie](https://www.atlassian.com/software/opsgenie). + -NOTE: **Note:** -If you enable the Opsgenie integration, you can't have other GitLab alert services, -such as [Generic Alerts](./generic_alerts.md) or -Prometheus alerts, active at the same time. +#### View an alert's logs -To enable Opsgenie integration: +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/201846) in GitLab Ultimate 12.8. and [improved](https://gitlab.com/gitlab-org/gitlab/-/issues/217768) in GitLab 13.3. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/25455) to [GitLab Core](https://about.gitlab.com/pricing/) 12.9. -1. Sign in as a user with Maintainer or Owner [permissions](../../user/permissions.md). -1. Navigate to **{cloud-gear}** **Operations > Alerts**. -1. In the **Integrations** select box, select Opsgenie. -1. Click the **Active** toggle. -1. In the **API URL**, enter the base URL for your Opsgenie integration, such - as `https://app.opsgenie.com/alert/list`. -1. Click **Save changes**. +Viewing logs from a metrics panel can be useful if you're triaging an +application incident and need to [explore logs](../metrics/dashboards/index.md#chart-context-menu) +from across your application. These logs help you understand what's affecting +your application's performance and how to resolve any problems. -After enabling the integration, navigate to the Alerts list page at -**{cloud-gear}** **Operations > Alerts**, and click **View alerts in Opsgenie**. +To view the logs for an alert: -## Alert severity +1. Sign in as a user with Developer or higher [permissions](../../user/permissions.md). +1. Navigate to **Operations > Alerts**. +1. Select the alert you want to view. +1. Below the title of the alert, select the **Metrics** tab. +1. Select the [menu](../metrics/dashboards/index.md#chart-context-menu) of + the metric chart to view options. +1. Select **View logs**. -Each level of alert contains a uniquely shaped and color-coded icon to help -you identify the severity of a particular alert. These severity icons help you -immediately identify which alerts you should prioritize investigating: +### Activity feed tab - +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3066) in GitLab 13.1. -Alerts contain one of the following icons: +The **Activity feed** tab is a log of activity on the alert. When you take action on an alert, this is logged as a system note. This gives you a linear +timeline of the alert's investigation and assignment history. + +The following actions will result in a system note: + +- [Updating the status of an alert](#update-an-alerts-status) +- [Creating an incident based on an alert](#create-an-incident-from-an-alert) +- [Assignment of an alert to a user](#assign-an-alert) + + + +## Alert actions + +There are different actions avilable in GitLab to help triage and respond to alerts. + +### Update an alert's status + +The Alert detail view enables you to update the Alert Status. +See [Create and manage alerts in GitLab](./alerts.md) for more details. + +### Create an incident from an alert + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/217745) in GitLab 13.1. + +The Alert detail view enables you to create an issue with a +description populated from an alert. To create the issue, +select the **Create Issue** button. You can then view the issue from the +alert by selecting the **View Issue** button. + +Closing a GitLab issue associated with an alert changes the alert's status to +Resolved. See [Create and manage alerts in GitLab](alerts.md) for more details +about alert statuses. + +### Assign an alert + +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3066) in GitLab 13.1. + +In large teams, where there is shared ownership of an alert, it can be +difficult to track who is investigating and working on it. Assigning alerts eases collaboration and delegation by indicating which user is owning the alert. GitLab supports only a single assignee per alert. + +To assign an alert: + +1. To display the list of current alerts, navigate to **Operations > Alerts**: + +  + +1. Select your desired alert to display its **Alert Details View**: + +  + +1. If the right sidebar is not expanded, select + **{angle-double-right}** **Expand sidebar** to expand it. +1. In the right sidebar, locate the **Assignee**, and then select **Edit**. + From the dropdown menu, select each user you want to assign to the alert. + GitLab creates a [to-do item](../../user/todos.md) for each user. + +  + +After completing their portion of investigating or fixing the alert, users can +unassign themselves from the alert. To remove an assignee, select **Edit** next to the **Assignee** dropdown menu +and deselect the user from the list of assignees, or select **Unassigned**. + +### Create a to do from an alert + +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3066) in GitLab 13.1. + +You can manually create [To-Do list items](../../user/todos.md) for yourself +from the Alert details screen, and view them later on your **To-Do List**. To +add a to do: + +1. To display the list of current alerts, navigate to **Operations > Alerts**. +1. Select your desired alert to display its **Alert Management Details View**. +1. Select the **Add a To-Do** button in the right sidebar: + +  + +Select the **To-Do List** **{todo-done}** in the navigation bar to view your current to-do list. + + + +## Link runbooks to alerts + +> Runbook URLs [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/39315) in GitLab 13.3. + +When creating alerts from the metrics dashboard for +[managed Prometheus instances](../metrics/alerts.md#managed-prometheus-instances), +you can link a runbook. When the alert triggers, you can access the runbook through +the [chart context menu](../metrics/dashboards/index.md#chart-context-menu) in the +upper-right corner of the metrics chart, making it easy for you to locate and access +the correct runbook: + + + +## View the environment that generated the alert + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/232492) in GitLab 13.5. +> - It's [deployed behind a feature flag](../../user/feature_flags.md), disabled by default. +> - It's disabled on GitLab.com. +> - It's not recommended for production use. +> - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enable-or-disable-environment-link-in-alert-details). **(CORE ONLY)** + +CAUTION: **Warning:** +This feature might not be available to you. Check the **version history** note above for details. + +The environment information and the link are displayed in the [Alert Details tab](#alert-details-tab). + +### Enable or disable Environment Link in Alert Details **(CORE ONLY)** + +Viewing the environment is under development and not ready for production use. It is +deployed behind a feature flag that is **disabled by default**. +[GitLab administrators with access to the GitLab Rails console](../../administration/feature_flags.md) +can enable it. + +To enable it: + +```ruby +Feature.enable(:expose_environment_path_in_alert_details) +``` + +To enable for just a particular project: + +```ruby +project = Project.find_by_full_path('your-group/your-project') +Feature.enable(:expose_environment_path_in_alert_details, project) +``` + +To disable it: + +```ruby +Feature.disable(:expose_environment_path_in_alert_details) +``` + +To disable for just a particular project: -| Severity | Icon | Color (hexadecimal) | -|---|---|---| -| Critical | **{severity-critical}** | `#8b2615` | -| High | **{severity-high}** | `#c0341d` | -| Medium | **{severity-medium}** | `#fca429` | -| Low | **{severity-low}** | `#fdbc60` | -| Info | **{severity-info}** | `#418cd8` | -| Unknown | **{severity-unknown}** | `#bababa` | +```ruby +project = Project.find_by_full_path('your-group/your-project') +Feature.disable(:expose_environment_path_in_alert_details, project) +``` diff --git a/doc/operations/incident_management/generic_alerts.md b/doc/operations/incident_management/generic_alerts.md index 11d4dbc6924..a8f2f9a58a6 100644 --- a/doc/operations/incident_management/generic_alerts.md +++ b/doc/operations/incident_management/generic_alerts.md @@ -1,126 +1,5 @@ --- -stage: Monitor -group: Health -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +redirect_to: alert_notifications.md --- -# Generic alerts integration - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/13203) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.4. -> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/42640) to [GitLab Core](https://about.gitlab.com/pricing/) in 12.8. - -GitLab can accept alerts from any source via a generic webhook receiver. -When you set up the generic alerts integration, a unique endpoint will -be created which can receive a payload in JSON format, and will in turn -create an issue with the payload in the body of the issue. You can always -[customize the payload](#customizing-the-payload) to your liking. - -The entire payload will be posted in the issue discussion as a comment -authored by the GitLab Alert Bot. - -NOTE: **Note:** -In GitLab versions 13.1 and greater, you can configure -[External Prometheus instances](../metrics/alerts.md#external-prometheus-instances) -to use this endpoint. - -## Setting up generic alerts - -To obtain credentials for setting up a generic alerts integration: - -- Sign in to GitLab as a user with maintainer [permissions](../../user/permissions.md) for a project. -- Navigate to the **Operations** page for your project, depending on your installed version of GitLab: - - *In GitLab versions 13.1 and greater,* navigate to **Settings > Operations** in your project. - - *In GitLab versions prior to 13.1,* navigate to **Settings > Integrations** in your project. GitLab will display a banner encouraging you to enable the Alerts endpoint in **Settings > Operations** instead. -- Click **Alerts endpoint**. -- Toggle the **Active** alert setting to display the **URL** and **Authorization Key** for the webhook configuration. - -## Customizing the payload - -You can customize the payload by sending the following parameters. All fields other than `title` are optional: - -| Property | Type | Description | -| -------- | ---- | ----------- | -| `title` | String | The title of the incident. Required. | -| `description` | String | A high-level summary of the problem. | -| `start_time` | DateTime | The time of the incident. If none is provided, a timestamp of the issue will be used. | -| `end_time` | DateTime | For existing alerts only. When provided, the alert is resolved and the associated incident is closed. | -| `service` | String | The affected service. | -| `monitoring_tool` | String | The name of the associated monitoring tool. | -| `hosts` | String or Array | One or more hosts, as to where this incident occurred. | -| `severity` | String | The severity of the alert. Must be one of `critical`, `high`, `medium`, `low`, `info`, `unknown`. Default is `critical`. | -| `fingerprint` | String or Array | The unique identifier of the alert. This can be used to group occurrences of the same alert. | -| `gitlab_environment_name` | String | The name of the associated GitLab [environment](../../ci/environments/index.md). This can be used to associate your alert to your environment. | - -You can also add custom fields to the alert's payload. The values of extra parameters -are not limited to primitive types, such as strings or numbers, but can be a nested -JSON object. For example: - -```json -{ "foo": { "bar": { "baz": 42 } } } -``` - -TIP: **Payload size:** -Ensure your requests are smaller than the [payload application limits](../../administration/instance_limits.md#generic-alert-json-payloads). - -Example request: - -```shell -curl --request POST \ - --data '{"title": "Incident title"}' \ - --header "Authorization: Bearer <authorization_key>" \ - --header "Content-Type: application/json" \ - <url> -``` - -The `<authorization_key>` and `<url>` values can be found when [setting up generic alerts](#setting-up-generic-alerts). - -Example payload: - -```json -{ - "title": "Incident title", - "description": "Short description of the incident", - "start_time": "2019-09-12T06:00:55Z", - "service": "service affected", - "monitoring_tool": "value", - "hosts": "value", - "severity": "high", - "fingerprint": "d19381d4e8ebca87b55cda6e8eee7385", - "foo": { - "bar": { - "baz": 42 - } - } -} -``` - -## Triggering test alerts - -> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3066) in GitLab Core in 13.2. - -After a [project maintainer or owner](#setting-up-generic-alerts) -[configures generic alerts](#setting-up-generic-alerts), you can trigger a -test alert to confirm your integration works properly. - -1. Sign in as a user with Developer or greater [permissions](../../user/permissions.md). -1. Navigate to **Settings > Operations** in your project. -1. Click **Alerts endpoint** to expand the section. -1. Enter a sample payload in **Alert test payload** (valid JSON is required). -1. Click **Test alert payload**. - -GitLab displays an error or success message, depending on the outcome of your test. - -## Automatic grouping of identical alerts **(PREMIUM)** - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/214557) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.2. - -In GitLab versions 13.2 and greater, GitLab groups alerts based on their payload. -When an incoming alert contains the same payload as another alert (excluding the -`start_time` and `hosts` attributes), GitLab groups these alerts together and -displays a counter on the -[Alert Management List](./incidents.md) -and details pages. - -If the existing alert is already `resolved`, then a new alert will be created instead. - - +This document was moved to [another location](alert_notifications.md). diff --git a/doc/operations/incident_management/img/alert_detail_activity_feed_v13_5.png b/doc/operations/incident_management/img/alert_detail_activity_feed_v13_5.png Binary files differnew file mode 100644 index 00000000000..2c1c4c39515 --- /dev/null +++ b/doc/operations/incident_management/img/alert_detail_activity_feed_v13_5.png diff --git a/doc/operations/incident_management/img/incident_highlight_bar_v13_5.png b/doc/operations/incident_management/img/incident_highlight_bar_v13_5.png Binary files differnew file mode 100644 index 00000000000..6a40e97820c --- /dev/null +++ b/doc/operations/incident_management/img/incident_highlight_bar_v13_5.png diff --git a/doc/operations/incident_management/img/incident_list_v13_4.png b/doc/operations/incident_management/img/incident_list_v13_4.png Binary files differdeleted file mode 100644 index bf00e630c67..00000000000 --- a/doc/operations/incident_management/img/incident_list_v13_4.png +++ /dev/null diff --git a/doc/operations/incident_management/img/incident_list_v13_5.png b/doc/operations/incident_management/img/incident_list_v13_5.png Binary files differnew file mode 100644 index 00000000000..88942a70e88 --- /dev/null +++ b/doc/operations/incident_management/img/incident_list_v13_5.png diff --git a/doc/operations/incident_management/img/incident_sla_settings_v13_5.png b/doc/operations/incident_management/img/incident_sla_settings_v13_5.png Binary files differnew file mode 100644 index 00000000000..94c8b840210 --- /dev/null +++ b/doc/operations/incident_management/img/incident_sla_settings_v13_5.png diff --git a/doc/operations/incident_management/img/integrations_list_v13_5.png b/doc/operations/incident_management/img/integrations_list_v13_5.png Binary files differnew file mode 100644 index 00000000000..babaa785ad6 --- /dev/null +++ b/doc/operations/incident_management/img/integrations_list_v13_5.png diff --git a/doc/operations/incident_management/img/link_runbooks_to_alerts_v13_5.png b/doc/operations/incident_management/img/link_runbooks_to_alerts_v13_5.png Binary files differnew file mode 100644 index 00000000000..a63001b4cde --- /dev/null +++ b/doc/operations/incident_management/img/link_runbooks_to_alerts_v13_5.png diff --git a/doc/operations/incident_management/img/timeline_view_toggle_v13_5.png b/doc/operations/incident_management/img/timeline_view_toggle_v13_5.png Binary files differnew file mode 100644 index 00000000000..542ca139f7e --- /dev/null +++ b/doc/operations/incident_management/img/timeline_view_toggle_v13_5.png diff --git a/doc/operations/incident_management/incidents.md b/doc/operations/incident_management/incidents.md index 3ff02b3dc6b..3d85fa0ebd8 100644 --- a/doc/operations/incident_management/incidents.md +++ b/doc/operations/incident_management/incidents.md @@ -4,16 +4,88 @@ group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# Create and manage incidents in GitLab +# Incidents -While no configuration is required to use the [manual features](#create-an-incident-manually) -of incident management, some simple [configuration](#configure-incidents) is needed to automate incident creation. +Incidents are critical entities in incident management workflows. They represent a service disruption or outage that needs to be restored urgently. GitLab provides tools for the triage, response, and remediation of incidents. -For users with at least Developer [permissions](../../user/permissions.md), the -Incident Management list is available at **Operations > Incidents** +## Incident Creation + +You can create an incident manually or automatically. + +### Create incidents manually + +If you have at least Guest [permissions](../../user/permissions.md), to create an Incident, you have two options to do this manually. + +**From the Incidents List:** + +> [Moved](https://gitlab.com/gitlab-org/monitor/health/-/issues/24) to GitLab core in 13.3. + +- Navigate to **Operations > Incidents** and click **Create Incident**. +- Create a new issue using the `incident` template available when creating it. +- Create a new issue and assign the `incident` label to it. + + + +**From the Issues List:** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/230857) in GitLab 13.4. + +- Navigate to **Issues > List** and click **Create Issue**. +- Create a new issue using the `type` drop-down and select `Incident`. +- The page refreshes and the page only displays fields relevant to Incidents. + + + +### Create incidents automatically + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/4925) in GitLab Ultimate 11.11. + +With Maintainer or higher [permissions](../../user/permissions.md), you can enable + GitLab to create incident automatically whenever an alert is triggered: + +1. Navigate to **Settings > Operations > Incidents** and expand + **Incidents**: + +  + +1. Check the **Create an incident** + checkbox. +1. To customize the incident, select an [issue templates](../../user/project/description_templates.md#creating-issue-templates). +1. To send [an email notification](alert_notifications.md#email-notifications) to users + with [Developer permissions](../../user/permissions.md), select + **Send a separate email notification to Developers**. Email notifications will also be sent to users with **Maintainer** and **Owner** permissions. +1. Click **Save changes**. + +### Create incidents via the PagerDuty webhook + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/119018) in GitLab 13.3. + +You can set up a webhook with PagerDuty to automatically create a GitLab incident +for each PagerDuty incident. This configuration requires you to make changes +in both PagerDuty and GitLab: + +1. Sign in as a user with Maintainer [permissions](../../user/permissions.md). +1. Navigate to **Settings > Operations > Incidents** and expand **Incidents**. +1. Select the **PagerDuty integration** tab: + +  + +1. Activate the integration, and save the changes in GitLab. +1. Copy the value of **Webhook URL** for use in a later step. +1. Follow the steps described in the + [PagerDuty documentation](https://support.pagerduty.com/docs/webhooks) + to add the webhook URL to a PagerDuty webhook integration. + +To confirm the integration is successful, trigger a test incident from PagerDuty to +confirm that a GitLab incident is created from the incident. + +## Incident list + +For users with at least Guest [permissions](../../user/permissions.md), the +Incident list is available at **Operations > Incidents** in your project's sidebar. The list contains the following metrics: - + - **Status** - To filter incidents by their status, click **Open**, **Closed**, or **All** above the incident list. @@ -27,8 +99,8 @@ in your project's sidebar. The list contains the following metrics: - **{severity-low}** **Low - S4** - **{severity-unknown}** **Unknown** - NOTE: **Note:** - Editing incident severity on the incident details page was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/229402) in GitLab 13.4. + [Editing incident severity](#change-severity) on the incident details page was + [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/229402) in GitLab 13.4. - **Incident** - The description of the incident, which attempts to capture the most meaningful data. @@ -44,113 +116,117 @@ The Incident list displays incidents sorted by incident created date. To see if a column is sortable, point your mouse at the header. Sortable columns display an arrow next to the column name. +Incidents share the [Issues API](../../user/project/issues/index.md). + TIP: **Tip:** For a live example of the incident list in action, visit this [demo project](https://gitlab.com/gitlab-examples/ops/incident-setup/everyone/tanuki-inc/-/incidents). -NOTE: **Note:** -Incidents share the [Issues API](../../user/project/issues/index.md). +## Incident details + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/230847) in GitLab 13.4. -## Configure incidents +Users with at least Reporter [permissions](../../user/permissions.md) can view +the Incident Details page. Navigate to **Operations > Incidents** in your project's +sidebar, and select an incident from the list. -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/4925) in GitLab Ultimate 11.11. +When you take any of these actions on an incident, GitLab logs a system note and +displays it in the Incident Details view: -With Maintainer or higher [permissions](../../user/permissions.md), you can enable -or disable Incident Management features in the GitLab user interface -to create issues when alerts are triggered: +- Updating the severity of an incident + ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/42358) in GitLab 13.5.) -1. Navigate to **Settings > Operations > Incidents** and expand - **Incidents**: +For live examples of GitLab incidents, visit the `tanuki-inc` project's +[incident list page](https://gitlab.com/gitlab-examples/ops/incident-setup/everyone/tanuki-inc/-/incidents). +Click any incident in the list to display its incident details page. -  +### Summary -1. For GitLab versions 11.11 and greater, you can select the **Create an issue** - checkbox to create an issue based on your own - [issue templates](../../user/project/description_templates.md#creating-issue-templates). - For more information, see - [Trigger actions from alerts](../metrics/alerts.md#trigger-actions-from-alerts) **(ULTIMATE)**. -1. To create issues from alerts, select the template in the **Issue Template** - select box. -1. To send [separate email notifications](index.md#notify-developers-of-alerts) to users - with [Developer permissions](../../user/permissions.md), select - **Send a separate email notification to Developers**. -1. Click **Save changes**. +The summary section for incidents provides both critical details about and the +contents of the issue template (if one was used). The highlighted bar at the top +of the incident displays from left to right: -Appropriately configured alerts include an -[embedded chart](../metrics/embed.md#embedding-metrics-based-on-alerts-in-incident-issues) -for the query corresponding to the alert. You can also configure GitLab to -[close issues](../metrics/alerts.md#trigger-actions-from-alerts) -when you receive notification that the alert is resolved. +- The link to the original alert. +- The alert start time. +- The event count. -## Create an incident manually +Beneath the highlight bar, GitLab displays a summary that includes the following fields: -If you have at least Developer [permissions](../../user/permissions.md), to create an Incident, you have two options. +- Start time +- Severity +- `full_query` +- Monitoring tool -### From the Incidents List +Comments are displayed in threads, but can be displayed chronologically +[in a timeline view](#timeline-view). -> [Moved](https://gitlab.com/gitlab-org/monitor/health/-/issues/24) to GitLab core in 13.3. +### Alert details -- Navigate to **Operations > Incidents** and click **Create Incident**. -- Create a new issue using the `incident` template available when creating it. -- Create a new issue and assign the `incident` label to it. +Incidents show the details of linked alerts in a separate tab. To populate this +tab, the incident must have been created with a linked alert. Incidents +created automatically from alerts have this +field populated. - + -### From the Issues List +### Timeline view -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/230857) in GitLab 13.4. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/227836) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.5. -- Navigate to **Issues > List** and click **Create Issue**. -- Create a new issue using the `type` drop-down and select `Incident`. -- The page refreshes and the page only displays fields relevant to Incidents. +To quickly see the latest updates on an incident, click +**{comments}** **Turn timeline view on** in the comment bar to display comments +un-threaded and ordered chronologically, newest to oldest: - + -## Configure PagerDuty integration +### Service Level Agreement countdown timer -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/119018) in GitLab 13.3. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/241663) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.5. -You can set up a webhook with PagerDuty to automatically create a GitLab issue -for each PagerDuty incident. This configuration requires you to make changes -in both PagerDuty and GitLab: +After enabling **Incident SLA** in the Incident Management configuration, newly-created +incidents display a SLA (Service Level Agreement) timer showing the time remaining before +the SLA period expires. If the incident is not closed before the SLA period ends, GitLab +adds a `missed::SLA` label to the incident. -1. Sign in as a user with Maintainer [permissions](../../user/permissions.md). -1. Navigate to **Settings > Operations > Incidents** and expand **Incidents**. -1. Select the **PagerDuty integration** tab: +## Incident Actions -  +There are different actions avilable to help triage and respond to incidents. -1. Activate the integration, and save the changes in GitLab. -1. Copy the value of **Webhook URL** for use in a later step. -1. Follow the steps described in the - [PagerDuty documentation](https://support.pagerduty.com/docs/webhooks) - to add the webhook URL to a PagerDuty webhook integration. +### Assign incidents -To confirm the integration is successful, trigger a test incident from PagerDuty to -confirm that a GitLab issue is created from the incident. +Assign incidents to users that are actively responding. Select **Edit** in the right-hand side bar to select or deselect assignees. -## Incident details +### Change severity -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/230847) in GitLab 13.4. +See [Incident List](#incident-list) for a full description of the severities available. Select **Edit** in the right-hand side bar to change the severity of an incident. -### Summary +### Add a to do -The summary section for incidents provides both critical details about and the -contents of the issue template (if one was used). The highlighted bar at the top -of the incident displays from left to right: the link to the original alert, the -alert start time, and the event count. Beneath the highlight bar, GitLab -displays a summary that includes the following fields: +Add a to-do for incidents that you want to track in your to-do list. Clicke the **Add a to do** button at the top of the right-hand side bar to add a to do. -- Start time -- Severity -- `full_query` -- Monitoring tool +### Manage incidents from Slack -### Alert details +Slack slash commands allow you to control GitLab and view GitLab content without leaving Slack. -Incidents show the details of linked alerts in a separate tab. To populate this -tab, the incident must have been created with a linked alert. Incidents -[created automatically](#configure-incidents) from alerts will have this -field populated. +Learn how to [set up Slack slash commands](../../user/project/integrations/slack_slash_commands.md) +and how to [use the available slash commands](../../integration/slash_commands.md). - +### Associate Zoom calls + +GitLab enables you to [associate a Zoom meeting with an issue](../../user/project/issues/associate_zoom_meeting.md) +for synchronous communication during incident management. After starting a Zoom +call for an incident, you can associate the conference call with an issue. Your +team members can join the Zoom call without requesting a link. + +### Embed metrics in incidents + +You can embed metrics anywhere [GitLab Markdown](../../user/markdown.md) is +used, such as descriptions, comments on issues, and merge requests. Embedding +metrics helps you share them when discussing incidents or performance issues. +You can output the dashboard directly into any issue, merge request, epic, or +any other Markdown text field in GitLab by +[copying and pasting the link to the metrics dashboard](../metrics/embed.md#embedding-gitlab-managed-kubernetes-metrics). + +You can embed both [GitLab-hosted metrics](../metrics/embed.md) and +[Grafana metrics](../metrics/embed_grafana.md) in incidents and issue +templates. diff --git a/doc/operations/incident_management/index.md b/doc/operations/incident_management/index.md index 28e69a6bbfe..60571c03d74 100644 --- a/doc/operations/incident_management/index.md +++ b/doc/operations/incident_management/index.md @@ -8,74 +8,11 @@ info: To determine the technical writer assigned to the Stage/Group associated w > [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/2877) in GitLab 13.0. -Incident Management enables developers to easily discover and view the alerts -generated by their application. By surfacing alert information where the code is -being developed, efficiency and awareness can be increased. - -GitLab offers solutions for handling incidents in your applications and services, -such as [setting up Prometheus alerts](#configure-prometheus-alerts), -[displaying metrics](./alert_details.md#embed-metrics-in-incidents-and-issues), and sending notifications. - -## Alert notifications - -### Slack Notifications - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/216326) in GitLab 13.1. - -You can be alerted via a Slack message when a new alert has been received. - -See the [Slack Notifications Service docs](../../user/project/integrations/slack.md) for information on how to set this up. - -### Notify developers of alerts - -GitLab can react to the alerts triggered from your applications and services -by creating issues and alerting developers through email. By default, GitLab -sends these emails to [owners and maintainers](../../user/permissions.md) of the project. -These emails contain details of the alert, and a link for more information. - -To send separate email notifications to users with -[Developer permissions](../../user/permissions.md), see -[Configure incidents](./incidents.md#configure-incidents). - -## Configure Prometheus alerts - -You can set up Prometheus alerts in: - -- [GitLab-managed Prometheus](../metrics/alerts.md) installations. -- [Self-managed Prometheus](../metrics/alerts.md#external-prometheus-instances) installations. - -Prometheus alerts are created by the special Alert Bot user. You can't remove this -user, but it does not count toward your license limit. - -## Configure external generic alerts - -GitLab can accept alerts from any source through a generic webhook receiver. -When [configuring the generic alerts integration](./generic_alerts.md), GitLab -creates a unique endpoint which receives a JSON-formatted, customizable payload. - -After configuration, you can manage your alerts using either the -[alerts section](./alerts.md) or the [alert details section](./alert_details.md). - -## Integrate incidents with Slack - -Slack slash commands allow you to control GitLab and view GitLab content without leaving Slack. - -Learn how to [set up Slack slash commands](../../user/project/integrations/slack_slash_commands.md) -and how to [use the available slash commands](../../integration/slash_commands.md). - -## Integrate issues with Zoom - -GitLab enables you to [associate a Zoom meeting with an issue](../../user/project/issues/associate_zoom_meeting.md) -for synchronous communication during incident management. After starting a Zoom -call for an incident, you can associate the conference call with an issue. Your -team members can join the Zoom call without requesting a link. - -## More information - -For information about GitLab and incident management, see: - -- [Generic alerts](generic_alerts.md) -- [Alerts](alerts.md) -- [Alert details](alert_details.md) -- [Incidents](incidents.md) -- [Status page](status_page.md) +Incident Management enables developers to easily triage and view the alerts and incidents +generated by their application. By surfacing alerts and incidents where the code is +being developed, efficiency and awareness can be increased. Check out the following sections for more information: + +- [Integrate your monitoring tools](alert_integrations.md). +- Receive [notifications](alert_notifications.md) for triggered alerts. +- Triage [Alerts](alerts.md) and [Incidents](incidents.md). +- Inform stakeholders with [Status Page](status_page.md). diff --git a/doc/operations/incident_management/integrations.md b/doc/operations/incident_management/integrations.md new file mode 100644 index 00000000000..9d4f32ab2bf --- /dev/null +++ b/doc/operations/incident_management/integrations.md @@ -0,0 +1,16 @@ +--- +stage: Monitor +group: Health +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Integrations + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/245331) in [GitLab Core](https://about.gitlab.com/pricing/) 13.5. + +With Maintainer or higher [permissions](../../user/permissions.md), you can view +the list of configured alerts integrations by navigating to +**Settings > Operations** in your project's sidebar menu, and expanding **Alerts** section. +The list displays the integration name, type, and status (enabled or disabled): + + diff --git a/doc/operations/incident_management/status_page.md b/doc/operations/incident_management/status_page.md index 9db3593caec..e5d0ae1ddbb 100644 --- a/doc/operations/incident_management/status_page.md +++ b/doc/operations/incident_management/status_page.md @@ -4,7 +4,7 @@ group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# GitLab Status Page **(ULTIMATE)** +# Status Page > [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/2479) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.10. @@ -25,7 +25,7 @@ Clicking an incident displays a detail page with more information about a partic valid image extension. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/205166) in GitLab 13.1. - A chronological ordered list of updates to the incident. -## Set up a GitLab Status Page +## Set up a Status Page To configure a GitLab Status Page you must: @@ -37,11 +37,10 @@ To configure a GitLab Status Page you must: ### Configure GitLab with cloud provider information -To provide GitLab with the AWS account information needed to push content to your Status Page: - -NOTE: **Note:** Only AWS S3 is supported as a deploy target. +To provide GitLab with the AWS account information needed to push content to your Status Page: + 1. Sign into GitLab as a user with Maintainer or greater [permissions](../../user/permissions.md). 1. Navigate to **{settings}** **Settings > Operations**. Next to **Status Page**, click **Expand**. @@ -74,8 +73,6 @@ the necessary CI/CD variables to deploy the Status Page to AWS S3: 1. Scroll to **Variables**, and click **Expand**. 1. Add the following variables from your Amazon Console: - `S3_BUCKET_NAME` - The name of the Amazon S3 bucket. - - NOTE: **Note:** If no bucket with the provided name exists, the first pipeline run creates one and configures it for [static website hosting](https://docs.aws.amazon.com/AmazonS3/latest/dev/HostingWebsiteOnS3Setup.html). @@ -128,10 +125,7 @@ To publish an incident: 1. Create an issue in the project you enabled the GitLab Status Page settings in. 1. A [project or group owner](../../user/permissions.md) must use the `/publish` [quick action](../../user/project/quick_actions.md) to publish the - issue to the GitLab Status Page. - - NOTE: **Note:** - Confidential issues can't be published. + issue to the GitLab Status Page. Confidential issues can't be published. A background worker publishes the issue onto the Status Page using the credentials you provided during setup. As part of publication, GitLab will: diff --git a/doc/operations/index.md b/doc/operations/index.md index 7ab34502277..8488f939893 100644 --- a/doc/operations/index.md +++ b/doc/operations/index.md @@ -1,6 +1,6 @@ --- stage: Monitor -group: APM +group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- @@ -35,7 +35,7 @@ using metrics and logs, and promote the critical alerts to incidents. Are your alerts too noisy? Alerts configured on GitLab metrics can configured and fine-tuned in GitLab immediately following a fire-fight. -- [Manage alerts and incidents](../user/incident_management/index.md) in GitLab. +- [Manage alerts and incidents](incident_management/index.md) in GitLab. - [Configure alerts for metrics](metrics/alerts.md#set-up-alerts-for-prometheus-metrics) in GitLab. - Create a [status page](incident_management/status_page.md) to communicate efficiently to your users during an incident. diff --git a/doc/operations/metrics/alerts.md b/doc/operations/metrics/alerts.md index 5b880ab9746..79e3bdbd69c 100644 --- a/doc/operations/metrics/alerts.md +++ b/doc/operations/metrics/alerts.md @@ -1,6 +1,6 @@ --- stage: Monitor -group: APM +group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- @@ -15,7 +15,7 @@ your team when environment performance falls outside of the boundaries you set. ## Managed Prometheus instances -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/6590) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.2 for [custom metrics](index.md#adding-custom-metrics), and GitLab 11.3 for [library metrics](../../user/project/integrations/prometheus_library/metrics.md). +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/6590) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.2 for [custom metrics](index.md#adding-custom-metrics), and GitLab 11.3 for [library metrics](../../user/project/integrations/prometheus_library/index.md). For managed Prometheus instances using auto configuration, you can [configure alerts for metrics](index.md#adding-custom-metrics) directly in the @@ -70,14 +70,14 @@ receivers: bearer_token: 9e1cbfcd546896a9ea8be557caf13a76 send_resolved: true url: http://192.168.178.31:3001/root/manual_prometheus/prometheus/alerts/notify.json - ... + # Rest of configuration omitted + # ... ``` For GitLab to associate your alerts with an [environment](../../ci/environments/index.md), you must configure a `gitlab_environment_name` label on the alerts you set up in Prometheus. The value of this should match the name of your environment in GitLab. -NOTE: **Note:** In GitLab versions 13.1 and greater, you can configure your manually configured Prometheus server to use the [Generic alerts integration](../incident_management/generic_alerts.md). diff --git a/doc/operations/metrics/dashboards/default.md b/doc/operations/metrics/dashboards/default.md index f086d7737bd..11e96114f38 100644 --- a/doc/operations/metrics/dashboards/default.md +++ b/doc/operations/metrics/dashboards/default.md @@ -1,6 +1,6 @@ --- stage: Monitor -group: APM +group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- @@ -25,7 +25,6 @@ metrics about the [deployed application](../index.md#configure-prometheus-to-gat ## Kubernetes pod health dashboard -NOTE: **Note:** This dashboard requires Kubernetes v1.14 or higher, due to the [change in metric labels](https://github.com/kubernetes/kubernetes/pull/69099) in Kubernetes 1.14. diff --git a/doc/operations/metrics/dashboards/develop.md b/doc/operations/metrics/dashboards/develop.md index b621f5fd727..9254bfe075f 100644 --- a/doc/operations/metrics/dashboards/develop.md +++ b/doc/operations/metrics/dashboards/develop.md @@ -1,6 +1,6 @@ --- stage: Monitor -group: APM +group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- diff --git a/doc/operations/metrics/dashboards/img/metrics_dashboard_panel_preview_v13_3.png b/doc/operations/metrics/dashboards/img/metrics_dashboard_panel_preview_v13_3.png Binary files differindex 4f6d3b3dfa4..3c3203265e1 100644 --- a/doc/operations/metrics/dashboards/img/metrics_dashboard_panel_preview_v13_3.png +++ b/doc/operations/metrics/dashboards/img/metrics_dashboard_panel_preview_v13_3.png diff --git a/doc/operations/metrics/dashboards/index.md b/doc/operations/metrics/dashboards/index.md index ffcb7dc92c6..4aa340a9e59 100644 --- a/doc/operations/metrics/dashboards/index.md +++ b/doc/operations/metrics/dashboards/index.md @@ -1,6 +1,6 @@ --- stage: Monitor -group: APM +group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- @@ -14,7 +14,6 @@ includes a few key metrics, but you can also define your own custom dashboards. You may create a [new dashboard from scratch](#add-a-new-dashboard-to-your-project) or [duplicate a GitLab-defined Prometheus dashboard](#duplicate-a-gitlab-defined-dashboard). -NOTE: **Note:** The metrics as defined below do not support alerts, unlike [custom metrics](../index.md#adding-custom-metrics). @@ -51,16 +50,16 @@ To create a new dashboard from the command line: - group: 'Group Title' panels: - type: area-chart - title: "Chart Title" - y_label: "Y-Axis" + title: 'Chart Title' + y_label: 'Y-Axis' y_axis: format: number precision: 0 metrics: - id: my_metric_id query_range: 'http_requests_total' - label: "Instance: {{instance}}, method: {{method}}" - unit: "count" + label: 'Instance: {{instance}}, method: {{method}}' + unit: 'count' ``` 1. Save the file, commit, and push to your repository. The file must be present in your **default** branch. @@ -86,7 +85,7 @@ with the **Add Panel** page: 1. Click **Add panel** in the **{ellipsis_v}** **More actions** menu. NOTE: **Note:** - You can add panel only to custom dashboards. + You can only add panels to custom dashboards.  1. In the **Define and preview panel** section, paste in the YAML you want to @@ -100,16 +99,12 @@ with the **Add Panel** page: > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/37238) in GitLab 12.7. > - From [GitLab 12.8 onwards](https://gitlab.com/gitlab-org/gitlab/-/issues/39505), custom metrics are also duplicated when you duplicate a dashboard. -You can save a complete copy of a GitLab defined dashboard along with all custom metrics added to it. +You can save a complete copy of a GitLab-defined dashboard along with all custom metrics added to it. The resulting `.yml` file can be customized and adapted to your project. You can decide to save the dashboard `.yml` file in the project's **default** branch or in a -new branch. +new branch. To duplicate a GitLab-defined dashboard: 1. Click **Duplicate current dashboard** in the **{ellipsis_v}** **More actions** menu. - - NOTE: **Note:** - You can duplicate only GitLab-defined dashboards. - 1. Enter the filename and other information, such as the new commit's message, and click **Duplicate**. 1. Select a branch to add your dashboard to: - *If you select your **default** branch,* the new dashboard becomes immediately available. diff --git a/doc/operations/metrics/dashboards/panel_types.md b/doc/operations/metrics/dashboards/panel_types.md index b2cbdcb88d9..fd9d2bf7899 100644 --- a/doc/operations/metrics/dashboards/panel_types.md +++ b/doc/operations/metrics/dashboards/panel_types.md @@ -1,6 +1,6 @@ --- stage: Monitor -group: APM +group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- @@ -17,16 +17,16 @@ dashboard: 'Dashboard Title' panel_groups: - group: 'Group Title' panels: - - type: area-chart # or line-chart + - type: area-chart # or line-chart title: 'Area Chart Title' - y_label: "Y-Axis" + y_label: 'Y-Axis' y_axis: format: number precision: 0 metrics: - id: area_http_requests_total query_range: 'http_requests_total' - label: "Instance: {{instance}}, Method: {{method}}" + label: 'Instance: {{instance}}, Method: {{method}}' unit: "count" ``` @@ -55,23 +55,23 @@ panel_groups: - group: 'Group Title' panels: - type: anomaly-chart - title: "Chart Title" + title: 'Chart Title' y_label: "Y-Axis" metrics: - id: anomaly_requests_normal query_range: 'http_requests_total' - label: "# of Requests" - unit: "count" + label: '# of Requests' + unit: 'count' metrics: - id: anomaly_requests_upper_limit query_range: 10000 - label: "Max # of requests" - unit: "count" + label: 'Max # of requests' + unit: 'count' metrics: - id: anomaly_requests_lower_limit query_range: 2000 - label: "Min # of requests" - unit: "count" + label: 'Min # of requests' + unit: 'count' ``` Note the following properties: @@ -93,13 +93,13 @@ panel_groups: - group: 'Group title' panels: - type: bar - title: "Http Handlers" + title: 'HTTP Handlers' x_label: 'Response Size' y_axis: - name: "Handlers" + name: 'Handlers' metrics: - id: prometheus_http_response_size_bytes_bucket - query_range: "sum(increase(prometheus_http_response_size_bytes_bucket[1d])) by (handler)" + query_range: 'sum(increase(prometheus_http_response_size_bytes_bucket[1d])) by (handler)' unit: 'Bytes' ``` @@ -121,13 +121,13 @@ dashboard: 'Dashboard Title' panel_groups: - group: 'Group title' panels: - - title: "Column" - type: "column" + - title: 'Column' + type: 'column' metrics: - id: 1024_memory query: 'avg(sum(container_memory_usage_bytes{container_name!="POD",pod_name=~"^%{ci_environment_slug}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="%{kube_namespace}"}) by (job)) without (job) / count(avg(container_memory_usage_bytes{container_name!="POD",pod_name=~"^%{ci_environment_slug}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="%{kube_namespace}"}) without (job)) /1024/1024' unit: MB - label: "Memory Usage" + label: 'Memory Usage' ``` Note the following properties: @@ -153,19 +153,19 @@ panel_groups: priority: 5 panels: - type: 'stacked-column' - title: "Stacked column" - y_label: "y label" + title: 'Stacked column' + y_label: 'y label' x_label: 'x label' metrics: - id: memory_1 query_range: 'memory_query' - label: "memory query 1" - unit: "count" + label: 'memory query 1' + unit: 'count' series_name: 'group 1' - id: memory_2 query_range: 'memory_query_2' - label: "memory query 2" - unit: "count" + label: 'memory query 2' + unit: 'count' series_name: 'group 2' ``` @@ -185,13 +185,13 @@ dashboard: 'Dashboard Title' panel_groups: - group: 'Group Title' panels: - - title: "Single Stat" - type: "single-stat" + - title: 'Single Stat' + type: 'single-stat' metrics: - id: 10 query: 'max(go_memstats_alloc_bytes{job="prometheus"})' unit: MB - label: "Total" + label: 'Total' ``` Note the following properties: @@ -215,14 +215,14 @@ dashboard: 'Dashboard Title' panel_groups: - group: 'Group Title' panels: - - title: "Single Stat" - type: "single-stat" + - title: 'Single Stat' + type: 'single-stat' max_value: 100 metrics: - id: 10 query: 'max(go_memstats_alloc_bytes{job="prometheus"})' unit: '%' - label: "Total" + label: 'Total' ``` For example, if you have a query value of `53.6`, adding `%` as the unit results in a single stat value of `53.6%`, but if the maximum expected value of the query is `120`, the value would be `44.6%`. Adding the `max_value` causes the correct percentage value to display. @@ -242,15 +242,15 @@ dashboard: 'Dashboard Title' panel_groups: - group: 'Group Title' panels: - - title: "Gauge" - type: "gauge" + - title: 'Gauge' + type: 'gauge' min_value: 0 max_value: 1000 split: 5 thresholds: values: [60, 90] - mode: "percentage" - format: "kilobytes" + mode: 'percentage' + format: 'kilobytes' metrics: - id: 10 query: 'floor(max(prometheus_http_response_size_bytes_bucket)/1000)' @@ -289,13 +289,13 @@ dashboard: 'Dashboard Title' panel_groups: - group: 'Group Title' panels: - - title: "Heatmap" - type: "heatmap" + - title: 'Heatmap' + type: 'heatmap' metrics: - id: 10 query: 'sum(rate(nginx_upstream_responses_total{upstream=~"%{kube_namespace}-%{ci_environment_slug}-.*"}[60m])) by (status_code)' unit: req/sec - label: "Status code" + label: 'Status code' ``` Note the following properties: diff --git a/doc/operations/metrics/dashboards/settings.md b/doc/operations/metrics/dashboards/settings.md index a4aef6b1674..aa0b9a81771 100644 --- a/doc/operations/metrics/dashboards/settings.md +++ b/doc/operations/metrics/dashboards/settings.md @@ -1,6 +1,6 @@ --- stage: Monitor -group: APM +group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- diff --git a/doc/operations/metrics/dashboards/templating_variables.md b/doc/operations/metrics/dashboards/templating_variables.md index 71025d41281..1c0b05b0e53 100644 --- a/doc/operations/metrics/dashboards/templating_variables.md +++ b/doc/operations/metrics/dashboards/templating_variables.md @@ -1,6 +1,6 @@ --- stage: Monitor -group: APM +group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- diff --git a/doc/operations/metrics/dashboards/variables.md b/doc/operations/metrics/dashboards/variables.md index 22c8814e8bd..2103f8e66db 100644 --- a/doc/operations/metrics/dashboards/variables.md +++ b/doc/operations/metrics/dashboards/variables.md @@ -1,6 +1,6 @@ --- stage: Monitor -group: APM +group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- @@ -16,7 +16,10 @@ Queries that continue to use the old format will show no data. ## Predefined variables -GitLab supports a limited set of [CI variables](../../../ci/variables/README.md) in the Prometheus query. This is particularly useful for identifying a specific environment, for example with `ci_environment_slug`. The supported variables are: +GitLab supports a limited set of [CI variables](../../../ci/variables/README.md) +in the Prometheus query. This is particularly useful for identifying a specific +environment, for example with `ci_environment_slug`. Variables for Prometheus queries +must be lowercase. The supported variables are: - `environment_filter` - `ci_environment_slug` @@ -27,9 +30,6 @@ GitLab supports a limited set of [CI variables](../../../ci/variables/README.md) - `ci_environment_name` - `__range` -NOTE: **Note:** -Variables for Prometheus queries must be lowercase. - ### environment_filter `environment_filter` is automatically expanded to `container_name!="POD",environment="ENVIRONMENT_NAME"` diff --git a/doc/operations/metrics/dashboards/yaml.md b/doc/operations/metrics/dashboards/yaml.md index f92ba4079e9..c3523327c51 100644 --- a/doc/operations/metrics/dashboards/yaml.md +++ b/doc/operations/metrics/dashboards/yaml.md @@ -1,6 +1,6 @@ --- stage: Monitor -group: APM +group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- @@ -45,7 +45,6 @@ Read the documentation on [links](index.md#add-related-links-to-custom-dashboard Dashboards display panel groups in the order they are listed in the dashboard YAML file. -NOTE: **Note:** In GitLab versions 13.3 and below, panel groups were ordered by a `priority` key, which is no longer used. @@ -60,7 +59,6 @@ Panels in a panel group are laid out in rows consisting of two panels per row. A Dashboards display panels in the order they are listed in the dashboard YAML file. -NOTE: **Note:** In GitLab versions 13.3 and below, panels were ordered by a `weight` key, which is no longer used. @@ -103,8 +101,8 @@ When a static label is used and a query returns multiple time series, then all t metrics: - id: my_metric_id query_range: 'http_requests_total' - label: "Time Series" - unit: "count" + label: 'Time Series' + unit: 'count' ``` This may render a legend like this: @@ -117,8 +115,8 @@ For labels to be more explicit, using variables that reflect time series labels metrics: - id: my_metric_id query_range: 'http_requests_total' - label: "Instance: {{instance}}, method: {{method}}" - unit: "count" + label: 'Instance: {{instance}}, method: {{method}}' + unit: 'count' ``` The resulting rendered legend will look like this: @@ -131,8 +129,8 @@ There is also a shorthand value for dynamic dashboard labels that make use of on metrics: - id: my_metric_id query_range: 'http_requests_total' - label: "Method" - unit: "count" + label: 'Method' + unit: 'count' ``` This works by lowercasing the value of `label` and, if there are more words separated by spaces, replacing those spaces with an underscore (`_`). The transformed value is then checked against the labels of the time series returned by the Prometheus query. If a time series label is found that is equal to the transformed value, then the label value will be used and rendered in the legend like this: diff --git a/doc/operations/metrics/dashboards/yaml_number_format.md b/doc/operations/metrics/dashboards/yaml_number_format.md index 1a8bd6f4257..db1606faf8d 100644 --- a/doc/operations/metrics/dashboards/yaml_number_format.md +++ b/doc/operations/metrics/dashboards/yaml_number_format.md @@ -1,6 +1,6 @@ --- stage: Monitor -group: APM +group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- diff --git a/doc/operations/metrics/embed.md b/doc/operations/metrics/embed.md index fcf9679d164..c0b30f18156 100644 --- a/doc/operations/metrics/embed.md +++ b/doc/operations/metrics/embed.md @@ -1,6 +1,6 @@ --- stage: Monitor -group: APM +group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- @@ -17,8 +17,7 @@ metrics to others, and you want to have relevant information directly available. > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/29691) in GitLab 12.2. -NOTE: **Note:** -Requires [Kubernetes](../../user/project/integrations/prometheus_library/kubernetes.md) metrics. +This feature requires [Kubernetes](../../user/project/integrations/prometheus_library/kubernetes.md) metrics. Note: **Note:** In GitLab versions 13.3 and earlier, metrics dashboard links were in the form diff --git a/doc/operations/metrics/embed_grafana.md b/doc/operations/metrics/embed_grafana.md index 2843a4319a8..532bf150777 100644 --- a/doc/operations/metrics/embed_grafana.md +++ b/doc/operations/metrics/embed_grafana.md @@ -1,6 +1,6 @@ --- stage: Monitor -group: APM +group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- @@ -12,14 +12,13 @@ Grafana metrics can be embedded in [GitLab Flavored Markdown](../../user/markdow You can embed live [Grafana](https://docs.gitlab.com/omnibus/settings/grafana.html) charts in issues as a -[direct linked rendered image](https://grafana.com/docs/grafana/latest/reference/share_panel/#direct-link-rendered-image). -The **Direct link rendered image** sharing dialog within Grafana provides the link: +[direct linked rendered image](https://grafana.com/docs/grafana/latest/reference/share_panel/#direct-link-rendered-image). Your Grafana instance must be available to the +target user, either as a public dashboard or on the same network. The +**Direct link rendered image** sharing dialog within Grafana provides the link:  -NOTE: **Note:** -For this embed to display correctly, the Grafana instance must be available to the -target user, either as a public dashboard or on the same network. +For this embed to display correctly, the Copy the link and add an image tag as [inline HTML](../../user/markdown.md#inline-html) in your Markdown. You can tweak the query parameters to meet your needs, such as diff --git a/doc/operations/metrics/index.md b/doc/operations/metrics/index.md index 742e6acef0e..39d03ded373 100644 --- a/doc/operations/metrics/index.md +++ b/doc/operations/metrics/index.md @@ -1,6 +1,6 @@ --- stage: Monitor -group: APM +group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- diff --git a/doc/operations/product_analytics.md b/doc/operations/product_analytics.md index 8f660a16b47..65dced97002 100644 --- a/doc/operations/product_analytics.md +++ b/doc/operations/product_analytics.md @@ -1,6 +1,6 @@ --- -stage: Monitor -group: APM +stage: Growth +group: Product Analytics info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- diff --git a/doc/operations/tracing.md b/doc/operations/tracing.md index 07f60c37f1b..b6ef552772e 100644 --- a/doc/operations/tracing.md +++ b/doc/operations/tracing.md @@ -1,12 +1,13 @@ --- stage: Monitor -group: APM +group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# Tracing **(ULTIMATE)** +# Tracing -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/7903) in GitLab Ultimate 11.5. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/7903) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.5. +> - [Moved to GitLab Core](https://gitlab.com/gitlab-org/gitlab/-/issues/42645) in 13.5. Tracing provides insight into the performance and health of a deployed application, tracking each function or microservice which handles a given request. diff --git a/doc/pages/README.md b/doc/pages/README.md deleted file mode 100644 index c67847f1a83..00000000000 --- a/doc/pages/README.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../user/project/pages/index.md' ---- - -This document was moved to [another location](../user/project/pages/index.md). diff --git a/doc/pages/administration.md b/doc/pages/administration.md deleted file mode 100644 index 015dd54ec7f..00000000000 --- a/doc/pages/administration.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../administration/pages/index.md' ---- - -This document was moved to [another location](../administration/pages/index.md). diff --git a/doc/pages/getting_started_part_one.md b/doc/pages/getting_started_part_one.md deleted file mode 100644 index a0feed0b477..00000000000 --- a/doc/pages/getting_started_part_one.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../user/project/pages/getting_started_part_one.md' ---- - -This document was moved to [another location](../user/project/pages/getting_started_part_one.md). diff --git a/doc/pages/getting_started_part_three.md b/doc/pages/getting_started_part_three.md deleted file mode 100644 index 31a01a6c83b..00000000000 --- a/doc/pages/getting_started_part_three.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../user/project/pages/custom_domains_ssl_tls_certification/index.md' ---- - -This document was moved to [another location](../user/project/pages/custom_domains_ssl_tls_certification/index.md). diff --git a/doc/pages/getting_started_part_two.md b/doc/pages/getting_started_part_two.md deleted file mode 100644 index 05353c171fc..00000000000 --- a/doc/pages/getting_started_part_two.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../user/project/pages/getting_started_part_two.md' ---- - -This document was moved to [another location](../user/project/pages/getting_started_part_two.md). diff --git a/doc/policy/maintenance.md b/doc/policy/maintenance.md index d2e556f3c8d..77b4b22e6a8 100644 --- a/doc/policy/maintenance.md +++ b/doc/policy/maintenance.md @@ -115,9 +115,9 @@ Please see the table below for some examples: | Target version | Your version | Recommended upgrade path | Note | | --------------------- | ------------ | ------------------------ | ---- | -| `13.2.3` | `11.5.0` | `11.5.0` -> `11.11.8` -> `12.0.12` -> `12.10.14` -> `13.0.12` -> `13.2.3` | Four intermediate versions are required: the final `11.11`, `12.0`, and `12.10` releases, plus `13.0`. | -| `13.0.12` | `11.10.8` | `11.10.5` -> `11.11.8` -> `12.0.12` -> `12.10.14` -> `13.0.12` | Three intermediate versions are required: `11.11`, `12.0`, and `12.10`. | -| `12.10.14` | `11.3.4` | `11.3.4` -> `11.11.8` -> `12.0.12` -> `12.10.14` | Two intermediate versions are required: `11.11` and `12.0` | +| `13.4.3` | `12.9.2` | `12.9.2` -> `12.10.14` -> `13.0.14` -> `13.4.3` | Two intermediate versions are required: the final `12.10` release, plus `13.0`. | +| `13.2.10` | `11.5.0` | `11.5.0` -> `11.11.8` -> `12.0.12` -> `12.10.14` -> `13.0.14` -> `13.2.10` | Four intermediate versions are required: the final `11.11`, `12.0`, and `12.10` releases, plus `13.0`. | +| `12.10.14` | `11.3.4` | `11.3.4` -> `11.11.8` -> `12.0.12` -> `12.10.14` | Two intermediate versions are required: the final `11.11` release and `12.0.12` | | `12.9.5` | `10.4.5` | `10.4.5` -> `10.8.7` -> `11.11.8` -> `12.0.12` -> `12.9.5` | Three intermediate versions are required: `10.8`, `11.11`, and `12.0`, then `12.9.5` | | `12.2.5` | `9.2.6` | `9.2.6` -> `9.5.10` -> `10.8.7` -> `11.11.8` -> `12.0.12` -> `12.2.5` | Four intermediate versions are required: `9.5`, `10.8`, `11.11`, `12.0`, then `12.2`. | | `11.3.4` | `8.13.4` | `8.13.4` -> `8.17.7` -> `9.5.10` -> `10.8.7` -> `11.3.4` | `8.17.7` is the last version in version 8, `9.5.10` is the last version in version 9, `10.8.7` is the last version in version 10. | diff --git a/doc/profile/README.md b/doc/profile/README.md deleted file mode 100644 index 4932cf33b87..00000000000 --- a/doc/profile/README.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../user/profile/index.md' ---- - -This document was moved to [user/profile/account](../user/profile/index.md). diff --git a/doc/profile/preferences.md b/doc/profile/preferences.md deleted file mode 100644 index cf99bd61f5d..00000000000 --- a/doc/profile/preferences.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../user/profile/preferences.md' ---- - -This document was moved to [another location](../user/profile/preferences.md). diff --git a/doc/profile/two_factor_authentication.md b/doc/profile/two_factor_authentication.md deleted file mode 100644 index 453ac833f59..00000000000 --- a/doc/profile/two_factor_authentication.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../user/profile/account/two_factor_authentication.md' ---- - -This document was moved to [user/profile/account](../user/profile/account/two_factor_authentication.md). diff --git a/doc/project_services/bamboo.md b/doc/project_services/bamboo.md deleted file mode 100644 index b1d37898516..00000000000 --- a/doc/project_services/bamboo.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../user/project/integrations/bamboo.md' ---- - -This document was moved to [another location](../user/project/integrations/bamboo.md). diff --git a/doc/project_services/bugzilla.md b/doc/project_services/bugzilla.md deleted file mode 100644 index 17dff538c0e..00000000000 --- a/doc/project_services/bugzilla.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../user/project/integrations/bugzilla.md' ---- - -This document was moved to [another location](../user/project/integrations/bugzilla.md). diff --git a/doc/project_services/emails_on_push.md b/doc/project_services/emails_on_push.md deleted file mode 100644 index a7d91934ce9..00000000000 --- a/doc/project_services/emails_on_push.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../user/project/integrations/emails_on_push.md' ---- - -This document was moved to [another location](../user/project/integrations/emails_on_push.md). diff --git a/doc/project_services/hipchat.md b/doc/project_services/hipchat.md deleted file mode 100644 index a2fbbd5cce5..00000000000 --- a/doc/project_services/hipchat.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../user/project/integrations/hipchat.md' ---- - -This document was moved to [another location](../user/project/integrations/hipchat.md). diff --git a/doc/project_services/irker.md b/doc/project_services/irker.md deleted file mode 100644 index 70e46b1b364..00000000000 --- a/doc/project_services/irker.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../user/project/integrations/irker.md' ---- - -This document was moved to [another location](../user/project/integrations/irker.md). diff --git a/doc/project_services/jira.md b/doc/project_services/jira.md deleted file mode 100644 index 37eba25fb5a..00000000000 --- a/doc/project_services/jira.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../user/project/integrations/jira.md' ---- - -This document was moved to [another location](../user/project/integrations/jira.md). diff --git a/doc/project_services/kubernetes.md b/doc/project_services/kubernetes.md deleted file mode 100644 index 585c5ddb002..00000000000 --- a/doc/project_services/kubernetes.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../user/project/integrations/kubernetes.md' ---- - -This document was moved to [another location](../user/project/integrations/kubernetes.md). diff --git a/doc/project_services/mattermost.md b/doc/project_services/mattermost.md deleted file mode 100644 index 78888395031..00000000000 --- a/doc/project_services/mattermost.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../user/project/integrations/mattermost.md' ---- - -This document was moved to [another location](../user/project/integrations/mattermost.md). diff --git a/doc/project_services/mattermost_slash_commands.md b/doc/project_services/mattermost_slash_commands.md deleted file mode 100644 index 0c2774d95e0..00000000000 --- a/doc/project_services/mattermost_slash_commands.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../user/project/integrations/mattermost_slash_commands.md' ---- - -This document was moved to [another location](../user/project/integrations/mattermost_slash_commands.md). diff --git a/doc/project_services/project_services.md b/doc/project_services/project_services.md deleted file mode 100644 index 2d58e0ca065..00000000000 --- a/doc/project_services/project_services.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../user/project/integrations/overview.md' ---- - -This document was moved to [another location](../user/project/integrations/overview.md). diff --git a/doc/project_services/redmine.md b/doc/project_services/redmine.md deleted file mode 100644 index 141c72d6b6b..00000000000 --- a/doc/project_services/redmine.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../user/project/integrations/redmine.md' ---- - -This document was moved to [another location](../user/project/integrations/redmine.md). diff --git a/doc/project_services/services_templates.md b/doc/project_services/services_templates.md deleted file mode 100644 index 8b2c85802de..00000000000 --- a/doc/project_services/services_templates.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../user/project/integrations/services_templates.md' ---- - -This document was moved to [another location](../user/project/integrations/services_templates.md). diff --git a/doc/project_services/slack.md b/doc/project_services/slack.md deleted file mode 100644 index 815032a08d5..00000000000 --- a/doc/project_services/slack.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../user/project/integrations/slack.md' ---- - -This document was moved to [another location](../user/project/integrations/slack.md). diff --git a/doc/project_services/slack_slash_commands.md b/doc/project_services/slack_slash_commands.md deleted file mode 100644 index caae4d2ba4b..00000000000 --- a/doc/project_services/slack_slash_commands.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../user/project/integrations/slack_slash_commands.md' ---- - -This document was moved to [another location](../user/project/integrations/slack_slash_commands.md). diff --git a/doc/push_rules/push_rules.md b/doc/push_rules/push_rules.md index 1643c96d229..d2ec157788a 100644 --- a/doc/push_rules/push_rules.md +++ b/doc/push_rules/push_rules.md @@ -97,7 +97,7 @@ The following options are available. | Restrict by commit message (negative match)| **Starter** 11.1 | Only commit messages that do not match this regular expression are allowed to be pushed. Leave empty to allow any commit message. Uses multiline mode, which can be disabled using `(?-m)`. | | Restrict by branch name | **Starter** 9.3 | Only branch names that match this regular expression are allowed to be pushed. Leave empty to allow any branch name. | | Restrict by commit author's email | **Starter** 7.10 | Only commit author's email that match this regular expression are allowed to be pushed. Leave empty to allow any email. | -| Prohibited file names | **Starter** 7.10 | Any committed filenames that match this regular expression are not allowed to be pushed. Leave empty to allow any filenames. | +| Prohibited file names | **Starter** 7.10 | Any committed filenames that match this regular expression and do not already exist in the repository are not allowed to be pushed. Leave empty to allow any filenames. See [common examples](#prohibited-file-names). | | Maximum file size | **Starter** 7.12 | Pushes that contain added or updated files that exceed this file size (in MB) are rejected. Set to 0 to allow files of any size. Files tracked by Git LFS are exempted. | TIP: **Tip:** @@ -178,6 +178,44 @@ pry.history bash_history ``` +## Prohibited file names + +> Introduced in [GitLab Starter](https://about.gitlab.com/pricing/) 7.10. + +Each file name contained in a Git push is compared to the regular expression in this field. Filenames in Git consist of both the file's name and any directory that may precede it. A singular regular expression can contain multiple independent matches used as exclusions. File names can be broadly matched to any location in the repository, or restricted to specific locations. Filenames can also be partial matches used to exclude file types by extension. + +The following examples make use of regex string boundary characters which match the beginning of a string (`^`), and the end (`$`). They also include instances where either the directory path or the filename can include `.` or `/`. Both of these special regex characters have to be escaped with a backslash `\\` to be used as normal characters in a match condition. + +Example: prevent pushing any `.exe` files to any location in the repository. This is an example of a partial match, which can match any filename that contains `.exe` at the end: + +```plaintext +\.exe$ +``` + +Example: prevent a specific configuration file in the repository root from being pushed: + +```plaintext +^config\.yml$ +``` + +Example: prevent a specific configuration file in a known directory from being pushed: + +```plaintext +^directory-name\/config\.yml$ +``` + +Example: prevent the specific file named `install.exe` from being pushed to any location in the repository. Note that the parenthesized expression `(^|\/)` will match either a file following a directory separator or a file in the root directory of the repository: + +```plaintext +(^|\/)install\.exe$ +``` + +Example: combining all of the above in a single expression. Note that all of the preceding expressions rely on the end of string character `$`, so we can move that part of each expression to the end of the grouped collection of match conditions where it will be appended to all matches: + +```plaintext +(\.exe|^config\.yml|^directory-name\/config\.yml|(^|\/)install\.exe)$ +``` + <!-- ## Troubleshooting Include any troubleshooting steps that you can foresee. If you know beforehand what issues diff --git a/doc/raketasks/README.md b/doc/raketasks/README.md index 523486d5137..7aec74f1243 100644 --- a/doc/raketasks/README.md +++ b/doc/raketasks/README.md @@ -21,7 +21,7 @@ The following are available Rake tasks: | [Clean up](cleanup.md) | Clean up unneeded items from GitLab instances. | | [Development](../development/rake_tasks.md) | Tasks for GitLab contributors. | | [Doctor tasks](../administration/raketasks/doctor.md) | Checks for data integrity issues. | -| [Elasticsearch](../integration/elasticsearch.md#gitlab-elasticsearch-rake-tasks) **(STARTER ONLY)** | Maintain Elasticsearch in a GitLab instance. | +| [Elasticsearch](../integration/elasticsearch.md#gitlab-advanced-search-rake-tasks) **(STARTER ONLY)** | Maintain Elasticsearch in a GitLab instance. | | [Enable namespaces](features.md) | Enable usernames and namespaces for user projects. | | [General maintenance](../administration/raketasks/maintenance.md) | General maintenance and self-check tasks. | | [Geo maintenance](../administration/raketasks/geo.md) **(PREMIUM ONLY)** | [Geo](../administration/geo/index.md)-related maintenance. | diff --git a/doc/raketasks/backup_restore.md b/doc/raketasks/backup_restore.md index 68b076258ce..066a38d68de 100644 --- a/doc/raketasks/backup_restore.md +++ b/doc/raketasks/backup_restore.md @@ -10,48 +10,41 @@ of GitLab on which it was created. The best way to migrate your repositories from one server to another is through backup restore. CAUTION: **Warning:** -GitLab will not backup items that are not stored on the -filesystem. If using [object storage](../administration/object_storage.md), -remember to enable backups with your object storage provider if desired. +GitLab won't back up items that aren't stored in the filesystem. If you're +using [object storage](../administration/object_storage.md), be sure to enable +backups with your object storage provider, if desired. ## Requirements -In order to be able to backup and restore, you need one essential tool -installed on your system. +To be able to backup and restore, ensure that Rsync is installed on your +system. If you installed GitLab: -- **Rsync**: If you installed GitLab: - - Using the Omnibus package, you're all set. - - From source, make sure `rsync` is installed. For example: +- _Using the Omnibus package_, you're all set. +- _From source_, you need to determine if `rsync` is installed. For example: - ```shell - # Debian/Ubuntu - sudo apt-get install rsync + ```shell + # Debian/Ubuntu + sudo apt-get install rsync - # RHEL/CentOS - sudo yum install rsync - ``` + # RHEL/CentOS + sudo yum install rsync + ``` ## Backup timestamp -NOTE: **Note:** -In GitLab 9.2 the timestamp format was changed from `EPOCH_YYYY_MM_DD` to -`EPOCH_YYYY_MM_DD_GitLab_version`, for example `1493107454_2018_04_25` -would become `1493107454_2018_04_25_10.6.4-ce`. - The backup archive will be saved in `backup_path`, which is specified in the -`config/gitlab.yml` file. -The filename will be `[TIMESTAMP]_gitlab_backup.tar`, where `TIMESTAMP` -identifies the time at which each backup was created, plus the GitLab version. -The timestamp is needed if you need to restore GitLab and multiple backups are -available. +`config/gitlab.yml` file. The filename will be `[TIMESTAMP]_gitlab_backup.tar`, +where `TIMESTAMP` identifies the time at which each backup was created, plus +the GitLab version. The timestamp is needed if you need to restore GitLab and +multiple backups are available. For example, if the backup name is `1493107454_2018_04_25_10.6.4-ce_gitlab_backup.tar`, -then the timestamp is `1493107454_2018_04_25_10.6.4-ce`. +the timestamp is `1493107454_2018_04_25_10.6.4-ce`. ## Back up GitLab -GitLab provides a simple command line interface to back up your whole instance. -It backs up your: +GitLab provides a command line interface to back up your entire instance, +including: - Database - Attachments @@ -61,49 +54,62 @@ It backs up your: - LFS objects - Container Registry images - GitLab Pages content +- Snippets CAUTION: **Warning:** -GitLab does not back up any configuration files, SSL certificates, or system files. -You are highly advised to [read about storing configuration files](#storing-configuration-files). +GitLab does not back up any configuration files, SSL certificates, or system +files. You are highly advised to read about [storing configuration files](#storing-configuration-files). -Use this command if you've installed GitLab with the Omnibus package: +Depending on your version of GitLab, use the following command if you installed +GitLab using the Omnibus package: -```shell -sudo gitlab-backup create -``` +- GitLab 12.2 or later: -NOTE: **Note:** -For GitLab 12.1 and earlier, use `gitlab-rake gitlab:backup:create`. + ```shell + sudo gitlab-backup create + ``` -Use this if you've installed GitLab from source: +- GitLab 12.1 and earlier: + + ```shell + gitlab-rake gitlab:backup:create + ``` + +If you installed GitLab from source, use the following command: ```shell sudo -u git -H bundle exec rake gitlab:backup:create RAILS_ENV=production ``` -If you are running GitLab within a Docker container, you can run the backup from the host: +If you're running GitLab from within a Docker container, run the backup from +the host, based on your installed version of GitLab: -```shell -docker exec -t <container name> gitlab-backup create -``` +- GitLab 12.2 or later: -NOTE: **Note:** -For GitLab 12.1 and earlier, use `gitlab-rake gitlab:backup:create`. + ```shell + docker exec -t <container name> gitlab-backup create + ``` -If you are using the [GitLab Helm chart](https://gitlab.com/gitlab-org/charts/gitlab) on a -Kubernetes cluster, you can run the backup task using `backup-utility` script on -the GitLab task runner pod via `kubectl`. Refer to [backing up a GitLab installation](https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/backup-restore/backup.md#backing-up-a-gitlab-installation) for more details: +- GitLab 12.1 and earlier: + + ```shell + gitlab-rake gitlab:backup:create + ``` + +If you're using the [GitLab Helm chart](https://gitlab.com/gitlab-org/charts/gitlab) +on a Kubernetes cluster, you can run the backup task by using `kubectl` to run the `backup-utility` +script on the GitLab task runner pod. For more details, see +[backing up a GitLab installation](https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/backup-restore/backup.md#backing-up-a-gitlab-installation). ```shell kubectl exec -it <gitlab task-runner pod> backup-utility ``` -Similarly to the Kubernetes case, if you have scaled out your GitLab -cluster to use multiple application servers, you should pick a -designated node (that won't be auto-scaled away) for running the -backup Rake task. Because the backup Rake task is tightly coupled to -the main Rails application, this is typically a node on which you're -also running Unicorn/Puma and/or Sidekiq. +Similar to the Kubernetes case, if you have scaled out your GitLab cluster to +use multiple application servers, you should pick a designated node (that won't +be auto-scaled away) for running the backup Rake task. Because the backup Rake +task is tightly coupled to the main Rails application, this is typically a node +on which you're also running Unicorn/Puma or Sidekiq. Example output: @@ -136,11 +142,11 @@ Deleting old backups... [SKIPPING] ### Storing configuration files -The [backup Rake task](#back-up-gitlab) GitLab provides -does **not** store your configuration files. The primary reason for this is that your -database contains encrypted information for two-factor authentication, the CI/CD -'secure variables', and so on. Storing encrypted information along with its key in the -same place defeats the purpose of using encryption in the first place. +The [backup Rake task](#back-up-gitlab) GitLab provides does _not_ store your +configuration files. The primary reason for this is that your database contains +items including encrypted information for two-factor authentication and the +CI/CD _secure variables_. Storing encrypted information in the same location +as its key defeats the purpose of using encryption in the first place. CAUTION: **Warning:** The secrets file is essential to preserve your database encryption key. @@ -158,30 +164,31 @@ For installation from source: - `/home/git/gitlab/config/gitlab.yml` For [Docker installations](https://docs.gitlab.com/omnibus/docker/), you must -back up the volume where the configuration files are stored. If you have created -the GitLab container according to the documentation, it should be under -`/srv/gitlab/config`. +back up the volume where the configuration files are stored. If you created +the GitLab container according to the documentation, it should be in the +`/srv/gitlab/config` directory. -For [GitLab Helm chart Installations](https://gitlab.com/gitlab-org/charts/gitlab) on a -Kubernetes cluster, you must follow the [Backup the secrets](https://docs.gitlab.com/charts/backup-restore/backup.html#backup-the-secrets) instructions. +For [GitLab Helm chart installations](https://gitlab.com/gitlab-org/charts/gitlab) +on a Kubernetes cluster, you must follow the +[Backup the secrets](https://docs.gitlab.com/charts/backup-restore/backup.html#backup-the-secrets) +instructions. You may also want to back up any TLS keys and certificates, and your [SSH host keys](https://superuser.com/questions/532040/copy-ssh-keys-from-one-server-to-another-server/532079#532079). -If you use Omnibus GitLab, see some additional information -[to backup your configuration](https://docs.gitlab.com/omnibus/settings/backups.html). +If you use Omnibus GitLab, review additional information to +[backup your configuration](https://docs.gitlab.com/omnibus/settings/backups.html). In the unlikely event that the secrets file is lost, see the [troubleshooting section](#when-the-secrets-file-is-lost). ### Backup options -The command line tool GitLab provides to backup your instance can take more options. +The command line tool GitLab provides to backup your instance can accept more +options. #### Backup strategy option -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/8728) in GitLab 8.17. - The default backup strategy is to essentially stream data from the respective data locations to the backup using the Linux command `tar` and `gzip`. This works fine in most cases, but can cause problems when data is rapidly changing. @@ -203,8 +210,7 @@ To use the `copy` strategy instead of the default streaming strategy, specify sudo gitlab-backup create STRATEGY=copy ``` -NOTE: **Note:** -For GitLab 12.1 and earlier, use `gitlab-rake gitlab:backup:create`. +Users of GitLab 12.1 and earlier should use the command `gitlab-rake gitlab:backup:create` instead. #### Backup filename @@ -212,34 +218,41 @@ CAUTION: **Warning:** If you use a custom backup filename, you will not be able to [limit the lifetime of the backups](#limit-backup-lifetime-for-local-files-prune-old-backups). -By default a backup file is created according to the specification in [the Backup timestamp](#backup-timestamp) section above. You can however override the `[TIMESTAMP]` part of the filename by setting the `BACKUP` environment variable. For example: +By default, a backup file is created according to the specification in the +previous [Backup timestamp](#backup-timestamp) section. You can, however, +override the `[TIMESTAMP]` portion of the filename by setting the `BACKUP` +environment variable. For example: ```shell sudo gitlab-backup create BACKUP=dump ``` -NOTE: **Note:** -For GitLab 12.1 and earlier, use `gitlab-rake gitlab:backup:create`. +Users of GitLab 12.1 and earlier should use the command `gitlab-rake gitlab:backup:create` instead. -The resulting file will then be `dump_gitlab_backup.tar`. This is useful for systems that make use of rsync and incremental backups, and will result in considerably faster transfer speeds. +The resulting file will then be `dump_gitlab_backup.tar`. This is useful for +systems that make use of rsync and incremental backups, and will result in +considerably faster transfer speeds. #### Rsyncable -To make sure the generated archive is intelligently transferable by rsync, the `GZIP_RSYNCABLE=yes` option can be set. This will set the `--rsyncable` option to `gzip`. This is only useful in combination with setting [the Backup filename option](#backup-filename). +To ensure the generated archive is transferable by rsync, you can set the `GZIP_RSYNCABLE=yes` +option. This sets the `--rsyncable` option to `gzip`, which is useful only in +combination with setting [the Backup filename option](#backup-filename). -Note that the `--rsyncable` option in `gzip` is not guaranteed to be available on all distributions. To verify that it is available in your distribution you can run `gzip --help` or consult the man pages. +Note that the `--rsyncable` option in `gzip` isn't guaranteed to be available +on all distributions. To verify that it's available in your distribution, run +`gzip --help` or consult the man pages. ```shell sudo gitlab-backup create BACKUP=dump GZIP_RSYNCABLE=yes ``` -NOTE: **Note:** -For GitLab 12.1 and earlier, use `gitlab-rake gitlab:backup:create`. +Users of GitLab 12.1 and earlier should use the command `gitlab-rake gitlab:backup:create` instead. #### Excluding specific directories from the backup -You can choose what should be exempt from the backup up by adding the environment variable `SKIP`. -The available options are: +You can choose what should be exempt from the backup by adding the environment +variable `SKIP`. The available options are: - `db` (database) - `uploads` (attachments) @@ -252,8 +265,8 @@ The available options are: Use a comma to specify several options at the same time: -All wikis will be backed up as part of the `repositories` group. Non-existent wikis -will be skipped during a backup. +All wikis will be backed up as part of the `repositories` group. Non-existent +wikis will be skipped during a backup. For Omnibus GitLab packages: @@ -261,8 +274,7 @@ For Omnibus GitLab packages: sudo gitlab-backup create SKIP=db,uploads ``` -NOTE: **Note:** -For GitLab 12.1 and earlier, use `gitlab-rake gitlab:backup:create`. +Users of GitLab 12.1 and earlier should use the command `gitlab-rake gitlab:backup:create` instead. For installations from source: @@ -317,13 +329,15 @@ sudo -u git -H GITLAB_ASSUME_YES=1 bundle exec rake gitlab:backup:restore RAILS_ > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/37158) in GitLab 13.3. -Repositories can be backed up concurrently to help fully utilise CPU time. The following variables -are available to modify the default behavior of the Rake task: +Repositories can be backed up concurrently to help fully utilize CPU time. The +following variables are available to modify the default behavior of the Rake +task: -- `GITLAB_BACKUP_MAX_CONCURRENCY` sets the maximum number of projects to backup at the same time. - Defaults to 1. -- `GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY` sets the maximum number of projects to backup at the same time on each storage. This allows the repository backups to be spread across storages. - Defaults to 1. +- `GITLAB_BACKUP_MAX_CONCURRENCY`: The maximum number of projects to back up at + the same time. Defaults to `1`. +- `GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY`: The maximum number of projects to + back up at the same time on each storage. This allows the repository backups + to be spread across storages. Defaults to `1`. For example, for Omnibus GitLab installations: @@ -339,12 +353,11 @@ sudo -u git -H bundle exec rake gitlab:backup:create GITLAB_BACKUP_MAX_CONCURREN #### Uploading backups to a remote (cloud) storage -Starting with GitLab 7.4 you can let the backup script upload the `.tar` file it creates. -It uses the [Fog library](http://fog.io/) to perform the upload. -In the example below we use Amazon S3 for storage, but Fog also lets you use -[other storage providers](http://fog.io/storage/). GitLab -[imports cloud drivers](https://gitlab.com/gitlab-org/gitlab/blob/30f5b9a5b711b46f1065baf755e413ceced5646b/Gemfile#L88) -for AWS, Google, OpenStack Swift, Rackspace, and Aliyun as well. A local driver is +You can let the backup script upload (using the [Fog library](http://fog.io/)) +the `.tar` file it creates. In the following example, we use Amazon S3 for +storage, but Fog also lets you use [other storage providers](http://fog.io/storage/). +GitLab also [imports cloud drivers](https://gitlab.com/gitlab-org/gitlab/blob/da46c9655962df7d49caef0e2b9f6bbe88462a02/Gemfile#L113) +for AWS, Google, OpenStack Swift, Rackspace, and Aliyun. A local driver is [also available](#uploading-to-locally-mounted-shares). [Read more about using object storage with GitLab](../administration/object_storage.md). @@ -367,11 +380,12 @@ For Omnibus GitLab packages: gitlab_rails['backup_upload_remote_directory'] = 'my.s3.bucket' ``` -1. [Reconfigure GitLab](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect +1. [Reconfigure GitLab](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) + for the changes to take effect ##### Digital Ocean Spaces -This example can be used for a bucket in Amsterdam (AMS3). +This example can be used for a bucket in Amsterdam (AMS3): 1. Add the following to `/etc/gitlab/gitlab.rb`: @@ -386,20 +400,20 @@ This example can be used for a bucket in Amsterdam (AMS3). gitlab_rails['backup_upload_remote_directory'] = 'my.s3.bucket' ``` -1. [Reconfigure GitLab](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect +1. [Reconfigure GitLab](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) + for the changes to take effect -NOTE: **Note:** -If you see `400 Bad Request` by using Digital Ocean Spaces, the cause may be the -usage of backup encryption. Remove or comment the line that -contains `gitlab_rails['backup_encryption']` since Digital Ocean Spaces -doesn't support encryption. +If you see a `400 Bad Request` error message when using Digital Ocean Spaces, +the cause may be the use of backup encryption. Because Digital Ocean Spaces +doesn't support encryption, remove or comment the line that contains +`gitlab_rails['backup_encryption']`. ##### Other S3 Providers -Not all S3 providers are fully-compatible with the Fog library. For example, -if you see `411 Length Required` errors after attempting to upload, you may -need to downgrade the `aws_signature_version` value from the default value to -2 [due to this issue](https://github.com/fog/fog-aws/issues/428). +Not all S3 providers are fully compatible with the Fog library. For example, +if you see a `411 Length Required` error message after attempting to upload, +you may need to downgrade the `aws_signature_version` value from the default +value to `2`, [due to this issue](https://github.com/fog/fog-aws/issues/428). For installations from source: @@ -431,9 +445,10 @@ For installations from source: # storage_class: 'STANDARD' ``` -1. [Restart GitLab](../administration/restart_gitlab.md#installations-from-source) for the changes to take effect +1. [Restart GitLab](../administration/restart_gitlab.md#installations-from-source) + for the changes to take effect -If you are uploading your backups to S3 you will probably want to create a new +If you're uploading your backups to S3, you'll probably want to create a new IAM user with restricted access rights. To give the upload user access only for uploading backups create the following IAM profile, replacing `my.s3.bucket` with the name of your bucket: @@ -486,16 +501,16 @@ with the name of your bucket: ##### Using Google Cloud Storage -If you want to use Google Cloud Storage to save backups, you'll have to create -an access key from the Google console first: +To use Google Cloud Storage to save backups, you must first create an +access key from the Google console: -1. Go to the storage settings page <https://console.cloud.google.com/storage/settings> -1. Select "Interoperability" and create an access key -1. Make note of the "Access Key" and "Secret" and replace them in the - configurations below -1. In the buckets advanced settings ensure the Access Control option "Set object-level - and bucket-level permissions" is selected -1. Make sure you already have a bucket created +1. Go to the [Google storage settings page](https://console.cloud.google.com/storage/settings). +1. Select **Interoperability**, and then create an access key. +1. Make note of the **Access Key** and **Secret** and replace them in the + following configurations. +1. In the buckets advanced settings ensure the Access Control option + **Set object-level and bucket-level permissions** is selected. +1. Ensure you have already created a bucket. For Omnibus GitLab packages: @@ -516,7 +531,8 @@ For Omnibus GitLab packages: gitlab_rails['backup_upload_remote_directory'] = 'my.google.bucket' ``` -1. [Reconfigure GitLab](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect +1. [Reconfigure GitLab](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) + for the changes to take effect For installations from source: @@ -532,7 +548,8 @@ For installations from source: remote_directory: 'my.google.bucket' ``` -1. [Restart GitLab](../administration/restart_gitlab.md#installations-from-source) for the changes to take effect +1. [Restart GitLab](../administration/restart_gitlab.md#installations-from-source) + for the changes to take effect ##### Using Azure Blob storage @@ -552,7 +569,8 @@ For Omnibus GitLab packages: gitlab_rails['backup_upload_remote_directory'] = '<AZURE BLOB CONTAINER>' ``` -1. [Reconfigure GitLab](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect +1. [Reconfigure GitLab](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) + for the changes to take effect For installations from source: @@ -568,13 +586,14 @@ For installations from source: remote_directory: '<AZURE BLOB CONTAINER>' ``` -1. [Restart GitLab](../administration/restart_gitlab.md#installations-from-source) for the changes to take effect +1. [Restart GitLab](../administration/restart_gitlab.md#installations-from-source) + for the changes to take effect -See [the table of Azure parameters](../administration/object_storage.md#azure-blob-storage) for more details. +For more details, see the [table of Azure parameters](../administration/object_storage.md#azure-blob-storage). ##### Specifying a custom directory for backups -Note: This option only works for remote storage. If you want to group your backups +This option works only for remote storage. If you want to group your backups, you can pass a `DIRECTORY` environment variable: ```shell @@ -582,26 +601,25 @@ sudo gitlab-backup create DIRECTORY=daily sudo gitlab-backup create DIRECTORY=weekly ``` -NOTE: **Note:** -For GitLab 12.1 and earlier, use `gitlab-rake gitlab:backup:create`. +Users of GitLab 12.1 and earlier should use the command `gitlab-rake gitlab:backup:create` instead. #### Uploading to locally mounted shares -You may also send backups to a mounted share (for example, `NFS`,`CIFS`, or `SMB`) by -using the Fog [`Local`](https://github.com/fog/fog-local#usage) storage provider. -The directory pointed to by the `local_root` key **must** be owned by the `git` -user **when mounted** (mounting with the `uid=` of the `git` user for `CIFS` and -`SMB`) or the user that you are executing the backup tasks under (for Omnibus -packages, this is the `git` user). +You may also send backups to a mounted share (for example, `NFS`,`CIFS`, or +`SMB`) by using the Fog [`Local`](https://github.com/fog/fog-local#usage) +storage provider. The directory pointed to by the `local_root` key _must_ be +owned by the `git` user _when mounted_ (mounting with the `uid=` of the `git` +user for `CIFS` and `SMB`) or the user that you are executing the backup tasks +as (for Omnibus packages, this is the `git` user). -The `backup_upload_remote_directory` **must** be set in addition to the +The `backup_upload_remote_directory` _must_ be set in addition to the `local_root` key. This is the sub directory inside the mounted directory that backups will be copied to, and will be created if it does not exist. If the directory that you want to copy the tarballs to is the root of your mounted -directory, just use `.` instead. +directory, use `.` instead. -NOTE: **Note:** -Since file system performance may affect GitLab's overall performance, we do not recommend using EFS for storage. See the [relevant documentation](../administration/nfs.md#avoid-using-awss-elastic-file-system-efs) for more details. +Because file system performance may affect GitLab's overall performance, +[GitLab doesn't recommend using EFS for storage](../administration/nfs.md#avoid-using-awss-elastic-file-system-efs). For Omnibus GitLab packages: @@ -618,7 +636,8 @@ For Omnibus GitLab packages: gitlab_rails['backup_upload_remote_directory'] = 'gitlab_backups' ``` -1. [Reconfigure GitLab](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. +1. [Reconfigure GitLab](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) + for the changes to take effect. For installations from source: @@ -636,14 +655,16 @@ For installations from source: remote_directory: 'gitlab_backups' ``` -1. [Restart GitLab](../administration/restart_gitlab.md#installations-from-source) for the changes to take effect. +1. [Restart GitLab](../administration/restart_gitlab.md#installations-from-source) + for the changes to take effect. #### Backup archive permissions The backup archives created by GitLab (`1393513186_2014_02_27_gitlab_backup.tar`) -will have owner/group `git`/`git` and 0600 permissions by default. -This is meant to avoid other system users reading GitLab's data. -If you need the backup archives to have different permissions you can use the 'archive_permissions' setting. +will have owner/group `git`/`git` and 0600 permissions by default. This is +meant to avoid other system users reading GitLab's data. If you need the backup +archives to have different permissions, you can use the `archive_permissions` +setting. For Omnibus GitLab packages: @@ -653,7 +674,8 @@ For Omnibus GitLab packages: gitlab_rails['backup_archive_permissions'] = 0644 # Makes the backup archives world-readable ``` -1. [Reconfigure GitLab](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. +1. [Reconfigure GitLab](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) + for the changes to take effect. For installations from source: @@ -664,7 +686,8 @@ For installations from source: archive_permissions: 0644 # Makes the backup archives world-readable ``` -1. [Restart GitLab](../administration/restart_gitlab.md#installations-from-source) for the changes to take effect. +1. [Restart GitLab](../administration/restart_gitlab.md#installations-from-source) + for the changes to take effect. #### Configuring cron to make daily backups @@ -689,8 +712,7 @@ For Omnibus GitLab packages: 0 2 * * * /opt/gitlab/bin/gitlab-backup create CRON=1 ``` - NOTE: **Note:** - For GitLab 12.1 and earlier, use `gitlab-rake gitlab:backup:create`. + Users of GitLab 12.1 and earlier should use the command `gitlab-rake gitlab:backup:create` instead. For installations from source: @@ -707,26 +729,25 @@ For installations from source: 0 2 * * * cd /home/git/gitlab && PATH=/usr/local/bin:/usr/bin:/bin bundle exec rake gitlab:backup:create RAILS_ENV=production CRON=1 ``` -NOTE: **Note:** -The `CRON=1` environment setting tells the backup script to suppress all progress output if there are no errors. -This is recommended to reduce cron spam. +The `CRON=1` environment setting directs the backup script to hide all progress +output if there aren't any errors. This is recommended to reduce cron spam. ### Limit backup lifetime for local files (prune old backups) CAUTION: **Warning:** -This will not work if you have used a [custom filename](#backup-filename) +The process described in this section will not work if you used a [custom filename](#backup-filename) for your backups. -NOTE: **Note:** -This configuration option only manages local files. GitLab does not automatically -prune old files stored in a third-party [object storage](#uploading-backups-to-a-remote-cloud-storage) -because the user may not have permission to list and delete files. It is +To prevent regular backups from using all your disk space, you may want to set a limited lifetime +for backups. The next time the backup task runs, backups older than the `backup_keep_time` are +pruned. + +This configuration option manages only local files. GitLab doesn't prune old +files stored in a third-party [object storage](#uploading-backups-to-a-remote-cloud-storage) +because the user may not have permission to list and delete files. It's recommended that you configure the appropriate retention policy for your object storage (for example, [AWS S3](https://docs.aws.amazon.com/AmazonS3/latest/user-guide/create-lifecycle.html)). -You may want to set a limited lifetime for backups to prevent regular -backups using all your disk space. The next time the backup task is run, backups older than the `backup_keep_time` will be pruned. - For Omnibus GitLab packages: 1. Edit `/etc/gitlab/gitlab.rb`: @@ -736,7 +757,8 @@ For Omnibus GitLab packages: gitlab_rails['backup_keep_time'] = 604800 ``` -1. [Reconfigure GitLab](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. +1. [Reconfigure GitLab](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) + for the changes to take effect. For installations from source: @@ -748,61 +770,64 @@ For installations from source: keep_time: 604800 ``` -1. [Restart GitLab](../administration/restart_gitlab.md#installations-from-source) for the changes to take effect. +1. [Restart GitLab](../administration/restart_gitlab.md#installations-from-source) + for the changes to take effect. ## Restore GitLab -GitLab provides a simple command line interface to restore your whole installation, +GitLab provides a command line interface to restore your entire installation, and is flexible enough to fit your needs. The [restore prerequisites section](#restore-prerequisites) includes crucial -information. Make sure to read and test the whole restore process at least once -before attempting to perform it in a production environment. +information. Be sure to read and test the complete restore process at least +once before attempting to perform it in a production environment. -You can only restore a backup to **exactly the same version and type (CE/EE)** of -GitLab that you created it on, for example CE 9.1.0. +You can restore a backup only to _the exact same version and type (CE/EE)_ of +GitLab that you created it on (for example CE 9.1.0). -If your backup is a different version than the current installation, you will +If your backup is a different version than the current installation, you'll need to [downgrade your GitLab installation](https://docs.gitlab.com/omnibus/update/README.html#downgrade) before restoring the backup. ### Restore prerequisites -You need to have a working GitLab installation before you can perform -a restore. This is mainly because the system user performing the -restore actions (`git`) is usually not allowed to create or delete -the SQL database it needs to import data into (`gitlabhq_production`). -All existing data will be either erased (SQL) or moved to a separate -directory (repositories, uploads). +You need to have a working GitLab installation before you can perform a +restore. This is because the system user performing the restore actions (`git`) +is usually not allowed to create or delete the SQL database needed to import +data into (`gitlabhq_production`). All existing data will be either erased +(SQL) or moved to a separate directory (such as repositories and uploads). -To restore a backup, you will also need to restore `/etc/gitlab/gitlab-secrets.json` -(for Omnibus packages) or `/home/git/gitlab/.secret` (for installations -from source). This file contains the database encryption key, +To restore a backup, you'll also need to restore `/etc/gitlab/gitlab-secrets.json` +(for Omnibus packages) or `/home/git/gitlab/.secret` (for installations from +source). This file contains the database encryption key, [CI/CD variables](../ci/variables/README.md#gitlab-cicd-environment-variables), and variables used for [two-factor authentication](../user/profile/account/two_factor_authentication.md). If you fail to restore this encryption key file along with the application data backup, users with two-factor authentication enabled and GitLab Runner will lose access to your GitLab server. -You may also want to restore any TLS keys, certificates, or [SSH host keys](https://superuser.com/questions/532040/copy-ssh-keys-from-one-server-to-another-server/532079#532079). +You may also want to restore any TLS keys, certificates, or +[SSH host keys](https://superuser.com/questions/532040/copy-ssh-keys-from-one-server-to-another-server/532079#532079). -Starting with GitLab 12.9 if an untarred backup (like the ones made with +Starting with GitLab 12.9, if an untarred backup (like the ones made with `SKIP=tar`) is found, and no backup is chosen with `BACKUP=<timestamp>`, the untarred backup is used. Depending on your case, you might want to run the restore command with one or more of the following options: -- `BACKUP=timestamp_of_backup` - Required if more than one backup exists. +- `BACKUP=timestamp_of_backup`: Required if more than one backup exists. Read what the [backup timestamp is about](#backup-timestamp). -- `force=yes` - Does not ask if the authorized_keys file should get regenerated and assumes 'yes' for warning that database tables will be removed, enabling the "Write to authorized_keys file" setting, and updating LDAP providers. +- `force=yes`: Doesn't ask if the authorized_keys file should get regenerated, + and assumes 'yes' for warning that database tables will be removed, + enabling the "Write to authorized_keys file" setting, and updating LDAP + providers. -If you are restoring into directories that are mount points, you will need to make -sure these directories are empty before attempting a restore. Otherwise GitLab -will attempt to move these directories before restoring the new data and this -would cause an error. +If you're restoring into directories that are mount points, you must ensure these directories are +empty before attempting a restore. Otherwise, GitLab attempts to move these directories before +restoring the new data, which causes an error. -Read more on [configuring NFS mounts](../administration/nfs.md) +Read more about [configuring NFS mounts](../administration/nfs.md) ### Restore for installation from source @@ -845,7 +870,7 @@ Restoring repositories: Deleting tmp directories...[DONE] ``` -Next, restore `/home/git/gitlab/.secret` if necessary as mentioned above. +Next, restore `/home/git/gitlab/.secret` if necessary, as previously mentioned. Restart GitLab: @@ -862,7 +887,7 @@ This procedure assumes that: - You have run `sudo gitlab-ctl reconfigure` at least once. - GitLab is running. If not, start it using `sudo gitlab-ctl start`. -First make sure your backup tar file is in the backup directory described in the +First ensure your backup tar file is in the backup directory described in the `gitlab.rb` configuration `gitlab_rails['backup_path']`. The default is `/var/opt/gitlab/backups`. It needs to be owned by the `git` user. @@ -890,15 +915,16 @@ restore: sudo gitlab-backup restore BACKUP=11493107454_2018_04_25_10.6.4-ce ``` -NOTE: **Note:** -For GitLab 12.1 and earlier, use `gitlab-rake gitlab:backup:restore`. +Users of GitLab 12.1 and earlier should use the command `gitlab-rake gitlab:backup:restore` instead. CAUTION: **Warning:** -`gitlab-rake gitlab:backup:restore` does not set the right file system permissions on your Registry directory. -This is a [known issue](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/62759). On GitLab 12.2 or newer, you can -use `gitlab-backup restore` to avoid this issue. +`gitlab-rake gitlab:backup:restore` doesn't set the correct file system +permissions on your Registry directory. This is a [known issue](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/62759). +On GitLab 12.2 or later, you can use `gitlab-backup restore` to avoid this +issue. -Next, restore `/etc/gitlab/gitlab-secrets.json` if necessary as mentioned above. +Next, restore `/etc/gitlab/gitlab-secrets.json` if necessary, as previously +mentioned. Reconfigure, restart and check GitLab: @@ -908,29 +934,29 @@ sudo gitlab-ctl restart sudo gitlab-rake gitlab:check SANITIZE=true ``` -If there is a GitLab version mismatch between your backup tar file and the installed -version of GitLab, the restore command will abort with an error. Install the -[correct GitLab version](https://packages.gitlab.com/gitlab/) and try again. +If there's a GitLab version mismatch between your backup tar file and the +installed version of GitLab, the restore command aborts with an error +message. Install the [correct GitLab version](https://packages.gitlab.com/gitlab/), +and then try again. NOTE: **Note:** -There is currently a [known issue](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/3470) for restore not working -with `pgbouncer`. In order to workaround the issue, the Rails node will need to bypass `pgbouncer` and connect -directly to the primary database node. This can be done by setting `gitlab_rails['db_host']` and `gitlab_rails['port']` -to connect to the primary database node and [reconfiguring GitLab](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure). +There is a known issue with restore not working with `pgbouncer`. The [workaround is to bypass +`pgbouncer` and connect directly to the primary database node](../administration/postgresql/pgbouncer.md#procedure-for-bypassing-pgbouncer). +[Read more about backup and restore with `pgbouncer`](#backup-and-restore-for-installations-using-pgbouncer). ### Restore for Docker image and GitLab Helm chart installations -For GitLab installations using the Docker image or the GitLab Helm chart on -a Kubernetes cluster, the restore task expects the restore directories to be empty. -However, with Docker and Kubernetes volume mounts, some system level directories -may be created at the volume roots, like `lost+found` directory found in Linux -operating systems. These directories are usually owned by `root`, which can -cause access permission errors since the restore Rake task runs as `git` user. -So, to restore a GitLab installation, users have to confirm the restore target -directories are empty. +For GitLab installations using the Docker image or the GitLab Helm chart on a +Kubernetes cluster, the restore task expects the restore directories to be +empty. However, with Docker and Kubernetes volume mounts, some system level +directories may be created at the volume roots, such as the `lost+found` +directory found in Linux operating systems. These directories are usually owned +by `root`, which can cause access permission errors since the restore Rake task +runs as the `git` user. To restore a GitLab installation, users have to confirm +the restore target directories are empty. -For both these installation types, the backup tarball has to be available in the -backup location (default location is `/var/opt/gitlab/backups`). +For both these installation types, the backup tarball has to be available in +the backup location (default location is `/var/opt/gitlab/backups`). For Docker installations, the restore task can be run from host: @@ -953,43 +979,43 @@ docker restart <name of container> docker exec -it <name of container> gitlab-rake gitlab:check SANITIZE=true ``` -NOTE: **Note:** -For GitLab 12.1 and earlier, use `gitlab-rake gitlab:backup:restore`. +Users of GitLab 12.1 and earlier should use the command `gitlab-rake gitlab:backup:create` instead. CAUTION: **Warning:** -`gitlab-rake gitlab:backup:restore` does not set the right file system permissions on your Registry directory. -This is a [known issue](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/62759). On GitLab 12.2 or newer, you can -use `gitlab-backup restore` to avoid this issue. +`gitlab-rake gitlab:backup:restore` doesn't set the correct file system +permissions on your Registry directory. This is a [known issue](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/62759). +On GitLab 12.2 or later, you can use `gitlab-backup restore` to avoid this +issue. The GitLab Helm chart uses a different process, documented in [restoring a GitLab Helm chart installation](https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/backup-restore/restore.md). ### Restoring only one or a few project(s) or group(s) from a backup -While the Rake task used to restore a GitLab instance doesn't support -restoring a single project or group, you can use a workaround by -restoring your backup to a separate, temporary GitLab instance, and -export your project or group from there: +Although the Rake task used to restore a GitLab instance doesn't support +restoring a single project or group, you can use a workaround by restoring +your backup to a separate, temporary GitLab instance, and then export your +project or group from there: 1. [Install a new GitLab](../install/README.md) instance at the same version as the backed-up instance from which you want to restore. -1. [Restore the backup](#restore-gitlab) into this new instance and +1. [Restore the backup](#restore-gitlab) into this new instance, then export your [project](../user/project/settings/import_export.md) - or [group](../user/group/settings/import_export.md). Make sure to - read the **Important Notes** on either export feature's documentation - to understand what will be exported and what not. -1. Once the export is complete, go to the old instance and import it. -1. After importing only the project(s) or group(s) that you wanted is complete, - you may delete the new, temporary GitLab instance. + or [group](../user/group/settings/import_export.md). Be sure to read the + **Important Notes** on either export feature's documentation to understand + what is and isn't exported. +1. After the export is complete, go to the old instance and then import it. +1. After importing the projects or groups that you wanted is complete, you may + delete the new, temporary GitLab instance. -NOTE: **Note:** A feature request to provide direct restore of individual projects or groups is being discussed in [issue #17517](https://gitlab.com/gitlab-org/gitlab/-/issues/17517). ## Alternative backup strategies -If your GitLab server contains a lot of Git repository data you may find the GitLab backup script to be too slow. -In this case you can consider using filesystem snapshots as part of your backup strategy. +If your GitLab server contains a lot of Git repository data, you may find the +GitLab backup script to be too slow. In this case you can consider using +filesystem snapshots as part of your backup strategy. Example: Amazon EBS @@ -1006,29 +1032,56 @@ Example: LVM snapshots + rsync > Now we can have a longer running rsync job which will create a consistent replica on the remote server. > The replica includes all repositories, uploads and PostgreSQL data. -If you are running GitLab on a virtualized server you can possibly also create VM snapshots of the entire GitLab server. -It is not uncommon however for a VM snapshot to require you to power down the server, so this approach is probably of limited practical use. +If you're running GitLab on a virtualized server, you can possibly also create +VM snapshots of the entire GitLab server. It's not uncommon however for a VM +snapshot to require you to power down the server, which limits this solution's +practical use. + +## Backup and restore for installations using PgBouncer + +PgBouncer can cause the following errors when performing backups and restores: + +```ruby +ActiveRecord::StatementInvalid: PG::UndefinedTable +``` + +There is a [known issue](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/3470) for restore not working +with `pgbouncer`. + +To workaround this issue, the GitLab server will need to bypass `pgbouncer` and +[connect directly to the primary database node](../administration/postgresql/pgbouncer.md#procedure-for-bypassing-pgbouncer) +to perform the database restore. + +There is also a [known issue](https://gitlab.com/gitlab-org/gitlab/-/issues/23211) +with PostgreSQL 9 and running a database backup through PgBouncer that can cause +an outage to GitLab. If you're still on PostgreSQL 9 and upgrading PostgreSQL isn't +an option, workarounds include having a dedicated application node just for backups, +configured to connect directly the primary database node as noted above. You're +advised to upgrade your PostgreSQL version though, GitLab 11.11 shipped with PostgreSQL +10.7, and that is the recommended version for GitLab 12+. ## Additional notes -This documentation is for GitLab Community and Enterprise Edition. We backup -GitLab.com and make sure your data is secure, but you can't use these methods -to export / backup your data yourself from GitLab.com. +This documentation is for GitLab Community and Enterprise Edition. We back up +GitLab.com and ensure your data is secure. You can't, however, use these +methods to export or back up your data yourself from GitLab.com. -Issues are stored in the database. They can't be stored in Git itself. +Issues are stored in the database, and can't be stored in Git itself. -To migrate your repositories from one server to another with an up-to-date version of -GitLab, you can use the [import Rake task](import.md) to do a mass import of the -repository. Note that if you do an import Rake task, rather than a backup restore, you -will have all your repositories, but not any other data. +To migrate your repositories from one server to another with an up-to-date +version of GitLab, use the [import Rake task](import.md) to do a mass import of +the repository. If you do an import Rake task rather than a backup restore, +you get all of your repositories, but no other data. ## Troubleshooting -The following are possible problems you might encounter with possible solutions. +The following are possible problems you might encounter, along with potential +solutions. ### Restoring database backup using Omnibus packages outputs warnings -If you are using backup restore procedures you might encounter the following warnings: +If you're using backup restore procedures, you may encounter the following +warning messages: ```plaintext psql:/var/opt/gitlab/backups/db/database.sql:22: ERROR: must be owner of extension plpgsql @@ -1036,22 +1089,31 @@ psql:/var/opt/gitlab/backups/db/database.sql:2931: WARNING: no privileges could psql:/var/opt/gitlab/backups/db/database.sql:2933: WARNING: no privileges were granted for "public" (two occurrences) ``` -Be advised that, backup is successfully restored in spite of these warnings. +Be advised that the backup is successfully restored in spite of these warning +messages. -The Rake task runs this as the `gitlab` user which does not have the superuser access to the database. When restore is initiated it will also run as `gitlab` user but it will also try to alter the objects it does not have access to. -Those objects have no influence on the database backup/restore but they give this annoying warning. +The Rake task runs this as the `gitlab` user, which doesn't have superuser +access to the database. When restore is initiated, it also runs as the `gitlab` +user, but it also tries to alter the objects it doesn't have access to. +Those objects have no influence on the database backup or restore, but display +a warning message. -For more information see similar questions on PostgreSQL issue tracker [here](https://www.postgresql.org/message-id/201110220712.30886.adrian.klaver@gmail.com) and [here](https://www.postgresql.org/message-id/2039.1177339749@sss.pgh.pa.us) as well as [stack overflow](https://stackoverflow.com/questions/4368789/error-must-be-owner-of-language-plpgsql). +For more information, see: + +- PostgreSQL issue tracker: + - [Not being a superuser](https://www.postgresql.org/message-id/201110220712.30886.adrian.klaver@gmail.com). + - [Having different owners](https://www.postgresql.org/message-id/2039.1177339749@sss.pgh.pa.us). + +- Stack Overflow: [Resulting errors](https://stackoverflow.com/questions/4368789/error-must-be-owner-of-language-plpgsql). ### When the secrets file is lost -If you have failed to [back up the secrets file](#storing-configuration-files), you'll -need to perform a number of steps to get GitLab working properly again. +If you didn't [back up the secrets file](#storing-configuration-files), you +must complete several steps to get GitLab working properly again. -The secrets file is responsible for storing the encryption key for several -columns containing sensitive information. If the key is lost, GitLab will be -unable to decrypt those columns. This will break a wide range of functionality, -including (but not restricted to): +The secrets file is responsible for storing the encryption key for the columns +that contain required, sensitive information. If the key is lost, GitLab can't +decrypt those columns, preventing access to the following items: - [CI/CD variables](../ci/variables/README.md) - [Kubernetes / GCP integration](../user/project/clusters/index.md) @@ -1061,38 +1123,40 @@ including (but not restricted to): - [Project mirroring](../user/project/repository/repository_mirroring.md) - [Web hooks](../user/project/integrations/webhooks.md) -In cases like CI/CD variables and runner authentication, you might -experience some unexpected behavior such as: +In cases like CI/CD variables and runner authentication, you can experience +unexpected behaviors, such as: - Stuck jobs. - 500 errors. -In this case, you are required to reset all the tokens for CI/CD variables -and runner authentication, which is described in more detail below. After -resetting the tokens, you should be able to visit your project and the jobs -will have started running again. Use the information in the following sections at your own risk. +In this case, you must reset all the tokens for CI/CD variables and +runner authentication, which is described in more detail in the following +sections. After resetting the tokens, you should be able to visit your project +and the jobs will have started running again. + +Use the information in the following sections at your own risk. #### Check for undecryptable values -You can check whether you have undecryptable values in the database using -the [Secrets Doctor Rake task](../administration/raketasks/doctor.md). +You can determine if you have undecryptable values in the database by using the +[Secrets Doctor Rake task](../administration/raketasks/doctor.md). #### Take a backup -You will need to directly modify GitLab data to work around your lost secrets file. +You must directly modify GitLab data to work around your lost secrets file. CAUTION: **Warning:** -Make sure you've taken a backup beforehand, particularly a full database backup. +Be sure to create a full database backup before attempting any changes. #### Disable user two-factor authentication (2FA) -Users with 2FA enabled will not be able to log into GitLab. In that case, -you need to [disable 2FA for everyone](../security/two_factor_authentication.md#disabling-2fa-for-everyone) -and then users will have to reactivate 2FA from scratch. +Users with 2FA enabled can't sign in to GitLab. In that case, you must +[disable 2FA for everyone](../security/two_factor_authentication.md#disabling-2fa-for-everyone), +after which users must reactivate 2FA. #### Reset CI/CD variables -1. Enter the DB console: +1. Enter the database console: For Omnibus GitLab packages: @@ -1106,14 +1170,14 @@ and then users will have to reactivate 2FA from scratch. sudo -u git -H bundle exec rails dbconsole -e production ``` -1. Check the `ci_group_variables` and `ci_variables` tables: +1. Examine the `ci_group_variables` and `ci_variables` tables: ```sql SELECT * FROM public."ci_group_variables"; SELECT * FROM public."ci_variables"; ``` - Those are the variables that you need to delete. + These are the variables that you need to delete. 1. Drop the table: @@ -1122,12 +1186,11 @@ and then users will have to reactivate 2FA from scratch. DELETE FROM ci_variables; ``` -1. You may need to reconfigure or restart GitLab for the changes to take - effect. +You may need to reconfigure or restart GitLab for the changes to take effect. #### Reset runner registration tokens -1. Enter the DB console: +1. Enter the database console: For Omnibus GitLab packages: @@ -1141,11 +1204,11 @@ and then users will have to reactivate 2FA from scratch. sudo -u git -H bundle exec rails dbconsole -e production ``` -1. Clear all the tokens for projects, groups, and the whole instance: +1. Clear all tokens for projects, groups, and the entire instance: CAUTION: **Caution:** - The last UPDATE operation will stop the runners being able to pick up - new jobs. You must register new runners. + The final `UPDATE` operation stops the runners from being able to pick + up new jobs. You must register new runners. ```sql -- Clear project tokens @@ -1160,7 +1223,7 @@ and then users will have to reactivate 2FA from scratch. #### Reset pending pipeline jobs -1. Enter the DB console: +1. Enter the database console: For Omnibus GitLab packages: @@ -1181,19 +1244,18 @@ and then users will have to reactivate 2FA from scratch. UPDATE ci_builds SET token = null, token_encrypted = null; ``` -A similar strategy can be employed for the remaining features - by removing the -data that cannot be decrypted, GitLab can be brought back into working order, -and the lost data can be manually replaced. +A similar strategy can be employed for the remaining features. By removing the +data that can't be decrypted, GitLab can be returned to operation, and the +lost data can be manually replaced. #### Fix project integrations -If you've lost your secrets, the -[projects' integrations settings pages](../user/project/integrations/index.md) -are probably generating 500 errors. +If you've lost your secrets, the [projects' integrations settings pages](../user/project/integrations/index.md) +are probably displaying `500` error messages. The fix is to truncate the `web_hooks` table: -1. Enter the DB console: +1. Enter the database console: For Omnibus GitLab packages: @@ -1207,7 +1269,7 @@ The fix is to truncate the `web_hooks` table: sudo -u git -H bundle exec rails dbconsole -e production ``` -1. Truncate the table +1. Truncate the table: ```sql -- truncate web_hooks table @@ -1216,11 +1278,11 @@ The fix is to truncate the `web_hooks` table: ### Container Registry push failures after restoring from a backup -If you use the [Container Registry](../user/packages/container_registry/index.md), you -may see pushes to the registry fail after restoring your backup on an Omnibus -GitLab instance after restoring the registry data. +If you use the [Container Registry](../user/packages/container_registry/index.md), +pushes to the registry may fail after restoring your backup on an Omnibus GitLab +instance after restoring the registry data. -These failures will mention permission issues in the registry logs, like: +These failures mention permission issues in the registry logs, similar to: ```plaintext level=error @@ -1230,9 +1292,9 @@ err.detail="filesystem: mkdir /var/opt/gitlab/gitlab-rails/shared/registry/docke err.message="unknown error" ``` -This is caused by the restore being run as the unprivileged user `git` which was -unable to assign the correct ownership to the registry files during the restore -([issue 62759](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/62759 "Incorrect permissions on registry filesystem after restore")). +This issue is caused by the restore running as the unprivileged user `git`, +which is unable to assign the correct ownership to the registry files during +the restore process ([issue 62759](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/62759 "Incorrect permissions on registry filesystem after restore")). To get your registry working again: @@ -1240,14 +1302,12 @@ To get your registry working again: sudo chown -R registry:registry /var/opt/gitlab/gitlab-rails/shared/registry/docker ``` -NOTE: **Note:** -If you have changed the default filesystem location for the registry, you will -want to run the `chown` against your custom location instead of -`/var/opt/gitlab/gitlab-rails/shared/registry/docker`. +If you changed the default filesystem location for the registry, run `chown` +against your custom location, instead of `/var/opt/gitlab/gitlab-rails/shared/registry/docker`. ### Backup fails to complete with Gzip error -While running the backup, you may receive a Gzip error: +When running the backup, you may receive a Gzip error message: ```shell sudo /opt/gitlab/bin/gitlab-backup create @@ -1259,7 +1319,8 @@ gzip: stdout: Input/output error Backup failed ``` -If this happens, check the following: +If this happens, examine the following: -1. Confirm there is sufficient disk space for the Gzip operation. -1. If NFS is being used, check if the mount option `timeout` is set. The default is `600`, and changing this to smaller values have resulted in this error. +- Confirm there is sufficient disk space for the Gzip operation. +- If NFS is being used, check if the mount option `timeout` is set. The + default is `600`, and changing this to smaller values results in this error. diff --git a/doc/raketasks/cleanup.md b/doc/raketasks/cleanup.md index c4046b36c55..0b3da39d3d5 100644 --- a/doc/raketasks/cleanup.md +++ b/doc/raketasks/cleanup.md @@ -146,7 +146,7 @@ NOTE: **Note:** These commands will not work for artifacts stored on [object storage](../administration/object_storage.md). -When you notice there are more job artifacts files on disk than there +When you notice there are more job artifacts files and/or directories on disk than there should be, you can run: ```shell @@ -157,7 +157,7 @@ This command: - Scans through the entire artifacts folder. - Checks which files still have a record in the database. -- If no database record is found, the file is deleted from disk. +- If no database record is found, the file and directory is deleted from disk. By default, this task does not delete anything but shows what it can delete. Run the command with `DRY_RUN=false` if you actually want to diff --git a/doc/raketasks/import.md b/doc/raketasks/import.md index c8ca92b4bae..b0603a76211 100644 --- a/doc/raketasks/import.md +++ b/doc/raketasks/import.md @@ -127,11 +127,11 @@ Bare repositories are **not** importable by GitLab 10.4 to GitLab 11.6, if all t - It was not renamed, transferred, or migrated to [hashed storage](../administration/repository_storage_types.md#hashed-storage) in GitLab 10.4 to GitLab 11.6. - Its ancestor namespaces were not renamed or transferred in GitLab 10.4 to GitLab 11.6. -[Since GitLab 11.6](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/41776), all +[In GitLab 11.6](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/41776) and later, all bare repositories are importable. To manually migrate repositories yourself (for GitLab 10.4 to GitLab 11.6), you can use the -[Rails console](../administration/troubleshooting/debug.md#starting-a-rails-console-session) +[Rails console](../administration/operations/rails_console.md#starting-a-rails-console-session) to do so. In a Rails console session, run the following to migrate a project: ```ruby diff --git a/doc/security/asset_proxy.md b/doc/security/asset_proxy.md index 91a35c2f2a9..fdceecdf386 100644 --- a/doc/security/asset_proxy.md +++ b/doc/security/asset_proxy.md @@ -25,7 +25,7 @@ A Camo server is used to act as the proxy. To install a Camo server as an asset proxy: 1. Deploy a `go-camo` server. Helpful instructions can be found in - [building catus/go-camo](https://github.com/cactus/go-camo#building). + [building cactus/go-camo](https://github.com/cactus/go-camo#building). 1. Make sure your instance of GitLab is running, and that you have created a private API token. Using the API, configure the asset proxy settings on your GitLab instance. For example: diff --git a/doc/security/passwords_for_integrated_authentication_methods.md b/doc/security/passwords_for_integrated_authentication_methods.md index 704af49b2d2..f2597ef1578 100644 --- a/doc/security/passwords_for_integrated_authentication_methods.md +++ b/doc/security/passwords_for_integrated_authentication_methods.md @@ -11,4 +11,4 @@ However, to maintain data consistency, GitLab requires passwords for all user ac For such accounts, we use the [`friendly_token`](https://github.com/heartcombo/devise/blob/f26e05c20079c9acded3c0ee16da0df435a28997/lib/devise.rb#L492) method provided by the Devise gem to generate a random, unique and secure password and sets it as the account password during sign up. -The length of the generated password is the set based on the value of [maximum password length](password_length_limits.md#modify-maximum-password-length-using-configuration-file) as set in the Devise configuation. The default value is 128 characters. +The length of the generated password is the set based on the value of [maximum password length](password_length_limits.md#modify-maximum-password-length-using-configuration-file) as set in the Device configuration. The default value is 128 characters. diff --git a/doc/security/project_import_decompressed_archive_size_limits.md b/doc/security/project_import_decompressed_archive_size_limits.md index dd67db23d6b..16821e1f192 100644 --- a/doc/security/project_import_decompressed_archive_size_limits.md +++ b/doc/security/project_import_decompressed_archive_size_limits.md @@ -17,7 +17,7 @@ If you have a project with decompressed size exceeding this limit, it is possible to disable the validation by turning off the `validate_import_decompressed_archive_size` feature flag. -Start a [Rails console](../administration/troubleshooting/debug.md#starting-a-rails-console-session). +Start a [Rails console](../administration/operations/rails_console.md#starting-a-rails-console-session). ```ruby # Disable diff --git a/doc/security/rack_attack.md b/doc/security/rack_attack.md index d3de2222c39..b386917f399 100644 --- a/doc/security/rack_attack.md +++ b/doc/security/rack_attack.md @@ -4,8 +4,6 @@ type: reference, howto # Rack Attack initializer -## Overview - [Rack Attack](https://github.com/kickstarter/rack-attack), also known as Rack::Attack, is a Ruby gem that is meant to protect GitLab with the ability to customize throttling and to block user IP addresses. diff --git a/doc/security/rate_limits.md b/doc/security/rate_limits.md index af2c14be2cd..9e754cf1917 100644 --- a/doc/security/rate_limits.md +++ b/doc/security/rate_limits.md @@ -28,6 +28,25 @@ similarly mitigated by a rate limit. - [Protected paths](../user/admin_area/settings/protected_paths.md). - [Import/Export rate limits](../user/admin_area/settings/import_export_rate_limits.md). +## Non-configurable limits + +### Repository archives + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/25750) in GitLab 12.9. + +There is a rate limit for [downloading repository archives](../api/repositories.md#get-file-archive), +which applies to the project and to the user initiating the download either through the UI or the API. + +The **rate limit** is 5 requests per minute per user. + +### Webhook Testing + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/commit/35bc85c3ca093fee58d60dacdc9ed1fd9a15adec) in GitLab 13.4. + +There is a rate limit for [testing webhooks](../user/project/integrations/webhooks.md#testing-webhooks), which prevents abuse of the webhook functionality. + +The **rate limit** is 5 requests per minute per user. + ## Rack Attack initializer This method of rate limiting is cumbersome, but has some advantages. It allows diff --git a/doc/security/two_factor_authentication.md b/doc/security/two_factor_authentication.md index 9d49e1d3af2..995dea7809e 100644 --- a/doc/security/two_factor_authentication.md +++ b/doc/security/two_factor_authentication.md @@ -65,9 +65,22 @@ The following are important notes about 2FA: 2FA enabled, 2FA is **not** required for those individually added members. - If there are multiple 2FA requirements (for example, group + all users, or multiple groups) the shortest grace period will be used. +- It is possible to disallow subgroups from setting up their own 2FA requirements. + Navigate to the top-level group's **Settings > General > Permissions, LFS, 2FA > Two-factor authentication** and uncheck the **Allow subgroups to set up their own two-factor authentication rule** field. This action will cause all subgroups with 2FA requirements to stop requiring that from their members. ## Disabling 2FA for everyone +CAUTION: **Caution:** +Disabling 2FA for everyone does not disable the [enforce 2FA for all users](#enforcing-2fa-for-all-users) +or [enforce 2FA for all users in a group](#enforcing-2fa-for-all-users-in-a-group) +settings. In addition to the steps in this section, you will need to disable any enforced 2FA +settings so users aren't asked to set up 2FA again, the next time the user signs in to GitLab. +Disabling 2FA for everyone does not disable the [enforce 2FA for all users](#enforcing-2fa-for-all-users) +or [enforce 2FA for all users in a group](#enforcing-2fa-for-all-users-in-a-group) +settings if they have been configured. In addition to the steps in this section, +you will need to disable any enforced 2FA settings so users aren't asked to setup +2FA again when the next login to GitLab. + There may be some special situations where you want to disable 2FA for everyone even when forced 2FA is disabled. There is a Rake task for that: diff --git a/doc/ssh/README.md b/doc/ssh/README.md index 1c5654ae96c..9d851edb688 100644 --- a/doc/ssh/README.md +++ b/doc/ssh/README.md @@ -1,6 +1,6 @@ --- -stage: Create -group: Source Code +stage: Manage +group: Access info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" type: howto, reference --- @@ -24,7 +24,7 @@ connections to GitLab repositories. ## Requirements To support SSH, GitLab requires the installation of the OpenSSH client, which -comes pre-installed on GNU/Linux and macOS, but not on Windows. +comes pre-installed on GNU/Linux and macOS, as well as on Windows 10. Make sure that your system includes SSH version 6.5 or newer, as that excludes the now insecure MD5 signature scheme. The following command returns the version of diff --git a/doc/subscriptions/gitlab_com/index.md b/doc/subscriptions/gitlab_com/index.md index bce61cdad66..128e9d07282 100644 --- a/doc/subscriptions/gitlab_com/index.md +++ b/doc/subscriptions/gitlab_com/index.md @@ -39,8 +39,7 @@ subscription according to the maximum number of users enabled at once. You can add and remove users during the subscription period, as long as the total users at any given time is within your subscription count. -Every occupied seat, whether by person, job, or bot is counted in the subscription, -with the following exception: +Every occupied seat is counted in the subscription, with the following exception: - Members with Guest permissions on a Gold subscription. @@ -149,23 +148,13 @@ as the number of _users over license_, you must pay for the excess number of users either before renewal, or at the time of renewal. This is also known the _true up_ process. -There is no self-service option for purchasing additional seats. You must -request a quotation from GitLab Sales. To do so, contact GitLab via our -[support form](https://support.gitlab.com/hc/en-us/requests/new?ticket_form_id=360000071293). - -The amount charged per seat is calculated by one of the following methods: - -- If paid before renewal, the amount per seat is calculated on a prorated basis. - For example, if the user was added 3 months before the end of the subscription - period, the amount owing is calculated as: (3 / 12) x annual fee. -- If paid on renewal, the amount per seat is the standard annual fee. - ### Renew or change a GitLab.com subscription -NOTE: **Note:** -To renew for more users than are currently active in your GitLab.com plan, -contact our sales team via `renewals@gitlab.com` for assistance as this can't be -done in the Customers Portal. +You can adjust the number of users before renewing your GitLab.com subscription. + +- To renew for more users than are currently included in your GitLab.com plan, [add users to your subscription](#add-users-to-your-subscription). +- To renew for fewer users than are currently included in your GitLab.com plan, +either [disable](../../user/admin_area/activating_deactivating_users.md#deactivating-a-user) or [block](../../user/admin_area/blocking_unblocking_users.md#blocking-a-user) the user accounts you no longer need. For details on upgrading your subscription tier, see [Upgrade your GitLab.com subscription tier](#upgrade-your-gitlabcom-subscription-tier). @@ -187,12 +176,35 @@ generated for the renewal and available for viewing or download in the during the renewal process, contact our [support team](https://support.gitlab.com/hc/en-us/requests/new?ticket_form_id=360000071293) for assistance. +## Add users to your subscription + +You can add users to your subscription at any time during the subscription period. The cost of +additional users added during the subscription period is prorated from the date of purchase through +the end of the subscription period. + +To add users to a subscription: + +1. Log in to the [Customers Portal](https://customers.gitlab.com/). +1. Navigate to the **Manage Purchases** page. +1. Select **Add more seats** on the relevant subscription card. +1. Enter the number of additional users. +1. Select **Proceed to checkout**. +1. Review the **Subscription Upgrade Detail**. The system lists the total price for all users on the + system and a credit for what you've already paid. You will only be charged for the net change. +1. Select **Confirm Upgrade**. + +The following will be emailed to you: + +- A payment receipt. You can also access this information in the Customers Portal under + [**View invoices**](https://customers.gitlab.com/receipts). + ## Upgrade your GitLab.com subscription tier To upgrade your [GitLab tier](https://about.gitlab.com/pricing/): 1. Log in to the [Customers Portal](https://customers.gitlab.com/customers/sign_in). -1. Select the **Upgrade** button on the relevant subscription card on the [Manage purchases](https://customers.gitlab.com/subscriptions) page. +1. Select **Upgrade** on the relevant subscription card on the + [Manage purchases](https://customers.gitlab.com/subscriptions) page. 1. Select the desired upgrade. 1. Confirm the active form of payment, or add a new form of payment. 1. Check the **I accept the Privacy Policy and Terms of Service** checkbox. diff --git a/doc/subscriptions/self_managed/index.md b/doc/subscriptions/self_managed/index.md index a0eb998c545..5f232bd4ed2 100644 --- a/doc/subscriptions/self_managed/index.md +++ b/doc/subscriptions/self_managed/index.md @@ -23,15 +23,14 @@ For instances that aren't offline or on a closed network, the maximum number of simultaneous users in the self-managed installation is checked each quarter, using [Seat Link](#seat-link). -Every occupied seat, whether by person, job, or bot is counted in the subscription, -with the following exceptions: +Every occupied seat is counted in the subscription, with the following exceptions: -- [Deactivated](../../user/admin_area/activating_deactivating_users.md#deactivating-a-user) and +- [Deactivated](../../user/admin_area/activating_deactivating_users.md#deactivating-a-user), [pending approval](../../user/admin_area/approving_users.md) and [blocked](../../user/admin_area/blocking_unblocking_users.md) users who are restricted prior to the renewal of a subscription won't be counted as active users for the renewal subscription. They may count as active users in the subscription period in which they were originally added. - Members with Guest permissions on an Ultimate subscription. -- GitLab-created service accounts: `Ghost User`, `Support Bot` and [`Project bot users`](../../user/project/settings/project_access_tokens.md#project-bot-users). +- GitLab-created service accounts: `Ghost User` and bots (`Support Bot`, [`Project bot users`](../../user/project/settings/project_access_tokens.md#project-bot-users), etc.). ### Users statistics @@ -254,20 +253,24 @@ production: &base ## Upgrade your subscription tier -To upgrade your [GitLab tier](https://about.gitlab.com/pricing/), contact our sales team as this -can't be done in the Customers Portal. You can either send an email to `renewals@gitlab.com`, or -complete the [**Contact Sales**](https://about.gitlab.com/sales/) form. Include details of which subscription you want to upgrade and the desired tier in your message. +To upgrade your [GitLab tier](https://about.gitlab.com/pricing/): -After messaging the sales team, the workflow is as follows: +1. Log in to the [Customers Portal](https://customers.gitlab.com/customers/sign_in). +1. Select the **Upgrade** button on the relevant subscription card on the + [Manage purchases](https://customers.gitlab.com/subscriptions) page. +1. Select the desired upgrade. +1. Confirm the active form of payment, or add a new form of payment. +1. Select the **I accept the Privacy Policy and Terms of Service** checkbox. +1. Select **Purchase**. -1. Receive a reply from the sales team, asking for confirmation of the upgrade. -1. Reply to the sales team, confirming details of the upgrade. -1. Receive a quote from the sales team. -1. Sign and return the quote. -1. Receive the new license. -1. Upload the new license. For details, see [Uploading your license](../../user/admin_area/license.md#uploading-your-license). +The following is emailed to you: -The new subscription tier is active when the license file is uploaded. +- A payment receipt. You can also access this information in the Customers Portal under + [**View invoices**](https://customers.gitlab.com/receipts). +- A new license. + +[Upload the new license](../../user/admin_area/license.md#uploading-your-license) to your instance. +The new tier takes effect when the new license is uploaded. ## Subscription expiry diff --git a/doc/system_hooks/system_hooks.md b/doc/system_hooks/system_hooks.md index feea038b7d2..d2bdf935aa1 100644 --- a/doc/system_hooks/system_hooks.md +++ b/doc/system_hooks/system_hooks.md @@ -6,26 +6,27 @@ type: reference Your GitLab instance can perform HTTP POST requests on the following events: +- `group_create` +- `group_destroy` +- `group_rename` +- `key_create` +- `key_destroy` - `project_create` - `project_destroy` - `project_rename` - `project_transfer` - `project_update` +- `repository_update` +- `user_add_to_group` - `user_add_to_team` -- `user_remove_from_team` -- `user_update_for_team` - `user_create` - `user_destroy` - `user_failed_login` -- `user_rename` -- `key_create` -- `key_destroy` -- `group_create` -- `group_destroy` -- `group_rename` -- `user_add_to_group` - `user_remove_from_group` +- `user_remove_from_team` +- `user_rename` - `user_update_for_group` +- `user_update_for_team` The triggers for most of these are self-explanatory, but `project_update` and `project_rename` deserve some clarification: `project_update` is fired any time an attribute of a project is changed (name, description, tags, etc.) *unless* the `path` attribute is also changed. In that case, a `project_rename` is triggered instead (so that, for instance, if all you care about is the repository URL, you can just listen for `project_rename`). diff --git a/doc/telemetry/index.md b/doc/telemetry/index.md index 977b93b712e..4583dbe4753 100644 --- a/doc/telemetry/index.md +++ b/doc/telemetry/index.md @@ -1,5 +1,5 @@ --- -redirect_to: '../development/telemetry/index.md' +redirect_to: '../development/product_analytics/index.md' --- -This document was moved to [another location](../development/telemetry/index.md). +This document was moved to [another location](../development/product_analytics/index.md). diff --git a/doc/telemetry/snowplow.md b/doc/telemetry/snowplow.md index 977b93b712e..643893bcc22 100644 --- a/doc/telemetry/snowplow.md +++ b/doc/telemetry/snowplow.md @@ -1,5 +1,5 @@ --- -redirect_to: '../development/telemetry/index.md' +redirect_to: '../development/product_analytics/snowplow.md' --- -This document was moved to [another location](../development/telemetry/index.md). +This document was moved to [another location](../development/product_analytics/snowplow.md). diff --git a/doc/topics/autodevops/customize.md b/doc/topics/autodevops/customize.md index 13aa8f7e035..0026cf4d18a 100644 --- a/doc/topics/autodevops/customize.md +++ b/doc/topics/autodevops/customize.md @@ -332,7 +332,7 @@ applications. | `AUTO_DEVOPS_CHART_REPOSITORY_USERNAME` | From GitLab 11.11, used to set a username to connect to the Helm repository. Defaults to no credentials. Also set `AUTO_DEVOPS_CHART_REPOSITORY_PASSWORD`. | | `AUTO_DEVOPS_CHART_REPOSITORY_PASSWORD` | From GitLab 11.11, used to set a password to connect to the Helm repository. Defaults to no credentials. Also set `AUTO_DEVOPS_CHART_REPOSITORY_USERNAME`. | | `AUTO_DEVOPS_DEPLOY_DEBUG` | From GitLab 13.1, if this variable is present, Helm will output debug logs. | -| `AUTO_DEVOPS_ALLOW_TO_FORCE_DEPLOY_V<N>` | From [auto-deploy-image](https://gitlab.com/gitlab-org/cluster-integration/auto-deploy-image) v1.0.0, if this variable is present, a new major version of chart is forcibly deployed. [More details](upgrading_chart.md#ignore-warning-and-continue-deploying) | +| `AUTO_DEVOPS_ALLOW_TO_FORCE_DEPLOY_V<N>` | From [auto-deploy-image](https://gitlab.com/gitlab-org/cluster-integration/auto-deploy-image) v1.0.0, if this variable is present, a new major version of chart is forcibly deployed. For more information, see [Ignore warnings and continue deploying](upgrading_auto_deploy_dependencies.md#ignore-warnings-and-continue-deploying). | | `AUTO_DEVOPS_MODSECURITY_SEC_RULE_ENGINE` | From GitLab 12.5, used in combination with [ModSecurity feature flag](../../user/clusters/applications.md#web-application-firewall-modsecurity) to toggle [ModSecurity's `SecRuleEngine`](https://github.com/SpiderLabs/ModSecurity/wiki/Reference-Manual-(v2.x)#SecRuleEngine) behavior. Defaults to `DetectionOnly`. | | `BUILDPACK_URL` | Buildpack's full URL. Can point to either [a Git repository URL or a tarball URL](#custom-buildpacks). | | `CANARY_ENABLED` | From GitLab 11.0, used to define a [deploy policy for canary environments](#deploy-policy-for-canary-environments). | @@ -391,6 +391,8 @@ The following table lists variables used to disable jobs. | `REVIEW_DISABLED` | From GitLab 11.0, used to disable the `review` and the manual `review:stop` job. If the variable is present, these jobs won't be created. | | `SAST_DISABLED` | From GitLab 11.0, used to disable the `sast` job. If the variable is present, the job won't be created. | | `TEST_DISABLED` | From GitLab 11.0, used to disable the `test` job. If the variable is present, the job won't be created. | +| `SECRET_DETECTION_DISABLED` | From GitLab 13.1, used to disable the `secret_detection` job. If the variable is present, the job won't be created. | +| `CODE_INTELLIGENCE_DISABLED` | From GitLab 13.6, used to disable the `code_intelligence` job. If the variable is present, the job won't be created. | ### Application secret variables diff --git a/doc/topics/autodevops/index.md b/doc/topics/autodevops/index.md index a39f93a26e1..1952fadc076 100644 --- a/doc/topics/autodevops/index.md +++ b/doc/topics/autodevops/index.md @@ -3,10 +3,11 @@ > - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/37115) in GitLab 10.0. > - Generally available on GitLab 11.0. -Auto DevOps provides pre-defined CI/CD configuration allowing you to automatically -detect, build, test, deploy, and monitor your applications. Leveraging CI/CD -best practices and tools, Auto DevOps aims to simplify the setup and execution -of a mature and modern software development lifecycle. +Auto DevOps are default CI/CD templates that auto-discover the source code you have. They +enable GitLab to automatically detect, build, test, deploy, and monitor your applications. +Leveraging [CI/CD best practices](../../ci/pipelines/pipeline_efficiency.md) and tools, +Auto DevOps aims to simplify the setup and execution of a mature and modern software +development lifecycle. ## Overview @@ -83,9 +84,9 @@ project in a simple and automatic way: 1. [Auto Build](stages.md#auto-build) 1. [Auto Test](stages.md#auto-test) -1. [Auto Code Quality](stages.md#auto-code-quality) **(STARTER)** -1. [Auto SAST (Static Application Security Testing)](stages.md#auto-sast) **(ULTIMATE)** -1. [Auto Secret Detection](stages.md#auto-secret-detection) **(ULTIMATE)** +1. [Auto Code Quality](stages.md#auto-code-quality) +1. [Auto SAST (Static Application Security Testing)](stages.md#auto-sast) +1. [Auto Secret Detection](stages.md#auto-secret-detection) 1. [Auto Dependency Scanning](stages.md#auto-dependency-scanning) **(ULTIMATE)** 1. [Auto License Compliance](stages.md#auto-license-compliance) **(ULTIMATE)** 1. [Auto Container Scanning](stages.md#auto-container-scanning) **(ULTIMATE)** @@ -94,6 +95,7 @@ project in a simple and automatic way: 1. [Auto Deploy](stages.md#auto-deploy) 1. [Auto Browser Performance Testing](stages.md#auto-browser-performance-testing) **(PREMIUM)** 1. [Auto Monitoring](stages.md#auto-monitoring) +1. [Auto Code Intelligence](stages.md#auto-code-intelligence) As Auto DevOps relies on many different components, you should have a basic knowledge of the following: @@ -317,7 +319,7 @@ metadata: name: gitlab-managed-apps-default-proxy namespace: gitlab-managed-apps spec: - env: + env: - name: http_proxy value: "PUT_YOUR_HTTP_PROXY_HERE" - name: https_proxy @@ -417,6 +419,54 @@ DANGER: **Danger:** Setting `POSTGRES_ENABLED` to `false` permanently deletes any existing channel 1 database for your environment. +### Error: unable to recognize "": no matches for kind "Deployment" in version "extensions/v1beta1" + +After upgrading your Kubernetes cluster to [v1.16+](stages.md#kubernetes-116), +you may encounter this message when deploying with Auto DevOps: + +```plaintext +UPGRADE FAILED +Error: failed decoding reader into objects: unable to recognize "": no matches for kind "Deployment" in version "extensions/v1beta1" +``` + +This can occur if your current deployments on the environment namespace were deployed with a +deprecated/removed API that doesn't exist in Kubernetes v1.16+. For example, +if [your in-cluster PostgreSQL was installed in a legacy way](#detected-an-existing-postgresql-database), +the resource was created via the `extensions/v1beta1` API. However, the deployment resource +was moved to the `app/v1` API in v1.16. + +To recover such outdated resources, you must convert the current deployments by mapping legacy APIs +to newer APIs. There is a helper tool called [`mapkubeapis`](https://github.com/hickeyma/helm-mapkubeapis) +that works for this problem. Follow these steps to use the tool in Auto DevOps: + +1. Modify your `.gitlab-ci.yml` with: + + ```yaml + include: + - template: Auto-DevOps.gitlab-ci.yml + - remote: https://gitlab.com/shinya.maeda/ci-templates/-/raw/master/map-deprecated-api.gitlab-ci.yml + + variables: + HELM_VERSION_FOR_MAPKUBEAPIS: "v2" # If you're using auto-depoy-image v2 or above, please specify "v3". + ``` + +1. Run the job `<environment-name>:map-deprecated-api`. Ensure that this job succeeds before moving + to the next step. You should see something like the following output: + + ```shell + 2020/10/06 07:20:49 Found deprecated or removed Kubernetes API: + "apiVersion: extensions/v1beta1 + kind: Deployment" + Supported API equivalent: + "apiVersion: apps/v1 + kind: Deployment" + ``` + +1. Revert your `.gitlab-ci.yml` to the previous version. You no longer need to include the + supplemental template `map-deprecated-api`. + +1. Continue the deployments as usual. + ## Development guides [Development guide for Auto DevOps](../../development/auto_devops.md) diff --git a/doc/topics/autodevops/stages.md b/doc/topics/autodevops/stages.md index b58c369714e..d3f02889a2e 100644 --- a/doc/topics/autodevops/stages.md +++ b/doc/topics/autodevops/stages.md @@ -124,7 +124,10 @@ The supported buildpacks are: If your application needs a buildpack that is not in the above list, you might want to use a [custom buildpack](customize.md#custom-buildpacks). -## Auto Code Quality **(STARTER)** +## Auto Code Quality + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/1984) in [GitLab Starter](https://about.gitlab.com/pricing/) 9.3. +> - Made [available in all tiers](https://gitlab.com/gitlab-org/gitlab/-/issues/212499) in GitLab 13.2. Auto Code Quality uses the [Code Quality image](https://gitlab.com/gitlab-org/ci-cd/codequality) to run @@ -133,9 +136,10 @@ report, it's uploaded as an artifact which you can later download and check out. The merge request widget also displays any [differences between the source and target branches](../../user/project/merge_requests/code_quality.md). -## Auto SAST **(ULTIMATE)** +## Auto SAST -> Introduced in [GitLab Ultimate](https://about.gitlab.com/pricing/) 10.3. +> - Introduced in [GitLab Ultimate](https://about.gitlab.com/pricing/) 10.3. +> - Select functionality made available in all tiers beginning in 13.1 Static Application Security Testing (SAST) uses the [SAST Docker image](https://gitlab.com/gitlab-org/security-products/sast) to run static @@ -151,9 +155,10 @@ warnings. To learn more about [how SAST works](../../user/application_security/sast/index.md), see the documentation. -## Auto Secret Detection **(ULTIMATE)** +## Auto Secret Detection -> Introduced in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.1. +> - Introduced in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.1. +> - [Select functionality made available in all tiers](../../user/application_security/secret_detection/#making-secret-detection-available-to-all-gitlab-tiers) in 13.3 Secret Detection uses the [Secret Detection Docker image](https://gitlab.com/gitlab-org/security-products/analyzers/secrets) to run Secret Detection on the current code, and checks for leaked secrets. The @@ -190,7 +195,7 @@ see the documentation. > Introduced in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.0. License Compliance uses the -[License Compliance Docker image](https://gitlab.com/gitlab-org/security-products/license-management) +[License Compliance Docker image](https://gitlab.com/gitlab-org/security-products/analyzers/license-finder) to search the project dependencies for their license. The Auto License Compliance stage is skipped on licenses other than [Ultimate](https://about.gitlab.com/pricing/). @@ -242,7 +247,7 @@ Helm, which you can [customize](customize.md#custom-helm-chart). The application into the [Kubernetes namespace](../../user/project/clusters/index.md#deployment-variables) for the environment. -Since GitLab 11.4, [local Tiller](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/22036) is +In GitLab 11.4 and later, [local Tiller](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/22036) is used. Previous versions of GitLab had a Tiller installed in the project namespace. @@ -360,7 +365,7 @@ chart to deploy the application into the [Kubernetes namespace](../../user/project/clusters/index.md#deployment-variables) for the environment. -Since GitLab 11.4, a +In GitLab 11.4 and later, a [local Tiller](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/22036) is used. Previous versions of GitLab had a Tiller installed in the project namespace. @@ -466,7 +471,7 @@ application runs. ### Upgrade auto-deploy-app Chart -You can upgrade auto-deploy-app chart by following the [upgrade guide](upgrading_chart.md). +You can upgrade the auto-deploy-app chart by following the [upgrade guide](upgrading_auto_deploy_dependencies.md). ### Workers @@ -660,3 +665,7 @@ To use Auto Monitoring: whole Kubernetes cluster, navigate to **Operations > Metrics**.  + +## Auto Code Intelligence + +Code Intelligence is powered by [LSIF](https://lsif.dev/) and available for Go at this stage. We'll support more languages as they become available. diff --git a/doc/topics/autodevops/upgrading_auto_deploy_dependencies.md b/doc/topics/autodevops/upgrading_auto_deploy_dependencies.md new file mode 100644 index 00000000000..1aefb6b34df --- /dev/null +++ b/doc/topics/autodevops/upgrading_auto_deploy_dependencies.md @@ -0,0 +1,262 @@ +--- +stage: Release +group: Progressive Delivery +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: reference +--- + +# Upgrading deployments for newer Auto Deploy dependencies (Auto Deploy template, auto-deploy-image and auto-deploy-app chart) + +[Auto Deploy](stages.md#auto-deploy) is a feature that deploys your application to a Kubernetes cluster. +It consists of several dependencies: + +- [Auto Deploy template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml) is a set of pipeline jobs and scripts that makes use of `auto-deploy-image`. +- [`auto-deploy-image`](https://gitlab.com/gitlab-org/cluster-integration/auto-deploy-image) is the executable image that communicates with the Kubernetes cluster. +- [`auto-deploy-app chart`](https://gitlab.com/gitlab-org/cluster-integration/auto-deploy-image/-/tree/master/assets/auto-deploy-app) is the Helm chart for deploying your application. + +The `auto-deploy-image` and `auto-deploy-app` charts use [Semantic Versioning](https://semver.org/). +By default, your Auto DevOps project keeps using the stable and non-breaking version. +However, these dependencies could be upgraded in a major version release of GitLab +with breaking changes requiring you to upgrade your deployments. + +This guide explains how to upgrade your deployments with newer or different major versions of Auto Deploy dependencies. + +## Verify dependency versions + +The process to check the current versions differs depending on which template you +are using. First verify which template is in use: + +- For self-managed instances, the [stable Auto Deploy template bundled with the GitLab package](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml) + is being used. +- [The GitLab.com stable Auto Deploy template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml) + is being used if **one** of the following is true: + - Your Auto DevOps project doesn't have a `.gitlab-ci.yml` file. + - Your Auto DevOps project has a `.gitlab-ci.yml` and [includes](../../ci/yaml/README.md#includetemplate) + the `Auto-DevOps.gitlab-ci.yml` template. +- [The latest Auto Deploy template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml) + is being used if **both** of the following is true: + - Your Auto DevOps project has a `.gitlab-ci.yml` file and [includes](../../ci/yaml/README.md#includetemplate) + the `Auto-DevOps.gitlab-ci.yml` template. + - It also includes [the latest Auto Deploy template](#early-adopters) + +If you know what template is being used: + +- The `auto-deploy-image` version is in the template (for example `auto-deploy-image:v1.0.3`). +- The `auto-deploy-app` chart version is [in the auto-deploy-image repository](https://gitlab.com/gitlab-org/cluster-integration/auto-deploy-image/-/blob/v1.0.3/assets/auto-deploy-app/Chart.yaml) + (for example `version: 1.0.3`). + +## Compatibility + +The following table explains the version compatibility between GitLab and Auto Deploy dependencies: + +| GitLab version | `auto-deploy-image` version | Notes | +|------------------|-----------------------------|-------| +| v10.0 to v14.0 | v0.1.0 to v2.0.0 | v0 and v1 auto-deploy-image are backwards compatible. | +| v13.4 and higher | v2.0.0 and higher | v2 auto-deploy-image contains breaking changes, as explained in the [upgrade guide](#upgrade-deployments-to-the-v2-auto-deploy-image). | + +You can find the current stable version of auto-deploy-image in the [Auto Deploy stable template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml). + +## Upgrade Guide + +Projects using Auto DevOps must use the unmodified chart managed by GitLab. +[Customized charts](customize.md#custom-helm-chart) are unsupported. + +### Upgrade deployments to the v1 `auto-deploy-image` + +The v1 chart is backward compatible with the v0 chart, so no configuration changes are needed. + +### Upgrade deployments to the v2 `auto-deploy-image` + +The v2 auto-deploy-image contains multiple dependency and architectural changes. +If your Auto DevOps project has an active environment deployed with the v1 `auto-deploy-image`, +please proceed with the following upgrade guide. Otherwise, you can skip this process. + +#### Helm 3 + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/228609) in GitLab 13.4. + +The `auto-deploy-image` uses the Helm binary to manipulate the releases. +Previously, `auto-deploy-image` used Helm v2, which used Tiller in a cluster. +In the v2 `auto-deploy-image`, it uses Helm v3 that doesn't require Tiller anymore. + +If your Auto DevOps project has an active environment that was deployed with the v1 +`auto-deploy-image`, use the following steps to upgrade to v2, which uses Helm 3: + +1. Modify your `.gitlab-ci.yml` with: + + ```yaml + include: + - template: Auto-DevOps.gitlab-ci.yml + - remote: https://gitlab.com/hfyngvason/ci-templates/-/raw/master/Helm-2to3.gitlab-ci.yml + + variables: + # If this variable is not present, the migration jobs will not show up + MIGRATE_HELM_2TO3: "true" + + .auto-deploy: + image: registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v2.0.0-beta.1 + variables: + AUTO_DEVOPS_FORCE_DEPLOY_V2: 1 + ``` + +1. Run the `<environment-name>:helm-2to3:migrate` job. +1. Deploy your environment as usual. This deployment uses Helm 3. +1. If the deployment succeeds, you can safely run `environment:helm-2to3:cleanup`. + This deletes all Helm 2 release data from the namespace. + + If you accidentally delete the Helm 2 releases before you are ready, the `<environment-name>:helm2to3:migrate` + job saves a backup for 1 week in a job artifact called `helm-2-release-backups`. + The backup is in a Kubernetes manifest file that can be restored using + `kubectl apply -f $backup`. +1. Remove the `MIGRATE_HELM_2TO3` variable. + +#### Traffic routing change for canary deployments and incremental rollouts + +> [Introduced](https://gitlab.com/gitlab-org/cluster-integration/auto-deploy-image/-/merge_requests/109) in GitLab 13.4. + +Auto Deploy supports advanced deployment strategies such as [canary deployments](customize.md#deploy-policy-for-canary-environments) +and [incremental rollouts](../../ci/environments/incremental_rollouts.md). + +Previously, `auto-deploy-image` created one service to balance the traffic between +unstable and stable tracks by changing the replica ratio. In the v2 `auto-deploy-image`, +it controls the traffic with [Canary Ingress](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/#canary). + +For more details, see the [v2 `auto-deploy-app` chart resource architecture](#v2-chart-resource-architecture). + +If your Auto DevOps project has active `canary` or `rollout` track releases in the +`production` environment deployed with the v1 `auto-deploy-image`, use the following +steps to upgrade to v2: + +1. Verify your project is [using the v1 `auto-deploy-image`](#verify-dependency-versions). + If not, [specify the version](#use-a-specific-version-of-auto-deploy-dependencies). +1. If you're in the process of deploying `canary` or `rollout` deployments, promote + them to `production` first to delete the unstable tracks. +1. Verify your project is [using the v2 `auto-deploy-image`](#verify-dependency-versions). + If not, [specify the version](#use-a-specific-version-of-auto-deploy-dependencies). +1. Add an `AUTO_DEVOPS_FORCE_DEPLOY_V2` environment variable with a value of `true` + in the GitLab CI/CD settings. +1. Create a new pipeline and run the `production` job to renew the resource architecture + with the v2 `auto-deploy-app chart`. +1. Remove the `AUTO_DEVOPS_FORCE_DEPLOY_V2` environment variable. + +### Use a specific version of Auto Deploy dependencies + +To use a specifc version of Auto Deploy dependencies, specify the previous Auto Deploy +stable template that contains the [desired version of `auto-deploy-image` and `auto-deploy-app`](#verify-dependency-versions). + +For example, if the template is bundled in GitLab v13.3, change your `.gitlab-ci.yml` to: + +```yaml +include: + - template: Auto-DevOps.gitlab-ci.yml + - remote: https://gitlab.com/gitlab-org/gitlab/-/blob/v13.3.0-ee/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +``` + +### Ignore warnings and continue deploying + +If you are certain that the new chart version is safe to be deployed, you can add +the `AUTO_DEVOPS_FORCE_DEPLOY_V<major-version-number>` [environment variable](customize.md#build-and-deployment) +to force the deployment to continue. + +For example, if you want to deploy the `v2.0.0` chart on a deployment that previously +used the `v0.17.0` chart, add `AUTO_DEVOPS_FORCE_DEPLOY_V2`. + +## Early adopters + +If you want to use the latest beta or unstable version of `auto-deploy-image`, include +the latest Auto Deploy template into your `.gitlab-ci.yml`: + +```yaml +include: + - template: Auto-DevOps.gitlab-ci.yml + - remote: https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml +``` + +CAUTION: **Warning:** +Using a beta or unstable `auto-deploy-image` could cause unrecoverable damage to +your environments. Do not test it with important projects or environments. + +The next stable template update is [planned for GitLab v14.0](https://gitlab.com/gitlab-org/gitlab/-/issues/232788). + +## Resource Architectures of the `auto-deploy-app` chart + +### v0 and v1 chart resource architecture + +```mermaid +graph TD; +subgraph gl-managed-app +Z[Nginx Ingress] +end +Z[Nginx Ingress] --> A(Ingress); +Z[Nginx Ingress] --> B(Ingress); +subgraph stg namespace +B[Ingress] --> H(...); +end +subgraph prd namespace +A[Ingress] --> D(Service); +D[Service] --> E(Deployment:Pods:app:stable); +D[Service] --> F(Deployment:Pods:app:canary); +D[Service] --> I(Deployment:Pods:app:rollout); +E(Deployment:Pods:app:stable)---id1[(Pods:Postgres)] +F(Deployment:Pods:app:canary)---id1[(Pods:Postgres)] +I(Deployment:Pods:app:rollout)---id1[(Pods:Postgres)] +end +``` + +### v2 chart resource architecture + +```mermaid +graph TD; +subgraph gl-managed-app +Z[Nginx Ingress] +end +Z[Nginx Ingress] --> A(Ingress); +Z[Nginx Ingress] --> B(Ingress); +Z[Nginx Ingress] --> |If canary is present or incremental rollout/|J(Canary Ingress); +subgraph stg namespace +B[Ingress] --> H(...); +end +subgraph prd namespace +subgraph stable track +A[Ingress] --> D[Service]; +D[Service] --> E(Deployment:Pods:app:stable); +end +subgraph canary track +J(Canary Ingress) --> K[Service] +K[Service] --> F(Deployment:Pods:app:canary); +end +E(Deployment:Pods:app:stable)---id1[(Pods:Postgres)] +F(Deployment:Pods:app:canary)---id1[(Pods:Postgres)] +end +``` + +## Troubleshooting + +### Major version mismatch warning + +If deploying a chart that has a major version that is different from the previous one, +the new chart might not be correctly deployed. This could be due to an architectural +change. If that happens, the deployment job fails with a message similar to: + +```plaintext +************************************************************************************* + [WARNING] +Detected a major version difference between the the chart that is currently deploying (auto-deploy-app-v0.7.0), and the previously deployed chart (auto-deploy-app-v1.0.0). +A new major version might not be backward compatible with the current release (production). The deployment could fail or be stuck in an unrecoverable status. +... +``` + +To clear this error message and resume deployments, you must do one of the following: + +- Manually [upgrade the chart version](#upgrade-guide). +- [Use a specific chart version](#use-a-specific-version-of-auto-deploy-dependencies). + +### Error: `missing key "app.kubernetes.io/managed-by": must be set to "Helm"` + +If your cluster has a deployment that was deployed with the v1 `auto-deploy-image`, +you might encounter the following error: + +- `Error: rendered manifests contain a resource that already exists. Unable to continue with install: Secret "production-postgresql" in namespace "<project-name>-production" exists and cannot be imported into the current release: invalid ownership metadata; label validation error: missing key "app.kubernetes.io/managed-by": must be set to "Helm"; annotation validation error: missing key "meta.helm.sh/release-name": must be set to "production-postgresql"; annotation validation error: missing key "meta.helm.sh/release-namespace": must be set to "<project-name>-production"` + +This is because the previous deployment was deployed with Helm2, which is not compatible with Helm3. +To resolve the problem, please follow the [upgrade guide](#upgrade-deployments-to-the-v2-auto-deploy-image). diff --git a/doc/topics/autodevops/upgrading_chart.md b/doc/topics/autodevops/upgrading_chart.md index ffa485f6d2c..e4fb84d4509 100644 --- a/doc/topics/autodevops/upgrading_chart.md +++ b/doc/topics/autodevops/upgrading_chart.md @@ -1,72 +1,5 @@ -# Upgrading auto-deploy-app chart for Auto DevOps +--- +redirect_to: 'upgrading_auto_deploy_dependencies.md' +--- -Auto DevOps provides the auto-deploy-app chart for deploying your application to the -Kubernetes cluster with Helm/Tiller. Major version changes of this chart could have -a significantly different resource architecture, and may not be backwards compatible. - -This guide provides instructions on how to upgrade your deployments to use the latest -chart and resource architecture. - -## Compatibility - -The following table lists the version compatibility between GitLab and [auto-deploy-image](https://gitlab.com/gitlab-org/cluster-integration/auto-deploy-image) (with the [auto-deploy-app chart](https://gitlab.com/gitlab-org/cluster-integration/auto-deploy-image/-/tree/master/assets/auto-deploy-app)). - -| GitLab version | auto-deploy-image version | Notes | -|------------------|---------------------------|--------------------------------------------| -| v10.0 and higher | v0.1.0 and higher | v0 and v1 charts are backwards compatible. | - -## Upgrade Guide - -The Auto DevOps project must use the unmodified chart managed by GitLab. -[Customized charts](customize.md#custom-helm-chart) are unsupported. - -### v1 chart - -The v1 chart is backward compatible with the v0 chart, so no configuration changes are needed. - -## Troubleshooting - -### Major version mismatch warning - -If deploying a chart that has a major version that is different from the previous one, -the new chart might not be correctly deployed. This could be due to an architectural -change. If that happens, the deployment job fails with a message similar to: - -```plaintext -************************************************************************************* - [WARNING] -Detected a major version difference between the the chart that is currently deploying (auto-deploy-app-v0.7.0), and the previously deployed chart (auto-deploy-app-v1.0.0). -A new major version might not be backward compatible with the current release (production). The deployment could fail or be stuck in an unrecoverable status. -... -``` - -To clear this error message and resume deployments, you must do one of the following: - -- Manually [upgrade the chart version](#upgrade-guide). -- [Use a specific chart version](#use-a-specific-chart-version). - -#### Use a specific chart version - -To use a specific chart version, you must specify a corresponding version of [auto-deploy-image](https://gitlab.com/gitlab-org/cluster-integration/auto-deploy-image). -Do this by [customizing the image in your `.gitlab-ci.yml`](customize.md#customizing-gitlab-ciyml). - -For example, create the following `.gitlab-ci.yml` file in the project. It configures Auto DevOps -to use [auto-deploy-image](https://gitlab.com/gitlab-org/cluster-integration/auto-deploy-image) version `v0.17.0` -for deployment jobs. It will download the chart from [chart repository](https://charts.gitlab.io/): - -```yaml -include: - - template: Auto-DevOps.gitlab-ci.yml - -.auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.17.0" -``` - -#### Ignore warning and continue deploying - -If you are certain that the new chart version is safe to be deployed, -you can add the `AUTO_DEVOPS_FORCE_DEPLOY_V<N>` [environment variable](customize.md#build-and-deployment) -to force the deployment to continue, where `<N>` is the major version. - -For example, if you want to deploy the v2.0.0 chart on a deployment that previously -used the v0.17.0 chart, add `AUTO_DEVOPS_FORCE_DEPLOY_V2`. +This document was moved to [another location](upgrading_auto_deploy_dependencies.md). diff --git a/doc/topics/autodevops/upgrading_postgresql.md b/doc/topics/autodevops/upgrading_postgresql.md index 2ebe362280f..bcbc1d914be 100644 --- a/doc/topics/autodevops/upgrading_postgresql.md +++ b/doc/topics/autodevops/upgrading_postgresql.md @@ -178,8 +178,11 @@ You can also PostgreSQL. 1. Set `AUTO_DEVOPS_POSTGRES_DELETE_V1` to a non-empty value. This flag is a safeguard to prevent accidental deletion of databases. -1. Set `POSTGRES_VERSION` to `11.7`. This is the minimum PostgreSQL - version supported. + <!-- DO NOT REPLACE when upgrading GitLab's supported version. This is NOT related to GitLab's PostgreSQL version support, but the one deployed by Auto DevOps. --> +1. If you have a `POSTGRES_VERSION` set, make sure it is set to `9.6.16` *or +higher*. This is the + minimum PostgreSQL version supported by Auto DevOps. See also the list of + [tags available](https://hub.docker.com/r/bitnami/postgresql/tags). 1. Set `PRODUCTION_REPLICAS` to `0`. For other environments, use `REPLICAS` with an [environment scope](../../ci/environments/index.md#scoping-environments-with-specs). 1. If you have set the `DB_INITIALIZE` or `DB_MIGRATE` variables, either diff --git a/doc/topics/cron/index.md b/doc/topics/cron/index.md index a3dd3b77c22..851dd6d3f77 100644 --- a/doc/topics/cron/index.md +++ b/doc/topics/cron/index.md @@ -24,7 +24,7 @@ Cron scheduling uses a series of five numbers, separated by spaces: # * * * * * <command to execute> ``` -[Source: [Wikipedia](https://en.wikipedia.org/wiki/Cron)] +(Source: [Wikipedia](https://en.wikipedia.org/wiki/Cron)) In cron syntax, the asterisk (`*`) means 'every,' so the following cron strings are valid: diff --git a/doc/topics/git/git_rebase.md b/doc/topics/git/git_rebase.md new file mode 100644 index 00000000000..6f50dea26dd --- /dev/null +++ b/doc/topics/git/git_rebase.md @@ -0,0 +1,272 @@ +--- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: concepts, howto +description: "Introduction to Git rebase, force-push, and resolving merge conflicts through the command line." +--- + +# Introduction to Git rebase, force-push, and merge conflicts + +This guide helps you to get started with rebasing, force-pushing, and fixing +merge conflicts locally. + +Before diving into this document, make sure you are familiar with using +[Git through the command line](../../gitlab-basics/start-using-git.md). + +## Git rebase + +[Rebasing](https://git-scm.com/docs/git-rebase) is a very common operation in +Git. There are the following rebase options: + +- [Regular rebase](#regular-rebase). +- [Interactive rebase](#interactive-rebase). + +### Before rebasing + +CAUTION: **Warning:** +`git rebase` rewrites the commit history. It **can be harmful** to do it in +shared branches. It can cause complex and hard to resolve merge conflicts. In +these cases, instead of rebasing your branch against the default branch, +consider pulling it instead (`git pull origin master`). It has a similar +effect without compromising the work of your contributors. + +It's safer to back up your branch before rebasing to make sure you don't lose +any changes. For example, consider a [feature branch](../../gitlab-basics/start-using-git.md#branching) +called `my-feature-branch`: + +1. Open your feature branch in the terminal: + + ```shell + git checkout my-feature-branch + ``` + +1. Checkout a new branch from it: + + ```shell + git checkout -b my-feature-branch-backup + ``` + +1. Go back to your original branch: + + ```shell + git checkout my-feature-branch + ``` + +Now you can safely rebase it. If anything goes wrong, you can recover your +changes by resetting `my-feature-branch` against `my-feature-branch-backup`: + +1. Make sure you're in the correct branch (`my-feature-branch`): + + ```shell + git checkout my-feature-branch + ``` + +1. Reset it against `my-feature-branch-backup`: + + ```shell + git reset --hard my-feature-branch-backup + ``` + +Note that if you added changes to `my-feature-branch` after creating the backup branch, +you will lose them when resetting. + +### Regular rebase + +With a regular rebase you can update your feature branch with the default +branch (or any other branch). +This is an important step for Git-based development strategies. You can +ensure that the changes you're adding to the codebase do not break any +existing changes added to the target branch _after_ you created your feature +branch. + +For example, to update your branch `my-feature-branch` with `master`: + +1. Fetch the latest changes from `master`: + + ```shell + git fetch origin master + ``` + +1. Checkout your feature branch: + + ```shell + git checkout my-feature-branch + ``` + +1. Rebase it against `master`: + + ```shell + git rebase origin/master + ``` + +1. [Force-push](#force-push) to your branch. + +When you rebase: + +1. Git imports all the commits submitted to `master` _after_ the + moment you created your feature branch until the present moment. +1. Git puts the commits you have in your feature branch on top of all + the commits imported from `master`: + + + +You can replace `master` with any other branch you want to rebase against, for +example, `release-10-3`. You can also replace `origin` with other remote +repositories, for example, `upstream`. To check what remotes you have linked to your local +repository, you can run `git remote -v`. + +If there are [merge conflicts](#merge-conflicts), Git will prompt you to fix +them before continuing the rebase. + +To learn more, check Git's documentation on [rebasing](ttps://git-scm.com/book/en/v2/Git-Branching-Rebasing) +and [rebasing strategies](https://git-scm.com/book/en/v2/Git-Branching-Rebasing). + +### Interactive rebase + +You can use interactive rebase to modify commits. For example, amend a commit +message, squash (join multiple commits into one), edit, or delete +commits. It is handy for changing past commit messages, +as well as for organizing the commit history of your branch to keep it clean. + +TIP: **Tip:** +If you want to keep the default branch commit history clean, you don't need to +manually squash all your commits before merging every merge request; +with [Squash and Merge](../../user/project/merge_requests/squash_and_merge.md) +GitLab does it automatically. + +When you want to change anything in recent commits, use interactive +rebase by passing the flag `--interactive` (or `-i`) to the rebase command. + +For example, if you want to edit the last three commits in your branch +(`HEAD~3`), run: + +```shell +git rebase -i HEAD~3 +``` + +Git opens the last three commits in your terminal text editor and describes +all the interactive rebase options you can use. The default option is `pick`, +which maintains the commit unchanged. Replace the keyword `pick` according to +the operation you want to perform in each commit. To do so, you need to edit +the commits in your terminal's text editor. + +For example, if you're using [Vim](https://www.vim.org/) as the text editor in +a macOS's `ZSH` shell, and you want to **squash** all the three commits +(join them into one): + +1. Press <kbd>i</kbd> on your keyboard to switch to Vim's editing mode. +1. Navigate with your keyboard arrows to edit the **second** commit keyword + from `pick` to `squash` (or `s`). Do the same to the **third** commit. + The first commit should be left **unchanged** (`pick`) as we want to squash + the second and third into the first. +1. Press <kbd>Esc</kbd> to leave the editing mode. +1. Type `:wq` to "write" (save) and "quit". +1. Git outputs the commit message so you have a chance to edit it: + - All lines starting with `#` will be ignored and not included in the commit + message. Everything else will be included. + - To leave it as it is, type `:wq`. To edit the commit message: switch to the + editing mode, edit the commit message, and save it as you just did. +1. If you haven't pushed your commits to the remote branch before rebasing, + push your changes normally. If you had pushed these commits already, + [force-push](#force-push) instead. + +Note that the steps for editing through the command line can be slightly +different depending on your operating system and the shell you're using. + +See [Numerous undo possibilities in Git](numerous_undo_possibilities_in_git/index.md#with-history-modification) +for a deeper look into interactive rebase. + +## Force-push + +When you perform more complex operations, for example, squash commits, reset or +rebase your branch, you'll have to _force_ an update to the remote branch, +since these operations imply rewriting the commit history of the branch. +To force an update, pass the flag `--force` or `-f` to the `push` command. For +example: + +```shell +git push --force origin my-feature-branch +``` + +Forcing an update is **not** recommended when you're working on shared +branches. + +Alternatively, you can pass the flag [`--force-with-lease`](https://git-scm.com/docs/git-push#Documentation/git-push.txt---force-with-leaseltrefnamegt) +instead. It is safer, as it does not overwrite any work on the remote +branch if more commits were added to the remote branch by someone else: + +```shell +git push --force-with-lease origin my-feature-branch +``` + +If the branch you want to force-push is [protected](../../user/project/protected_branches.md), +you can't force-push to it unless you unprotect it first. Then you can +force-push and re-protect it. + +## Merge conflicts + +As Git is based on comparing versions of a file +line-by-line, whenever a line changed in your branch coincides with the same +line changed in the target branch (after the moment you created your feature branch from it), Git +identifies these changes as a merge conflict. To fix it, you need to choose +which version of that line you want to keep. + +Most conflicts can be [resolved through the GitLab UI](../../user/project/merge_requests/resolve_conflicts.md). + +For more complex cases, there are various methods for resolving them. There are +also [Git GUI apps](https://git-scm.com/downloads/guis) that can help by +visualizing the differences. + +To fix conflicts locally, you can use the following method: + +1. Open the terminal and checkout your feature branch, for example, `my-feature-branch`: + + ```shell + git checkout my-feature-branch + ``` + +1. [Rebase](#regular-rebase) your branch against the target branch so Git + prompts you with the conflicts: + + ```shell + git rebase origin/master + ``` + +1. Open the conflicting file in a code editor of your preference. +1. Look for the conflict block: + - It begins with the marker: `<<<<<<< HEAD`. + - Below, there is the content with your changes. + - The marker: `=======` indicates the end of your changes. + - Below, there's the content of the latest changes in the target branch. + - The marker `>>>>>>>` indicates the end of the conflict. +1. Edit the file: choose which version (before or after `=======`) you want to + keep, and then delete the portion of the content you don't want in the file. +1. Delete the markers. +1. Save the file. +1. Repeat the process if there are other conflicting files. +1. Stage your changes: + + ```shell + git add . + ``` + +1. Commit your changes: + + ```shell + git commit -m "Fix merge conflicts" + ``` + +1. Continue rebasing: + + ```shell + git rebase --continue + ``` + + CAUTION: **Caution:** + Up to this point, you can run `git rebase --abort` to stop the process. + Git aborts the rebase and rolls back the branch to the state you had before + running `git rebase`. + Once you run `git rebase --continue` the rebase **cannot** be aborted. + +1. [Force-push](#force-push) to your remote branch. diff --git a/doc/topics/git/img/git_rebase_v13_5.png b/doc/topics/git/img/git_rebase_v13_5.png Binary files differnew file mode 100644 index 00000000000..ff29fa97798 --- /dev/null +++ b/doc/topics/git/img/git_rebase_v13_5.png diff --git a/doc/topics/git/index.md b/doc/topics/git/index.md index 92181fb7bb0..cb2d7b74522 100644 --- a/doc/topics/git/index.md +++ b/doc/topics/git/index.md @@ -81,6 +81,7 @@ If you have problems with Git, the following may help: The following are advanced topics for those who want to get the most out of Git: +- [Introduction to Git rebase, force-push, and merge conflicts](git_rebase.md) - [Server Hooks](../../administration/server_hooks.md) - [Git Attributes](../../user/project/git_attributes.md) - Git Submodules: [Using Git submodules with GitLab CI](../../ci/git_submodules.md#using-git-submodules-with-gitlab-ci) diff --git a/doc/university/README.md b/doc/university/README.md index d029c91a19f..6f063e028b5 100644 --- a/doc/university/README.md +++ b/doc/university/README.md @@ -65,7 +65,6 @@ The GitLab University curriculum is composed of GitLab videos, screencasts, pres 1. [Making GitLab Great for Everyone - Video](https://www.youtube.com/watch?v=GGC40y4vMx0) - Response to "Dear GitHub" letter 1. [Using Innersourcing to Improve Collaboration](https://about.gitlab.com/blog/2014/09/05/innersourcing-using-the-open-source-workflow-to-improve-collaboration-within-an-organization/) 1. [The Software Development Market and GitLab - Video](https://www.youtube.com/watch?v=sXlhgPK1NTY&list=PLFGfElNsQthbQu_IWlNOxul0TbS_2JH-e&index=6) - [Slides](https://docs.google.com/presentation/d/1vCU-NbZWz8NTNK8Vu3y4zGMAHb5DpC8PE5mHtw1PWfI/edit) -1. [The GitLab Book Club](bookclub/index.md) 1. [GitLab Resources](https://about.gitlab.com/resources/) ### 1.7 Community and Support @@ -76,7 +75,6 @@ The GitLab University curriculum is composed of GitLab videos, screencasts, pres - Getting Technical Support - Being part of our Great Community and Contributing to GitLab 1. [Getting Started with the GitLab Development Kit (GDK)](https://about.gitlab.com/blog/2016/06/08/getting-started-with-gitlab-development-kit/) -1. [GitLab Training Workshops](training/end-user/README.md) 1. [GitLab Professional Services](https://about.gitlab.com/services/) ### 1.8 GitLab Training Material @@ -177,10 +175,10 @@ The GitLab University curriculum is composed of GitLab videos, screencasts, pres 1. [High Availability - Video](https://www.youtube.com/watch?v=36KS808u6bE&index=15&list=PLFGfElNsQthbQu_IWlNOxul0TbS_2JH-e) 1. [High Availability Documentation](https://about.gitlab.com/solutions/reference-architectures/) -### 3.8 Cycle Analytics +### 3.8 Value Stream Analytics -1. [GitLab Cycle Analytics Overview](https://about.gitlab.com/blog/2016/09/21/cycle-analytics-feature-highlight/) -1. [GitLab Cycle Analytics - Product Page](https://about.gitlab.com/stages-devops-lifecycle/value-stream-analytics/) +1. [GitLab Value Stream Analytics Overview (as of 2016)](https://about.gitlab.com/blog/2016/09/21/cycle-analytics-feature-highlight/) +1. [GitLab Value Stream Analytics - Product Page](https://about.gitlab.com/stages-devops-lifecycle/value-stream-analytics/) ### 3.9. Integrations diff --git a/doc/university/training/index.md b/doc/university/training/index.md index 69f82392027..4c54108b4fa 100644 --- a/doc/university/training/index.md +++ b/doc/university/training/index.md @@ -17,7 +17,7 @@ This section contains the following topics: - [Cherry pick](topics/cherry_picking.md). - [Code review and collaboration with Merge Requests](topics/merge_requests.md). - [Configure your environment](topics/env_setup.md). -- [Explore GitLab](topics/explore_gitlab.md). +- [Explore GitLab](../../gitlab-basics/README.md). - [Feature branching](topics/feature_branching.md). - [Getting started](topics/getting_started.md). - [GitLab flow](gitlab_flow.md). diff --git a/doc/update/10.0-ce-to-ee.md b/doc/update/10.0-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/10.0-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/10.0-to-10.1.md b/doc/update/10.0-to-10.1.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/10.0-to-10.1.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/10.1-ce-to-ee.md b/doc/update/10.1-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/10.1-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/10.1-to-10.2.md b/doc/update/10.1-to-10.2.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/10.1-to-10.2.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/10.2-ce-to-ee.md b/doc/update/10.2-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/10.2-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/10.2-to-10.3.md b/doc/update/10.2-to-10.3.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/10.2-to-10.3.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/10.3-ce-to-ee.md b/doc/update/10.3-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/10.3-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/10.3-to-10.4.md b/doc/update/10.3-to-10.4.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/10.3-to-10.4.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/10.4-ce-to-ee.md b/doc/update/10.4-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/10.4-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/10.4-to-10.5.md b/doc/update/10.4-to-10.5.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/10.4-to-10.5.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/10.5-ce-to-ee.md b/doc/update/10.5-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/10.5-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/10.5-to-10.6.md b/doc/update/10.5-to-10.6.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/10.5-to-10.6.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/10.6-ce-to-ee.md b/doc/update/10.6-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/10.6-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/10.6-to-10.7.md b/doc/update/10.6-to-10.7.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/10.6-to-10.7.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/10.7-ce-to-ee.md b/doc/update/10.7-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/10.7-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/10.7-to-10.8.md b/doc/update/10.7-to-10.8.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/10.7-to-10.8.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/10.8-ce-to-ee.md b/doc/update/10.8-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/10.8-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/10.8-to-11.0.md b/doc/update/10.8-to-11.0.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/10.8-to-11.0.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/11.0-ce-to-ee.md b/doc/update/11.0-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/11.0-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/11.0-to-11.1.md b/doc/update/11.0-to-11.1.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/11.0-to-11.1.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/11.1-ce-to-ee.md b/doc/update/11.1-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/11.1-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/11.1-to-11.2.md b/doc/update/11.1-to-11.2.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/11.1-to-11.2.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/11.2-ce-to-ee.md b/doc/update/11.2-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/11.2-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/11.2-to-11.3.md b/doc/update/11.2-to-11.3.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/11.2-to-11.3.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/11.3-ce-to-ee.md b/doc/update/11.3-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/11.3-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/11.3-to-11.4.md b/doc/update/11.3-to-11.4.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/11.3-to-11.4.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/11.4-ce-to-ee.md b/doc/update/11.4-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/11.4-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/11.4-to-11.5.md b/doc/update/11.4-to-11.5.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/11.4-to-11.5.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/11.5-ce-to-ee.md b/doc/update/11.5-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/11.5-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/11.5-to-11.6.md b/doc/update/11.5-to-11.6.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/11.5-to-11.6.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/11.6-ce-to-ee.md b/doc/update/11.6-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/11.6-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/11.6-to-11.7.md b/doc/update/11.6-to-11.7.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/11.6-to-11.7.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/11.7-ce-to-ee.md b/doc/update/11.7-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/11.7-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/11.7-to-11.8.md b/doc/update/11.7-to-11.8.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/11.7-to-11.8.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/11.8-ce-to-ee.md b/doc/update/11.8-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/11.8-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/2.6-to-3.0.md b/doc/update/2.6-to-3.0.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/2.6-to-3.0.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/2.9-to-3.0.md b/doc/update/2.9-to-3.0.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/2.9-to-3.0.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/3.0-to-3.1.md b/doc/update/3.0-to-3.1.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/3.0-to-3.1.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/3.1-to-4.0.md b/doc/update/3.1-to-4.0.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/3.1-to-4.0.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/4.0-to-4.1.md b/doc/update/4.0-to-4.1.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/4.0-to-4.1.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/4.1-to-4.2.md b/doc/update/4.1-to-4.2.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/4.1-to-4.2.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/4.2-to-5.0.md b/doc/update/4.2-to-5.0.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/4.2-to-5.0.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/5.0-to-5.1.md b/doc/update/5.0-to-5.1.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/5.0-to-5.1.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/5.1-to-5.2.md b/doc/update/5.1-to-5.2.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/5.1-to-5.2.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/5.1-to-5.4.md b/doc/update/5.1-to-5.4.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/5.1-to-5.4.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/5.1-to-6.0.md b/doc/update/5.1-to-6.0.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/5.1-to-6.0.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/5.2-to-5.3.md b/doc/update/5.2-to-5.3.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/5.2-to-5.3.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/5.3-to-5.4.md b/doc/update/5.3-to-5.4.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/5.3-to-5.4.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/5.4-to-6.0.md b/doc/update/5.4-to-6.0.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/5.4-to-6.0.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/6.0-ce-to-ee.md b/doc/update/6.0-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/6.0-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/6.0-to-6.1.md b/doc/update/6.0-to-6.1.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/6.0-to-6.1.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/6.1-ce-to-ee.md b/doc/update/6.1-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/6.1-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/6.1-to-6.2.md b/doc/update/6.1-to-6.2.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/6.1-to-6.2.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/6.2-ce-to-ee.md b/doc/update/6.2-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/6.2-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/6.2-to-6.3.md b/doc/update/6.2-to-6.3.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/6.2-to-6.3.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/6.3-ce-to-ee.md b/doc/update/6.3-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/6.3-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/6.3-to-6.4.md b/doc/update/6.3-to-6.4.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/6.3-to-6.4.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/6.4-ce-to-ee.md b/doc/update/6.4-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/6.4-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/6.4-to-6.5.md b/doc/update/6.4-to-6.5.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/6.4-to-6.5.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/6.5-ce-to-ee.md b/doc/update/6.5-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/6.5-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/6.5-to-6.6.md b/doc/update/6.5-to-6.6.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/6.5-to-6.6.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/6.6-ce-to-ee.md b/doc/update/6.6-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/6.6-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/6.6-to-6.7.md b/doc/update/6.6-to-6.7.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/6.6-to-6.7.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/6.7-ce-to-ee.md b/doc/update/6.7-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/6.7-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/6.7-to-6.8.md b/doc/update/6.7-to-6.8.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/6.7-to-6.8.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/6.8-ce-to-ee.md b/doc/update/6.8-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/6.8-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/6.8-to-6.9.md b/doc/update/6.8-to-6.9.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/6.8-to-6.9.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/6.9-ce-to-ee.md b/doc/update/6.9-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/6.9-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/6.9-to-7.0.md b/doc/update/6.9-to-7.0.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/6.9-to-7.0.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/6.x-or-7.x-to-7.14.md b/doc/update/6.x-or-7.x-to-7.14.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/6.x-or-7.x-to-7.14.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/7.0-ce-to-ee.md b/doc/update/7.0-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/7.0-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/7.0-to-7.1.md b/doc/update/7.0-to-7.1.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/7.0-to-7.1.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/7.1-ce-to-ee.md b/doc/update/7.1-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/7.1-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/7.1-to-7.2.md b/doc/update/7.1-to-7.2.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/7.1-to-7.2.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/7.10-ce-to-ee.md b/doc/update/7.10-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/7.10-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/7.10-to-7.11.md b/doc/update/7.10-to-7.11.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/7.10-to-7.11.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/7.11-ce-to-ee.md b/doc/update/7.11-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/7.11-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/7.11-to-7.12.md b/doc/update/7.11-to-7.12.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/7.11-to-7.12.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/7.12-ce-to-ee.md b/doc/update/7.12-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/7.12-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/7.12-to-7.13.md b/doc/update/7.12-to-7.13.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/7.12-to-7.13.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/7.13-ce-to-ee.md b/doc/update/7.13-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/7.13-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/7.13-to-7.14.md b/doc/update/7.13-to-7.14.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/7.13-to-7.14.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/7.14-ce-to-ee.md b/doc/update/7.14-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/7.14-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/7.14-to-8.0.md b/doc/update/7.14-to-8.0.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/7.14-to-8.0.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/7.2-to-7.3.md b/doc/update/7.2-to-7.3.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/7.2-to-7.3.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/7.3-ce-to-ee.md b/doc/update/7.3-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/7.3-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/7.3-to-7.4.md b/doc/update/7.3-to-7.4.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/7.3-to-7.4.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/7.4-ce-to-ee.md b/doc/update/7.4-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/7.4-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/7.4-to-7.5.md b/doc/update/7.4-to-7.5.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/7.4-to-7.5.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/7.5-ce-to-ee.md b/doc/update/7.5-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/7.5-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/7.5-to-7.6.md b/doc/update/7.5-to-7.6.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/7.5-to-7.6.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/7.6-ce-to-ee.md b/doc/update/7.6-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/7.6-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/7.6-to-7.7.md b/doc/update/7.6-to-7.7.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/7.6-to-7.7.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/7.7-ce-to-ee.md b/doc/update/7.7-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/7.7-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/7.7-to-7.8.md b/doc/update/7.7-to-7.8.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/7.7-to-7.8.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/7.8-ce-to-ee.md b/doc/update/7.8-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/7.8-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/7.8-to-7.9.md b/doc/update/7.8-to-7.9.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/7.8-to-7.9.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/7.9-ce-to-ee.md b/doc/update/7.9-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/7.9-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/7.9-to-7.10.md b/doc/update/7.9-to-7.10.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/7.9-to-7.10.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/8.0-ce-to-ee.md b/doc/update/8.0-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/8.0-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/8.0-to-8.1.md b/doc/update/8.0-to-8.1.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/8.0-to-8.1.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/8.1-ce-to-ee.md b/doc/update/8.1-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/8.1-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/8.1-to-8.2.md b/doc/update/8.1-to-8.2.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/8.1-to-8.2.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/8.10-ce-to-ee.md b/doc/update/8.10-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/8.10-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/8.10-to-8.11.md b/doc/update/8.10-to-8.11.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/8.10-to-8.11.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/8.11-ce-to-ee.md b/doc/update/8.11-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/8.11-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/8.11-to-8.12.md b/doc/update/8.11-to-8.12.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/8.11-to-8.12.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/8.12-ce-to-ee.md b/doc/update/8.12-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/8.12-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/8.12-to-8.13.md b/doc/update/8.12-to-8.13.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/8.12-to-8.13.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/8.13-ce-to-ee.md b/doc/update/8.13-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/8.13-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/8.13-to-8.14.md b/doc/update/8.13-to-8.14.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/8.13-to-8.14.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/8.14-ce-to-ee.md b/doc/update/8.14-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/8.14-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/8.14-to-8.15.md b/doc/update/8.14-to-8.15.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/8.14-to-8.15.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/8.15-ce-to-ee.md b/doc/update/8.15-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/8.15-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/8.15-to-8.16.md b/doc/update/8.15-to-8.16.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/8.15-to-8.16.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/8.16-ce-to-ee.md b/doc/update/8.16-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/8.16-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/8.16-to-8.17.md b/doc/update/8.16-to-8.17.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/8.16-to-8.17.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/8.17-ce-to-ee.md b/doc/update/8.17-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/8.17-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/8.17-to-9.0.md b/doc/update/8.17-to-9.0.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/8.17-to-9.0.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/8.2-ce-to-ee.md b/doc/update/8.2-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/8.2-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/8.2-to-8.3.md b/doc/update/8.2-to-8.3.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/8.2-to-8.3.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/8.3-ce-to-ee.md b/doc/update/8.3-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/8.3-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/8.3-to-8.4.md b/doc/update/8.3-to-8.4.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/8.3-to-8.4.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/8.4-ce-to-ee.md b/doc/update/8.4-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/8.4-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/8.4-to-8.5.md b/doc/update/8.4-to-8.5.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/8.4-to-8.5.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/8.5-ce-to-ee.md b/doc/update/8.5-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/8.5-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/8.5-to-8.6.md b/doc/update/8.5-to-8.6.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/8.5-to-8.6.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/8.6-ce-to-ee.md b/doc/update/8.6-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/8.6-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/8.6-to-8.7.md b/doc/update/8.6-to-8.7.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/8.6-to-8.7.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/8.7-ce-to-ee.md b/doc/update/8.7-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/8.7-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/8.7-to-8.8.md b/doc/update/8.7-to-8.8.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/8.7-to-8.8.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/8.8-ce-to-ee.md b/doc/update/8.8-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/8.8-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/8.8-to-8.9.md b/doc/update/8.8-to-8.9.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/8.8-to-8.9.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/8.9-ce-to-ee.md b/doc/update/8.9-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/8.9-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/8.9-to-8.10.md b/doc/update/8.9-to-8.10.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/8.9-to-8.10.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/9.0-ce-to-ee.md b/doc/update/9.0-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/9.0-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/9.0-to-9.1.md b/doc/update/9.0-to-9.1.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/9.0-to-9.1.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/9.1-ce-to-ee.md b/doc/update/9.1-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/9.1-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/9.1-to-9.2.md b/doc/update/9.1-to-9.2.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/9.1-to-9.2.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/9.2-ce-to-ee.md b/doc/update/9.2-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/9.2-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/9.2-to-9.3.md b/doc/update/9.2-to-9.3.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/9.2-to-9.3.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/9.3-ce-to-ee.md b/doc/update/9.3-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/9.3-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/9.3-to-9.4.md b/doc/update/9.3-to-9.4.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/9.3-to-9.4.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/9.4-ce-to-ee.md b/doc/update/9.4-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/9.4-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/9.4-to-9.5.md b/doc/update/9.4-to-9.5.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/9.4-to-9.5.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/9.5-ce-to-ee.md b/doc/update/9.5-ce-to-ee.md deleted file mode 100644 index 10c9e21fa81..00000000000 --- a/doc/update/9.5-ce-to-ee.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_ce_to_ee.md ---- - -This document was moved to [another location](upgrading_from_ce_to_ee.md). diff --git a/doc/update/9.5-to-10.0.md b/doc/update/9.5-to-10.0.md deleted file mode 100644 index 8514aa13f48..00000000000 --- a/doc/update/9.5-to-10.0.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: upgrading_from_source.md ---- - -This document was moved to [another location](upgrading_from_source.md). diff --git a/doc/update/README.md b/doc/update/README.md index a7f7aaf5887..b5e99671278 100644 --- a/doc/update/README.md +++ b/doc/update/README.md @@ -123,7 +123,7 @@ If using GitLab 12.9 and newer, run: sudo gitlab-rails runner -e production 'puts Gitlab::BackgroundMigration.remaining' ``` -If using GitLab 12.8 and older, run the following using a [Rails console](../administration/troubleshooting/debug.md#starting-a-rails-console-session): +If using GitLab 12.8 and older, run the following using a [Rails console](../administration/operations/rails_console.md#starting-a-rails-console-session): ```ruby puts Sidekiq::Queue.new("background_migration").size @@ -141,7 +141,7 @@ cd /home/git/gitlab sudo -u git -H bundle exec rails runner -e production 'puts Gitlab::BackgroundMigration.remaining' ``` -If using GitLab 12.8 and older, run the following using a [Rails console](../administration/troubleshooting/debug.md#starting-a-rails-console-session): +If using GitLab 12.8 and older, run the following using a [Rails console](../administration/operations/rails_console.md#starting-a-rails-console-session): ```ruby puts Sidekiq::Queue.new("background_migration").size diff --git a/doc/user/admin_area/abuse_reports.md b/doc/user/admin_area/abuse_reports.md index 0283f1a9eff..4c346d7dfe8 100644 --- a/doc/user/admin_area/abuse_reports.md +++ b/doc/user/admin_area/abuse_reports.md @@ -20,6 +20,9 @@ To receive notifications of new abuse reports by e-mail, follow these steps: 1. Expand the **Abuse reports** section. 1. Provide an email address. +The notification email address can also be set and retrieved +[using the API](../../api/settings.md#list-of-settings-that-can-be-accessed-via-api-calls). + ## Reporting abuse To find out more about reporting abuse, see [abuse reports user @@ -31,14 +34,14 @@ To access abuse reports, go to **Admin Area > Abuse Reports**. There are 3 ways to resolve an abuse report, with a button for each method: -- Remove user & report. This will: - - [Delete the reported user](../profile/account/delete_account.md) from the +- Remove user & report. This: + - [Deletes the reported user](../profile/account/delete_account.md) from the instance. - - Remove the abuse report from the list. + - Removes the abuse report from the list. - [Block user](#blocking-users). -- Remove report. This will: - - Remove the abuse report from the list. - - Remove access restrictions for the reported user. +- Remove report. This: + - Removes the abuse report from the list. + - Removes access restrictions for the reported user. The following is an example of the **Abuse Reports** page: @@ -54,8 +57,7 @@ Blocking a user: - Leaves them in the abuse report list. - Changes the **Block user** button to a disabled **Already blocked** button. -The user will be notified with the -[following message](https://gitlab.com/gitlab-org/gitlab/blob/master/app/workers/email_receiver_worker.rb#L38): +The user is notified with the following message: ```plaintext Your account has been blocked. If you believe this is in error, contact a staff member. diff --git a/doc/user/admin_area/activating_deactivating_users.md b/doc/user/admin_area/activating_deactivating_users.md index 29f162616bf..8b3a7682841 100644 --- a/doc/user/admin_area/activating_deactivating_users.md +++ b/doc/user/admin_area/activating_deactivating_users.md @@ -39,7 +39,7 @@ A user can be deactivated from the Admin Area. To do this: Please note that for the deactivation option to be visible to an admin, the user: - Must be currently active. -- Must not have signed in, or have any activity, in the last 180 days. +- Must not have signed in, or have any activity, in the last 90 days. Users can also be deactivated using the [GitLab API](../../api/users.md#deactivate-user). diff --git a/doc/user/admin_area/analytics/img/cohorts_v13_4.png b/doc/user/admin_area/analytics/img/cohorts_v13_4.png Binary files differindex 4af1841a033..6f7dd5101f2 100644 --- a/doc/user/admin_area/analytics/img/cohorts_v13_4.png +++ b/doc/user/admin_area/analytics/img/cohorts_v13_4.png diff --git a/doc/user/admin_area/analytics/img/dev_ops_report_v13_4.png b/doc/user/admin_area/analytics/img/dev_ops_report_v13_4.png Binary files differindex 1fa070a6915..d47d86cd514 100644 --- a/doc/user/admin_area/analytics/img/dev_ops_report_v13_4.png +++ b/doc/user/admin_area/analytics/img/dev_ops_report_v13_4.png diff --git a/doc/user/admin_area/analytics/index.md b/doc/user/admin_area/analytics/index.md index b3336b471f8..f79245c7325 100644 --- a/doc/user/admin_area/analytics/index.md +++ b/doc/user/admin_area/analytics/index.md @@ -4,7 +4,8 @@ Administrators have access to instance-wide analytics, as shown in **Admin Area > Analytics**. -There are two kinds of statistics: +There are several kinds of statistics: - [DevOps Report](dev_ops_report.md): Provides an overview of your entire instance's feature usage. +- [Instance Statistics](instance_statistics.md): Shows how much data your instance contains, and how that is changing. - [User Cohorts](user_cohorts.md): Display the monthly cohorts of new users and their activities over time. diff --git a/doc/user/admin_area/analytics/instance_statistics.md b/doc/user/admin_area/analytics/instance_statistics.md new file mode 100644 index 00000000000..bac0e845d2c --- /dev/null +++ b/doc/user/admin_area/analytics/instance_statistics.md @@ -0,0 +1,18 @@ +# Instance Statistics + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/235754) in GitLab 13.4. + +Instance Statistics gives you an overview of how much data your instance contains, and how quickly this volume is changing over time. + +## Total counts + +At the top of the page, Instance Statistics shows total counts for: + +- Users +- Projects +- Groups +- Issues +- Merge Requests +- Pipelines + +These figures can be useful for understanding how much data your instance contains in total. diff --git a/doc/user/admin_area/approving_users.md b/doc/user/admin_area/approving_users.md new file mode 100644 index 00000000000..486d0b6a25d --- /dev/null +++ b/doc/user/admin_area/approving_users.md @@ -0,0 +1,36 @@ +--- +stage: Manage +group: Access +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: howto +--- + +# Users pending approval + +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/4491) in GitLab 13.5. + +When [Require admin approval for new sign-ups](settings/sign_up_restrictions.md#require-admin-approval-for-new-sign-ups) is enabled, any user that signs up for an account using the registration form is placed under a **Pending approval** state. + +A user pending approval is functionally identical to a [blocked](blocking_unblocking_users.md) user. + +A user pending approval: + +- Will not be able to sign in. +- Cannot access Git repositories or the API. +- Will not receive any notifications from GitLab. +- Does not consume a [seat](../../subscriptions/self_managed/index.md#choose-the-number-of-users). + +## Approving a user + +A user that is pending approval can be approved from the Admin Area. To do this: + +1. Navigate to **Admin Area > Overview > Users**. +1. Click on the **Pending approval** tab. +1. Select a user. +1. Under the **Account** tab, click **Approve user**. + +Approving a user: + +1. Activates their account. +1. Changes the user's state to active and it consumes a +[seat](../../subscriptions/self_managed/index.md#choose-the-number-of-users). diff --git a/doc/user/admin_area/credentials_inventory.md b/doc/user/admin_area/credentials_inventory.md index 7f2d49dafea..b17d0ab3dd5 100644 --- a/doc/user/admin_area/credentials_inventory.md +++ b/doc/user/admin_area/credentials_inventory.md @@ -9,11 +9,9 @@ type: howto > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/20912) in GitLab 12.6. -## Overview - GitLab administrators are responsible for the overall security of their instance. To assist, GitLab provides a Credentials inventory to keep track of all the credentials that can be used to access their self-managed instance. -Using Credentials inventory, you can see all the personal access tokens (PAT) and SSH keys that exist in your GitLab instance. In addition, you can [revoke them](#revoke-a-users-personal-access-token) and see: +Using Credentials inventory, you can see all the personal access tokens (PAT) and SSH keys that exist in your GitLab instance. In addition, you can [revoke](#revoke-a-users-personal-access-token) and [delete](#delete-a-users-ssh-key) and see: - Who they belong to. - Their access scope. @@ -29,7 +27,7 @@ The following is an example of the Credentials inventory page: ## Revoke a user's personal access token -[Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/214811) in GitLab 13.4. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/214811) in GitLab 13.4. If you see a **Revoke** button, you can revoke that user's PAT. Whether you see a **Revoke** button depends on the token state, and if an expiration date has been set. For more information, see the following table: @@ -37,7 +35,15 @@ If you see a **Revoke** button, you can revoke that user's PAT. Whether you see |-------------|------------------------|--------------------|----------------------------------------------------------------------------| | Active | Yes | Yes | Allows administrators to revoke the PAT, such as for a compromised account | | Active | No | Yes | Allows administrators to revoke the PAT, such as for a compromised account | -| Expired | Yes | No | PAT expires automatically | -| Expired | No | Yes | The administrator may revoke the PAT to prevent indefinite use | -| Revoked | Yes | No | Not applicable; token is already revoked | -| Revoked | No | No | Not applicable; token is already revoked | +| Expired | Yes | No | PAT expires automatically | +| Expired | No | Yes | The administrator may revoke the PAT to prevent indefinite use | +| Revoked | Yes | No | Not applicable; token is already revoked | +| Revoked | No | No | Not applicable; token is already revoked | + +## Delete a user's SSH key + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/225248) in GitLab 13.5. + +You can **Delete** a user's SSH key by navigating to the credentials inventory's SSH Keys tab. + + diff --git a/doc/user/admin_area/img/admin_wrench.png b/doc/user/admin_area/img/admin_wrench.png Binary files differdeleted file mode 100644 index 17eee143e87..00000000000 --- a/doc/user/admin_area/img/admin_wrench.png +++ /dev/null diff --git a/doc/user/admin_area/img/credentials_inventory_ssh_keys_v13_5.png b/doc/user/admin_area/img/credentials_inventory_ssh_keys_v13_5.png Binary files differnew file mode 100644 index 00000000000..f5edf513bbf --- /dev/null +++ b/doc/user/admin_area/img/credentials_inventory_ssh_keys_v13_5.png diff --git a/doc/user/admin_area/index.md b/doc/user/admin_area/index.md index 58430ab615b..0ddbe17580a 100644 --- a/doc/user/admin_area/index.md +++ b/doc/user/admin_area/index.md @@ -118,8 +118,15 @@ To list users matching a specific criteria, click on one of the following tabs o - **[Deactivated](activating_deactivating_users.md)** - **Without projects** -For each user, their username, email address, are listed, also the date their account was -created and the date of last activity. To edit a user, click the **Edit** button in that user's +For each user, the following are listed: + +1. Username +1. Email address +1. Project membership count +1. Date of account creation +1. Date of last activity + +To edit a user, click the **Edit** button in that user's row. To delete the user, or delete the user and their contributions, click the cog dropdown in that user's row, and select the desired option. diff --git a/doc/user/admin_area/license.md b/doc/user/admin_area/license.md index ecbc615f56a..c37a61d6748 100644 --- a/doc/user/admin_area/license.md +++ b/doc/user/admin_area/license.md @@ -8,130 +8,126 @@ type: howto # Activate GitLab EE with a license **(STARTER ONLY)** To activate all GitLab Enterprise Edition (EE) functionality, you need to upload -a license. Once you've received your license from GitLab Inc., you can upload it -by **signing into your GitLab instance as an admin** or add it at +a license. After you've received your license from GitLab Inc., you can upload it +by **signing into your GitLab instance as an admin** or adding it at installation time. -The license has the form of a base64 encoded ASCII text with a `.gitlab-license` -extension and can be obtained when you [purchase one](https://about.gitlab.com/pricing/) or when you sign -up for a [free trial](https://about.gitlab.com/free-trial/). +The license is a base64-encoded ASCII text file with a `.gitlab-license` +extension. You can obtain the file by [purchasing a license](https://about.gitlab.com/pricing/) +or by signing up for a [free trial](https://about.gitlab.com/free-trial/). -NOTE: **Note:** As of GitLab Enterprise Edition 9.4.0, a newly-installed instance without an -uploaded license will only have the Core features active. A trial license will -activate all Ultimate features, but after +uploaded license only has the Core features active. A trial license +activates all Ultimate features, but after [the trial expires](#what-happens-when-your-license-expires), some functionality -will be locked. +is locked. ## Uploading your license The very first time you visit your GitLab EE installation signed in as an admin, you should see a note urging you to upload a license with a link that takes you -straight to **Admin Area > License**. +to **Admin Area > License**. Otherwise, you can: -1. Navigate manually to the **Admin Area** by clicking the wrench icon in the menu bar. +1. Navigate manually to the **Admin Area** by clicking the wrench (**{admin}**) icon in the menu bar. -  - -1. And then going to the **License** tab and click on **Upload New License**. +1. Navigate to the **License** tab, and click **Upload New License**.  -1. If you've received a `.gitlab-license` file, you should have already downloaded - it in your local machine. You can then upload it directly by choosing the - license file and clicking the **Upload license** button. In the image below, - you can see that the selected license file is named `GitLab.gitlab-license`. + - *If you've received a `.gitlab-license` file,* you should have already downloaded + it in your local machine. You can then upload it directly by choosing the + license file and clicking the **Upload license** button. In the image below, + the selected license file is named `GitLab.gitlab-license`. -  +  - If you've received your license as plain text, you need to select the - "Enter license key" option, copy the license, paste it into the "License key" - field and click **Upload license**. + - *If you've received your license as plain text,* select the + **Enter license key** option, copy the license, paste it into the **License key** + field, and click **Upload license**. ## Add your license at install time -A license can be automatically imported at install time, by placing a file named -`Gitlab.gitlab-license` in `/etc/gitlab/` for Omnibus, or `config/` for source installations. +A license can be automatically imported at install time by placing a file named +`Gitlab.gitlab-license` in `/etc/gitlab/` for Omnibus GitLab, or `config/` for source installations. -It is also possible to specify a custom location and filename for the license. +You can also specify a custom location and filename for the license: -Source installations should set the `GITLAB_LICENSE_FILE` environment -variable with the path to a valid GitLab Enterprise Edition license. +- Source installations should set the `GITLAB_LICENSE_FILE` environment + variable with the path to a valid GitLab Enterprise Edition license. -```shell -export GITLAB_LICENSE_FILE="/path/to/license/file" -``` + ```shell + export GITLAB_LICENSE_FILE="/path/to/license/file" + ``` -Omnibus installations should add this entry to `gitlab.rb`: +- Omnibus GitLab installations should add this entry to `gitlab.rb`: -```ruby -gitlab_rails['initial_license_file'] = "/path/to/license/file" -``` + ```ruby + gitlab_rails['initial_license_file'] = "/path/to/license/file" + ``` CAUTION: **Caution:** -These methods will only add a license at the time of installation. Use the -Admin Area in the web user interface to renew or upgrade licenses. +These methods only add a license at the time of installation. Use the +**{admin}** **Admin Area** in the web user interface to renew or upgrade licenses. --- -Once the license is uploaded, all GitLab Enterprise Edition functionality -will be active until the end of the license period. When that period ends, the +After the license is uploaded, all GitLab Enterprise Edition functionality +is active until the end of the license period. When that period ends, the instance will [fall back](#what-happens-when-your-license-expires) to Core-only functionality. -You can review the license details at any time in the License section of the -Admin Area. +You can review the license details at any time in the **License** section of the +**Admin Area**.  ## Notification before the license expires -One month before the license expires, a message informing when the expiration -is due to, will start appearing to GitLab admins. Make sure that you update your -license, otherwise you will miss all the paid features if it expires. +One month before the license expires, a message informing about the expiration +date is displayed to GitLab admins. Make sure that you update your +license, otherwise you miss all the paid features if your license expires. ## What happens when your license expires -In case your license expires, GitLab will lock down some features like Git pushes, -issue creation, etc., and a message to inform of the expired license will be -presented to all admins. +In case your license expires, GitLab locks down some features like Git pushes, +and issue creation, and displays a message to all admins to inform of the expired license. -In order to get back all the previous functionality, a new license must be uploaded. -To fall back to having only the Core features active, you'll need to delete the +To get back all the previous functionality, you must upload a new license. +To fall back to having only the Core features active, you must delete the expired license(s). ### Remove a license To remove a license from a self-managed instance: -1. Go to the [Admin Area](index.md) (click the wrench in the top navigation bar). +1. In the top navigation bar, click the **{admin}** wrench icon to navigate to the [Admin Area](index.md). 1. Click **License** in the left sidebar. 1. Click **Remove License**. ## License history -It's possible to upload and view more than one license, -but only the latest license will be used as the active license. +You can upload and view more than one license, +but only the latest license is used as the active license. ## Troubleshooting ### There is no License tab in the Admin Area -If you originally installed Community Edition rather than Enterprise Edition you will need to +If you originally installed Community Edition rather than Enterprise Edition you must [upgrade to Enterprise Edition](../../update/README.md#community-to-enterprise-edition) before uploading your license. -GitLab.com users cannot upload and use a self-managed license. If you -wish to use paid features on GitLab.com, a separate subscription may be -[purchased](../../subscriptions/gitlab_com/index.md). +GitLab.com users can't upload and use a self-managed license. If you +want to use paid features on GitLab.com, you can +[purchase a separate subscription](../../subscriptions/gitlab_com/index.md). ### Users exceed license limit upon renewal -If you've added new users to your GitLab instance prior to renewal you may need to -purchase additional seats to cover those users. If this is the case and a license -without enough users is uploaded a message is displayed prompting you to purchase +If you've added new users to your GitLab instance prior to renewal, you may need to +purchase additional seats to cover those users. If this is the case, and a license +without enough users is uploaded, GitLab displays a message prompting you to purchase additional users. More information on how to determine the required number of users and how to add additional seats can be found in the [licensing FAQ](https://about.gitlab.com/pricing/licensing-faq/). diff --git a/doc/user/admin_area/settings/gitaly_timeouts.md b/doc/user/admin_area/settings/gitaly_timeouts.md index 2003d02c9b3..cac05678a1a 100644 --- a/doc/user/admin_area/settings/gitaly_timeouts.md +++ b/doc/user/admin_area/settings/gitaly_timeouts.md @@ -3,36 +3,28 @@ stage: Create group: Gitaly info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" type: reference -type: reference --- -# Gitaly timeouts - - - -3 timeout types can be configured to make sure that long running -Gitaly calls don't needlessly take up resources. - -- Default timeout - -This timeout is the default for most Gitaly calls. -It should be shorter than the worker timeout that can be configured -for -[Puma](https://docs.gitlab.com/omnibus/settings/puma.html#puma-settings) -or [Unicorn](https://docs.gitlab.com/omnibus/settings/unicorn.html). -This makes sure that Gitaly calls made within a web request cannot -exceed these the entire request timeout. +# Gitaly timeouts **(CORE ONLY)** -The default for this timeout is 55 seconds. +[Gitaly](../../../administration/gitaly/index.md) timeouts are configurable. The timeouts can be +configured to make sure that long running Gitaly calls don't needlessly take up resources. -- Fast timeout +To access Gitaly timeout settings: -This is the timeout for very short Gitaly calls. +1. Go to **Admin Area > Settings > Preferences**. +1. Expand the **Gitaly** section. -The default for this timeout is 10 seconds. +## Available timeouts -- Medium timeout +The following timeouts can be modified: -This timeout should be between the default and the fast timeout +- **Default Timeout Period**. This timeout is the default for most Gitaly calls. It should be shorter than the + worker timeout that can be configured for [Puma](https://docs.gitlab.com/omnibus/settings/puma.html#puma-settings) + or [Unicorn](https://docs.gitlab.com/omnibus/settings/unicorn.html). Used to make sure that Gitaly + calls made within a web request cannot exceed the entire request timeout. + Defaults to 55 seconds. -The default for this timeout is 30 seconds. +- **Fast Timeout Period**. This is the timeout for very short Gitaly calls. Defaults to 10 seconds. +- **Medium Timeout Period**. This timeout should be between the default and the fast timeout. + Defaults to 30 seconds. diff --git a/doc/user/admin_area/settings/img/email_confirmation_v12_7.png b/doc/user/admin_area/settings/img/email_confirmation_v12_7.png Binary files differdeleted file mode 100644 index 6bcadb63b9a..00000000000 --- a/doc/user/admin_area/settings/img/email_confirmation_v12_7.png +++ /dev/null diff --git a/doc/user/admin_area/settings/img/gitaly_timeouts.png b/doc/user/admin_area/settings/img/gitaly_timeouts.png Binary files differdeleted file mode 100644 index 28394d238f7..00000000000 --- a/doc/user/admin_area/settings/img/gitaly_timeouts.png +++ /dev/null diff --git a/doc/user/admin_area/settings/img/sign_up_restrictions_v13_5.png b/doc/user/admin_area/settings/img/sign_up_restrictions_v13_5.png Binary files differnew file mode 100644 index 00000000000..ebbfad77e69 --- /dev/null +++ b/doc/user/admin_area/settings/img/sign_up_restrictions_v13_5.png diff --git a/doc/user/admin_area/settings/index.md b/doc/user/admin_area/settings/index.md index ba9bccbf3e7..bc8df63e33f 100644 --- a/doc/user/admin_area/settings/index.md +++ b/doc/user/admin_area/settings/index.md @@ -30,11 +30,11 @@ Access the default page for admin area settings by navigating to **Admin Area > | Option | Description | | ------ | ----------- | -| [Elasticsearch](../../../integration/elasticsearch.md#enabling-elasticsearch) | Elasticsearch integration. Elasticsearch AWS IAM. | +| [Elasticsearch](../../../integration/elasticsearch.md#enabling-advanced-search) | Elasticsearch integration. Elasticsearch AWS IAM. | | [PlantUML](../../../administration/integration/plantuml.md#gitlab) | Allow rendering of PlantUML diagrams in AsciiDoc documents. | | [Slack application](../../../user/project/integrations/gitlab_slack_application.md#configuration) **(FREE ONLY)** | Slack integration allows you to interact with GitLab via slash commands in a chat window. This option is only available on GitLab.com, though it may be [available for self-managed instances in the future](https://gitlab.com/gitlab-org/gitlab/-/issues/28164). | | [Third party offers](third_party_offers.md) | Control the display of third party offers. | -| [Snowplow](../../../development/telemetry/snowplow.md) | Configure the Snowplow integration. | +| [Snowplow](../../../development/product_analytics/snowplow.md) | Configure the Snowplow integration. | | [Google GKE](../../project/clusters/add_gke_clusters.md) | Google GKE integration allows you to provision GKE clusters from GitLab. | | [Amazon EKS](../../project/clusters/add_eks_clusters.md) | Amazon EKS integration allows you to provision EKS clusters from GitLab. | @@ -61,7 +61,7 @@ Access the default page for admin area settings by navigating to **Admin Area > | ------ | ----------- | | [Continuous Integration and Deployment](continuous_integration.md) | Auto DevOps, runners and job artifacts. | | [Required pipeline configuration](continuous_integration.md#required-pipeline-configuration) **(PREMIUM ONLY)** | Set an instance-wide auto included [pipeline configuration](../../../ci/yaml/README.md). This pipeline configuration will be run after the project's own configuration. | -| [Package Registry](continuous_integration.md#package-registry-configuration) | Settings related to the use and experience of using GitLab's Package Registry. Note there are [risks involved](./../../packages/container_registry/index.md#use-with-external-container-registries) in enabling some of these settings. | +| [Package Registry](continuous_integration.md#package-registry-configuration) | Settings related to the use and experience of using GitLab's Package Registry. Note there are [risks involved](../../packages/container_registry/index.md#use-with-external-container-registries) in enabling some of these settings. | ## Reporting @@ -102,7 +102,7 @@ Access the default page for admin area settings by navigating to **Admin Area > | Option | Description | | ------ | ----------- | | [Email](email.md) | Various email settings. | -| [Help page](../../../customization/help_message.md) | Help page text and support page URL. | +| [Help page](help_page.md) | Help page text and support page URL. | | [Pages](../../../administration/pages/index.md#custom-domain-verification) | Size and domain settings for static websites | | [Real-time features](../../../administration/polling.md) | Change this value to influence how frequently the GitLab UI polls for updates. | | [Gitaly timeouts](gitaly_timeouts.md) | Configure Gitaly timeouts. | diff --git a/doc/user/admin_area/settings/instance_template_repository.md b/doc/user/admin_area/settings/instance_template_repository.md index 97380b93295..20f2812bc39 100644 --- a/doc/user/admin_area/settings/instance_template_repository.md +++ b/doc/user/admin_area/settings/instance_template_repository.md @@ -9,8 +9,6 @@ type: reference > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/5986) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.3. -## Overview - In hosted systems, enterprises often have a need to share their own templates across teams. This feature allows an administrator to pick a project to be the instance-wide collection of file templates. These templates are then exposed to diff --git a/doc/user/admin_area/settings/project_integration_management.md b/doc/user/admin_area/settings/project_integration_management.md index e4fe7e36139..748d608676d 100644 --- a/doc/user/admin_area/settings/project_integration_management.md +++ b/doc/user/admin_area/settings/project_integration_management.md @@ -4,38 +4,53 @@ group: Ecosystem info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# Project integration management **(CORE ONLY)** +# Project integration management -> [Introduced in](https://gitlab.com/groups/gitlab-org/-/epics/2137) GitLab 13.3. +Project integrations can be configured and enabled by project administrators. As a GitLab instance +administrator, you can set default configuration parameters for a given integration that all projects +can inherit and use. This enables the integration for all projects that are not already using custom +settings. -Project integrations can be configured and enabled by project administrators. As a GitLab instance administrator, you can set default configuration parameters for a given integration that all projects can inherit and use, enabling the integration for all projects that are not already using custom settings. +You can update these default settings at any time, changing the settings used for all projects that +are set to use instance-level defaults. Updating the default settings also enables the integration +for all projects that didn't have it already enabled. -You can update these default settings at any time, changing the settings in use for all projects that are set to use instance-level defaults. This also enables the integration for all projects on which it was not already enabled. +Only the complete settings for an integration can be inherited. Per-field inheritance is +[planned](https://gitlab.com/groups/gitlab-org/-/epics/2137) as is +[group-level management](https://gitlab.com/groups/gitlab-org/-/epics/2543) of integration settings. -It is only possible to inherit the complete settings for an integration. Per-field inheritance is [planned](https://gitlab.com/groups/gitlab-org/-/epics/2137), as well as [group-level management](https://gitlab.com/groups/gitlab-org/-/epics/2543) of integration settings. +## Manage instance-level default settings for a project integration **(CORE ONLY)** -## Manage default settings for a project integration +> [Introduced in](https://gitlab.com/groups/gitlab-org/-/epics/2137) GitLab 13.3. 1. Navigate to **Admin Area > Settings > Integrations**. 1. Select a project integration. 1. Enter configuration details and click **Save changes**. CAUTION: **Caution:** -This may affect all or most of the projects on your GitLab instance. Please review the details below. +This may affect all or most of the projects on your GitLab instance. Please review the details +below. If this is the first time you are setting up instance-level settings for an integration: -- The integration is enabled for all projects that do not already have this integration configured if you have the **Enable integration** toggle turned on in the instance-level settings. -- Projects that already have the integration configured are not affected, but can choose to use the inherited settings at any time. +- The integration is enabled for all projects that don't already have this integration configured, + if you have the **Enable integration** toggle turned on in the instance-level settings. +- Projects that already have the integration configured are not affected, but can choose to use the + inherited settings at any time. When you make further changes to the instance defaults: -- They are immediately applied to all projects that have the integration set to use instance-level default settings. -- They are immediately applied to newer projects, created since you last saved defaults for the integration. - - If your instance-level default setting has the **Enable integration** toggle turned on, the integration is automatically enabled for all such projects. -- Projects with custom settings selected for the integration are not immediately affected and may choose to use the latest instance-level defaults at any time. +- They are immediately applied to all projects that have the integration set to use default settings. +- They are immediately applied to newer projects, created since you last saved defaults for the + integration. If your instance-level default setting has the **Enable integration** toggle turned + on, the integration is automatically enabled for all such projects. +- Projects with custom settings selected for the integration are not immediately affected and may + choose to use the latest defaults at any time. -It is only possible to inherit the complete settings for an integration. Per-field inheritance is [planned](https://gitlab.com/groups/gitlab-org/-/epics/2137). This would allow instance administrators to update settings inherited by projects without enabling the integration on all non-configured projects by default. +Only the complete settings for an integration can be inherited. Per-field inheritance +is [planned](https://gitlab.com/groups/gitlab-org/-/epics/2137). This would allow +administrators to update settings inherited by projects without enabling the +integration on all non-configured projects by default. ## Use instance-level default settings for a project integration @@ -47,7 +62,7 @@ It is only possible to inherit the complete settings for an integration. Per-fie ## Use custom settings for a project integration -1. Navigate to **Project > Settings > Integrations**. +1. Navigate to project's **Settings > Integrations**. 1. Choose the integration you want to enable or update. 1. From the drop-down, select **Use custom settings**. 1. Ensure the toggle is set to **Enable integration** and enter all required settings. diff --git a/doc/user/admin_area/settings/sign_up_restrictions.md b/doc/user/admin_area/settings/sign_up_restrictions.md index 80092102091..f57cf7c2045 100644 --- a/doc/user/admin_area/settings/sign_up_restrictions.md +++ b/doc/user/admin_area/settings/sign_up_restrictions.md @@ -7,6 +7,7 @@ type: reference You can use sign-up restrictions to: - Disable new sign-ups. +- Require admin approval for new sign-ups. - Require user email confirmation. - Denylist or allowlist email addresses belonging to specific domains. @@ -32,12 +33,20 @@ Alternatively, you could also consider setting up a [allowlist](#allowlist-email-domains) or [denylist](#denylist-email-domains) on email domains to prevent malicious users from creating accounts. +## Require admin approval for new sign-ups + +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/4491) in GitLab 13.5. + +When this setting is enabled, any user visiting your GitLab domain and signing up for a new account will have to be explicitly [approved](../approving_users.md#approving-a-user) by an administrator before they can start using their account. + + + ## Require email confirmation You can send confirmation emails during sign-up and require that users confirm their email address before they are allowed to sign in. - + ## Minimum password length limit diff --git a/doc/user/admin_area/settings/usage_statistics.md b/doc/user/admin_area/settings/usage_statistics.md index 55bbcfbe1d8..140d149555a 100644 --- a/doc/user/admin_area/settings/usage_statistics.md +++ b/doc/user/admin_area/settings/usage_statistics.md @@ -58,7 +58,7 @@ sequenceDiagram ## Usage Ping **(CORE ONLY)** -See [Usage Ping guide](../../../development/telemetry/usage_ping.md). +See [Usage Ping guide](../../../development/product_analytics/usage_ping.md). ## Instance-level statistics **(CORE ONLY)** diff --git a/doc/user/analytics/img/mr_throughput_chart_v13_3.png b/doc/user/analytics/img/mr_throughput_chart_v13_3.png Binary files differindex 04fa54f323c..100c9a8557c 100644 --- a/doc/user/analytics/img/mr_throughput_chart_v13_3.png +++ b/doc/user/analytics/img/mr_throughput_chart_v13_3.png diff --git a/doc/user/analytics/img/mr_throughput_table_v13_3.png b/doc/user/analytics/img/mr_throughput_table_v13_3.png Binary files differindex 63ffb9389f4..bb63770dc3f 100644 --- a/doc/user/analytics/img/mr_throughput_table_v13_3.png +++ b/doc/user/analytics/img/mr_throughput_table_v13_3.png diff --git a/doc/user/analytics/img/new_value_stream_v13_3.png b/doc/user/analytics/img/new_value_stream_v13_3.png Binary files differindex 4284b8ab194..bc8502e85a6 100644 --- a/doc/user/analytics/img/new_value_stream_v13_3.png +++ b/doc/user/analytics/img/new_value_stream_v13_3.png diff --git a/doc/user/analytics/img/vsa_filter_bar_v13.3.png b/doc/user/analytics/img/vsa_filter_bar_v13.3.png Binary files differindex 71e59892434..506765f63cb 100644 --- a/doc/user/analytics/img/vsa_filter_bar_v13.3.png +++ b/doc/user/analytics/img/vsa_filter_bar_v13.3.png diff --git a/doc/user/analytics/index.md b/doc/user/analytics/index.md index 044b9eb3e64..54b4c702c5c 100644 --- a/doc/user/analytics/index.md +++ b/doc/user/analytics/index.md @@ -25,10 +25,8 @@ The following analytics features are available at the group level: - [Contribution](../group/contribution_analytics/index.md). **(STARTER)** - [Insights](../group/insights/index.md). **(ULTIMATE)** - [Issue](../group/issues_analytics/index.md). **(PREMIUM)** -- [Productivity](productivity_analytics.md), enabled with the `productivity_analytics` - [feature flag](../../development/feature_flags/development.md#enabling-a-feature-flag-locally-in-development). **(PREMIUM)** -- [Value Stream](value_stream_analytics.md), enabled with the `cycle_analytics` - [feature flag](../../development/feature_flags/development.md#enabling-a-feature-flag-locally-in-development). **(PREMIUM)** +- [Productivity](productivity_analytics.md) **(PREMIUM)** +- [Value Stream](value_stream_analytics.md). **(PREMIUM)** ## Project-level analytics @@ -40,6 +38,5 @@ The following analytics features are available at the project level: - [Issue](../group/issues_analytics/index.md). **(PREMIUM)** - [Merge Request](merge_request_analytics.md), enabled with the `project_merge_request_analytics` [feature flag](../../development/feature_flags/development.md#enabling-a-feature-flag-locally-in-development). **(STARTER)** -- [Repository](repository_analytics.md). -- [Value Stream](value_stream_analytics.md), enabled with the `cycle_analytics` - [feature flag](../../development/feature_flags/development.md#enabling-a-feature-flag-locally-in-development). **(STARTER)** +- [Repository](repository_analytics.md). **(CORE)** +- [Value Stream](value_stream_analytics.md). **(CORE)** diff --git a/doc/user/analytics/merge_request_analytics.md b/doc/user/analytics/merge_request_analytics.md index 6a18d46fd1a..04a5fa71e19 100644 --- a/doc/user/analytics/merge_request_analytics.md +++ b/doc/user/analytics/merge_request_analytics.md @@ -73,18 +73,3 @@ The **Merge Request Analytics** feature can be accessed only: - On [Starter](https://about.gitlab.com/pricing/) and above. - By users with [Reporter access](../permissions.md) and above. - -## Enable and disable related feature flags - -Merge Request Analytics is disabled by default but can be enabled using the following -[feature flag](../../development/feature_flags/development.md#enabling-a-feature-flag-locally-in-development): - -- `project_merge_request_analytics` - -A GitLab administrator can: - -- Enable this feature by running the following command in a Rails console: - - ```ruby - Feature.enable(:project_merge_request_analytics) - ``` diff --git a/doc/user/analytics/productivity_analytics.md b/doc/user/analytics/productivity_analytics.md index 653836de8be..951be1c550b 100644 --- a/doc/user/analytics/productivity_analytics.md +++ b/doc/user/analytics/productivity_analytics.md @@ -13,7 +13,7 @@ Track development velocity with Productivity Analytics. For many companies, the development cycle is a black box and getting an estimate of how long, on average, it takes to deliver features is an enormous endeavor. -While [Value Stream Analytics](../project/cycle_analytics.md) focuses on the entire +While [Value Stream Analytics](../analytics/value_stream_analytics.md) focuses on the entire Software Development Life Cycle (SDLC) process, Productivity Analytics provides a way for Engineering Management to drill down in a systematic way to uncover patterns and causes for success or failure at an individual, project, or group level. Productivity can slow down for many reasons ranging from degrading code base to quickly growing teams. In order to investigate, department or team leaders can start by visualizing the time it takes for merge requests to be merged. @@ -62,29 +62,3 @@ The **Productivity Analytics** dashboard can be accessed only: - On [Premium or Silver tier](https://about.gitlab.com/pricing/) and above. - By users with [Reporter access](../permissions.md) and above. - -## Enabling and disabling using feature flags - -Productivity Analytics is: - -- [Enabled by default](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/18754) from GitLab 12.4, - but can be disabled using the following feature flags: - - `productivity_analytics`. - - `productivity_analytics_scatterplot_enabled`. -- Disabled by default in GitLab 12.3, but can be enabled using the following feature flag: - - `productivity_analytics`. - -A GitLab administrator can: - -- Disable this feature from GitLab 12.4 by running the follow in a Rails console: - - ```ruby - Feature.disable(:productivity_analytics) - Feature.disable(:productivity_analytics_scatterplot_enabled) - ``` - -- Enable this feature in GitLab 12.3 by running the following in a Rails console: - - ```ruby - Feature.enable(:productivity_analytics) - ``` diff --git a/doc/user/analytics/value_stream_analytics.md b/doc/user/analytics/value_stream_analytics.md index 14012d4a28d..86525587b30 100644 --- a/doc/user/analytics/value_stream_analytics.md +++ b/doc/user/analytics/value_stream_analytics.md @@ -12,26 +12,26 @@ info: To determine the technical writer assigned to the Stage/Group associated w Value Stream Analytics measures the time spent to go from an [idea to production](https://about.gitlab.com/blog/2016/08/05/continuous-integration-delivery-and-deployment-with-gitlab/#from-idea-to-production-with-gitlab) -(also known as cycle time) for each of your projects. Value Stream Analytics displays the median time +(also known as cycle time) for each of your projects or groups. Value Stream Analytics displays the median time spent in each stage defined in the process. -For information on how to contribute to the development of Value Stream Analytics, see our [contributor documentation](../../development/value_stream_analytics.md). - Value Stream Analytics is useful in order to quickly determine the velocity of a given project. It points to bottlenecks in the development process, enabling management to uncover, triage, and identify the root cause of slowdowns in the software development life cycle. -Value Stream Analytics is tightly coupled with the [GitLab flow](../../topics/gitlab_flow.md) and -calculates a separate median for each stage. +For information on how to contribute to the development of Value Stream Analytics, see our [contributor documentation](../../development/value_stream_analytics.md). + +## Project Level Value Stream Analytics **(CORE)** + +Project Level Value Stream Analytics is available via **Project > Analytics > Value Stream**. -## Overview +## Group Level Value Stream Analytics **(PREMIUM)** -Value Stream Analytics is available: +From GitLab 12.9, group level Value Stream Analytics is available via **Group > Analytics > Value Stream**. -- From GitLab 12.9, at the group level via **Group > Analytics > Value Stream**. **(PREMIUM)** -- At the project level via **Project > Analytics > Value Stream**. +## Default stages -There are seven stages that are tracked as part of the Value Stream Analytics calculations. +The stages tracked by Value Stream Analytics by default represent the [GitLab flow](../../topics/gitlab_flow.md). These stages can be customized in Group Level Value Stream Analytics. - **Issue** (Tracker) - Time to schedule an issue (by milestone or by adding it to an issue board) @@ -96,8 +96,7 @@ Value Stream Analytics records stage time and data based on the project issues w exception of the staging stage, where only data deployed to production are measured. -Specifically, if your CI is not set up and you have not defined a `production` -or `production/*` [environment](../../ci/yaml/README.md#environment), then you will not have any +Specifically, if your CI is not set up and you have not defined a [production environment](#how-the-production-environment-is-identified), then you will not have any data for this stage. Each stage of Value Stream Analytics is further described in the table below. @@ -109,7 +108,7 @@ Each stage of Value Stream Analytics is further described in the table below. | Code | Measures the median time between pushing a first commit (previous stage) and creating a merge request (MR) related to that commit. The key to keep the process tracked is to include the [issue closing pattern](../project/issues/managing_issues.md#closing-issues-automatically) to the description of the merge request (for example, `Closes #xxx`, where `xxx` is the number of the issue related to this merge request). If the issue closing pattern is not present in the merge request description, the MR is not considered to the measurement time of the stage. | | Test | Measures the median time to run the entire pipeline for that project. It's related to the time GitLab CI/CD takes to run every job for the commits pushed to that merge request defined in the previous stage. It is basically the start->finish time for all pipelines. | | Review | Measures the median time taken to review the merge request that has a closing issue pattern, between its creation and until it's merged. | -| Staging | Measures the median time between merging the merge request with a closing issue pattern until the very first deployment to production. It's tracked by the environment set to `production` or matching `production/*` (case-sensitive, `Production` won't work) in your GitLab CI/CD configuration. If there isn't a production environment, this is not tracked. | +| Staging | Measures the median time between merging the merge request with a closing issue pattern until the very first deployment to a [production environment](#how-the-production-environment-is-identified). If there isn't a production environment, this is not tracked. | How this works, behind the scenes: @@ -123,13 +122,23 @@ How this works, behind the scenes: we need for the stages, like issue creation date, merge request merge time, etc. -To sum up, anything that doesn't follow [GitLab flow](../../workflow/gitlab_flow.md) will not be tracked and the +To sum up, anything that doesn't follow [GitLab flow](../../topics/gitlab_flow.md) will not be tracked and the Value Stream Analytics dashboard will not present any data for: - Merge requests that do not close an issue. - Issues not labeled with a label present in the Issue Board or for issues not assigned a milestone. -- Staging stage, if the project has no `production` or `production/*` - environment. +- Staging stage, if the project has no [production environment](#how-the-production-environment-is-identified). + +## How the production environment is identified + +Value Stream Analytics identifies production environments by looking for project [environments](../../ci/yaml/README.md#environment) with a name matching any of these patterns: + +- `prod` or `prod/*` +- `production` or `production/*` + +These patterns are not case-sensitive. + +You can change the name of a project environment in your GitLab CI/CD configuration. ## Example workflow @@ -345,7 +354,7 @@ administrator can open a Rails console and disable it with the following command Feature.disable(:cycle_analytics_scatterplot_enabled) ``` -## Type of work - Tasks by type chart **(PREMIUM)** +## Type of work - Tasks by type chart > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/32421) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.10. @@ -368,11 +377,8 @@ The current permissions on the Project Value Stream Analytics dashboard are: You can [read more about permissions](../../user/permissions.md) in general. -For Value Stream Analytics functionality introduced in GitLab 12.3 and later: - -- Users must have Reporter access or above. -- Features are available only on - [Premium or Silver tiers](https://about.gitlab.com/pricing/) and above. +For Value Stream Analytics functionality introduced in GitLab 12.3 and later, +users must have Reporter access or above. ## More resources diff --git a/doc/user/application_security/api_fuzzing/index.md b/doc/user/application_security/api_fuzzing/index.md index ae22655e30b..145422f8736 100644 --- a/doc/user/application_security/api_fuzzing/index.md +++ b/doc/user/application_security/api_fuzzing/index.md @@ -8,8 +8,8 @@ type: reference, howto # Web API Fuzz Testing **(ULTIMATE)** You can add web API fuzzing to your [GitLab CI/CD](../../../ci/README.md) -pipelines. This helps you discover bugs and potential security issues that other QA processes may miss. -API fuzzing performs fuzz testing of API operation parameters. +pipelines. This helps you discover bugs and potential security issues that other QA processes may miss. +API fuzzing performs fuzz testing of API operation parameters. Fuzz testing sets operation parameters to unexpected values in an effort to cause unexpected behavior and errors in the API backend. We recommend that you use fuzz testing in addition to [GitLab Secure](../index.md)'s @@ -443,7 +443,7 @@ Example usage for setting a single header: ```json { "headers": { - "Authorization": "Bearer dXNlcm5hbWU6cGFzc3dvcmQ=" + "Authorization": "Bearer dXNlcm5hbWU6cGFzc3dvcmQ=" } } ``` @@ -453,7 +453,7 @@ Example usage for setting both a header and cookie: ```json { "headers": { - "Authorization": "Bearer dXNlcm5hbWU6cGFzc3dvcmQ=" + "Authorization": "Bearer dXNlcm5hbWU6cGFzc3dvcmQ=" }, "cookies": { "flags": "677" diff --git a/doc/user/application_security/configuration/index.md b/doc/user/application_security/configuration/index.md index a6ad701360e..ead34ca227e 100644 --- a/doc/user/application_security/configuration/index.md +++ b/doc/user/application_security/configuration/index.md @@ -19,8 +19,9 @@ then in the left sidebar go to **Security & Compliance > Configuration**. For each security control the page displays: -- **Status** - Status of the security control: enabled, not enabled, or available. -- **Manage** - A management option or a link to the documentation. +- **Security Control:** Name, description, and a documentation link. +- **Status:** The security control's status (enabled, not enabled, or available). +- **Manage:** A management option or a documentation link. ## Status @@ -29,12 +30,11 @@ The status of each security control is determined by the project's latest defaul If a job with the expected security report artifact exists in the pipeline, the feature's status is _enabled_. -For SAST, click **View history** to see the `.gitlab-ci.yml` file’s history. - -NOTE: **Note:** If the latest pipeline used [Auto DevOps](../../../topics/autodevops/index.md), all security features are configured by default. +For SAST, click **View history** to see the `.gitlab-ci.yml` file's history. + ## Manage You can configure the following security controls: diff --git a/doc/user/application_security/container_scanning/index.md b/doc/user/application_security/container_scanning/index.md index 880e5a3875a..9e7f98dd4fc 100644 --- a/doc/user/application_security/container_scanning/index.md +++ b/doc/user/application_security/container_scanning/index.md @@ -9,8 +9,6 @@ info: To determine the technical writer assigned to the Stage/Group associated w > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/3672) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 10.4. -## Overview - Your application's Docker image may itself be based on Docker images that contain known vulnerabilities. By including an extra job in your pipeline that scans for those vulnerabilities and displays them in a merge request, you can use GitLab to audit your Docker-based apps. @@ -19,7 +17,6 @@ By default, container scanning in GitLab is based on [Clair](https://github.com/ containers. [GitLab's Klar analyzer](https://gitlab.com/gitlab-org/security-products/analyzers/klar/) scans the containers and serves as a wrapper for Clair. -NOTE: **Note:** To integrate security scanners other than Clair and Klar into GitLab, see [Security scanner integration](../../../development/integrations/secure.md). @@ -46,7 +43,7 @@ To enable container scanning in your pipeline, you need the following: or [`kubernetes`](https://docs.gitlab.com/runner/install/kubernetes.html) executor. - Docker `18.09.03` or higher installed on the same computer as the runner. If you're using the shared runners on GitLab.com, then this is already the case. -- [Build and push](../../packages/container_registry/index.md#container-registry-examples-with-gitlab-cicd) +- [Build and push](../../packages/container_registry/index.md#build-and-push-by-using-gitlab-cicd) your Docker image to your project's container registry. The name of the Docker image should use the following [predefined environment variables](../../../ci/variables/predefined_variables.md): @@ -65,7 +62,7 @@ To enable container scanning in your pipeline, you need the following: variables: IMAGE_TAG: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG:$CI_COMMIT_SHA script: - - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - docker build -t $IMAGE_TAG . - docker push $IMAGE_TAG ``` @@ -119,7 +116,7 @@ build: IMAGE: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG:$CI_COMMIT_SHA script: - docker info - - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - docker build -t $IMAGE . - docker push $IMAGE @@ -219,14 +216,21 @@ To use container scanning in an offline environment, you need: - GitLab Runner with the [`docker` or `kubernetes` executor](#requirements). - To configure a local Docker container registry with copies of the container scanning [analyzer](https://gitlab.com/gitlab-org/security-products/analyzers/klar) images, found in the [container scanning container registry](https://gitlab.com/gitlab-org/security-products/analyzers/klar/container_registry). -NOTE: **Note:** -GitLab Runner has a [default `pull policy` of `always`](https://docs.gitlab.com/runner/executors/docker.html#using-the-always-pull-policy), +Note that GitLab Runner has a [default `pull policy` of `always`](https://docs.gitlab.com/runner/executors/docker.html#using-the-always-pull-policy), meaning the runner tries to pull Docker images from the GitLab container registry even if a local copy is available. The GitLab Runner [`pull_policy` can be set to `if-not-present`](https://docs.gitlab.com/runner/executors/docker.html#using-the-if-not-present-pull-policy) in an offline environment if you prefer using only locally available Docker images. However, we recommend keeping the pull policy setting to `always` if not in an offline environment, as this enables the use of updated scanners in your CI/CD pipelines. +##### Support for Custom Certificate Authorities + +Support for custom certificate authorities was introduced in the following versions: + +| Analyzer | Version | +| -------- | ------- | +| `klar` | [v2.3.0](https://gitlab.com/gitlab-org/security-products/analyzers/klar/-/releases/v2.3.0) | + #### Make GitLab container scanning analyzer images available inside your Docker registry For container scanning, import the following default images from `registry.gitlab.com` into your @@ -287,7 +291,7 @@ build_latest_vulnerabilities: script: - docker pull arminc/clair-db:latest - docker tag arminc/clair-db:latest $CI_REGISTRY/namespace/clair-vulnerabilities-db - - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - docker push $CI_REGISTRY/namespace/clair-vulnerabilities-db ``` @@ -433,3 +437,7 @@ This is a result of a bug in Docker which is now [fixed](https://github.com/cont To prevent the error, ensure the Docker version that the runner is using is `18.09.03` or higher. For more information, see [issue #10241](https://gitlab.com/gitlab-org/gitlab/-/issues/10241 "Investigate why Container Scanning is not working with NFS mounts"). + +### Getting warning message `gl-container-scanning-report.json: no matching files` + +For information on this, see the [general Application Security troubleshooting section](../../../ci/pipelines/job_artifacts.md#error-message-no-files-to-upload). diff --git a/doc/user/application_security/coverage_fuzzing/index.md b/doc/user/application_security/coverage_fuzzing/index.md index dff71cb9445..9508407ccae 100644 --- a/doc/user/application_security/coverage_fuzzing/index.md +++ b/doc/user/application_security/coverage_fuzzing/index.md @@ -175,6 +175,52 @@ To use coverage fuzzing in an offline environment, follow these steps: `NEW_URL_GITLAB_COV_FUZ` is the URL of the private `gitlab-cov-fuzz` clone that you set up in the first step. +### Continuous fuzzing (long-running async fuzzing jobs) + +It's also possible to run the fuzzing jobs longer and without blocking your main pipeline. This +configuration uses the GitLab [parent-child pipelines](../../../ci/parent_child_pipelines.md). +The full example is available in the [repository](https://gitlab.com/gitlab-org/security-products/demos/coverage-fuzzing/go-fuzzing-example/-/tree/continuous_fuzzing#running-go-fuzz-from-ci). +This example uses Go, but is applicable for any other supported languages. + +The suggested workflow in this scenario is to have long-running, async fuzzing jobs on a +main/development branch, and short, blocking sync fuzzing jobs on all other branches and MRs. This +is a good way to balance the needs of letting a developer's per-commit pipeline complete quickly, +and also giving the fuzzer a large amount of time to fully explore and test the app. + +Long-running fuzzing jobs are usually necessary for the coverage guided fuzzer to find deeper bugs +in your latest code base. THe following is an example of what `.gitlab-ci.yml` looks like in this +workflow (for the full example, see the [repository](https://gitlab.com/gitlab-org/security-products/demos/coverage-fuzzing/go-fuzzing-example/-/tree/continuous_fuzzing)): + +```yaml + +sync_fuzzing: + variables: + COVFUZZ_ADDITIONAL_ARGS: '-max_total_time=300' + trigger: + include: .covfuzz-ci.yml + strategy: depend + rules: + - if: $CI_COMMIT_BRANCH != 'continuous_fuzzing' && $CI_PIPELINE_SOURCE != 'merge_request_event' + +async_fuzzing: + variables: + COVFUZZ_ADDITIONAL_ARGS: '-max_total_time=3600' + trigger: + include: .covfuzz-ci.yml + rules: + - if: $CI_COMMIT_BRANCH == 'continuous_fuzzing' && $CI_PIPELINE_SOURCE != 'merge_request_event' +``` + +This essentially creates two steps: + +1. `sync_fuzzing`: Runs all your fuzz targets for a short period of time in a blocking + configuration. This finds simple bugs and allows you to be confident that your MRs aren't + introducing new bugs or causing old bugs to reappear. +1. `async_fuzzing`: Runs on your branch and finds deep bugs in your code without blocking your + development cycle and MRs. + +The `covfuzz-ci.yml` is the same as that in the [original synchronous example](https://gitlab.com/gitlab-org/security-products/demos/coverage-fuzzing/go-fuzzing-example#running-go-fuzz-from-ci). + ### Glossary - Seed corpus: The set of test cases given as initial input to the fuzz target. This usually speeds diff --git a/doc/user/application_security/dast/img/dast_v13_2.png b/doc/user/application_security/dast/img/dast_v13_2.png Binary files differdeleted file mode 100644 index bbf7944eb40..00000000000 --- a/doc/user/application_security/dast/img/dast_v13_2.png +++ /dev/null diff --git a/doc/user/application_security/dast/img/dast_v13_4.png b/doc/user/application_security/dast/img/dast_v13_4.png Binary files differnew file mode 100644 index 00000000000..d9c1d1b5c66 --- /dev/null +++ b/doc/user/application_security/dast/img/dast_v13_4.png diff --git a/doc/user/application_security/dast/index.md b/doc/user/application_security/dast/index.md index 73a8e727389..fffaf4ad26b 100644 --- a/doc/user/application_security/dast/index.md +++ b/doc/user/application_security/dast/index.md @@ -9,17 +9,17 @@ type: reference, howto > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/4348) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 10.4. -NOTE: **Note:** -The whitepaper ["A Seismic Shift in Application Security"](https://about.gitlab.com/resources/whitepaper-seismic-shift-application-security/) -explains how **4 of the top 6 attacks were application based**. Download it -to learn how to protect your organization. - Running [static checks](../sast/index.md) on your code is the first step to detect vulnerabilities that can put the security of your code at risk. Yet, once deployed, your application is exposed to a new category of possible attacks, such as cross-site scripting or broken authentication flaws. This is where Dynamic Application Security Testing (DAST) comes into place. +NOTE: **Note:** +The whitepaper ["A Seismic Shift in Application Security"](https://about.gitlab.com/resources/whitepaper-seismic-shift-application-security/) +explains how 4 of the top 6 attacks were application based. Download it to learn how to protect your +organization. + ## Overview If you're using [GitLab CI/CD](../../../ci/README.md), you can analyze your running web applications @@ -32,11 +32,10 @@ provided by [Auto DevOps](../../../topics/autodevops/index.md). GitLab checks the DAST report, compares the found vulnerabilities between the source and target branches, and shows the information on the merge request. -NOTE: **Note:** -This comparison logic uses only the latest pipeline executed for the target branch's base commit. -Running the pipeline on any other commit has no effect on the merge request. +Note that this comparison logic uses only the latest pipeline executed for the target branch's base +commit. Running the pipeline on any other commit has no effect on the merge request. - + By clicking on one of the detected linked vulnerabilities, you can see the details and the URL(s) affected. @@ -53,12 +52,11 @@ However, DAST can be [configured](#full-scan) to also perform an *active scan*: attack your application and produce a more extensive security report. It can be very useful combined with [Review Apps](../../../ci/review_apps/index.md). -NOTE: **Note:** -A pipeline may consist of multiple jobs, including SAST and DAST scanning. If any -job fails to finish for any reason, the security dashboard doesn't show DAST scanner -output. For example, if the DAST job finishes but the SAST job fails, the security -dashboard doesn't show DAST results. The analyzer outputs an -[exit code](../../../development/integrations/secure.md#exit-code) on failure. +Note that a pipeline may consist of multiple jobs, including SAST and DAST scanning. If any job +fails to finish for any reason, the security dashboard doesn't show DAST scanner output. For +example, if the DAST job finishes but the SAST job fails, the security dashboard doesn't show DAST +results. On failure, the analyzer outputs an +[exit code](../../../development/integrations/secure.md#exit-code). ## Use cases @@ -206,8 +204,8 @@ variables: DAST_FULL_SCAN_ENABLED: "true" ``` -NOTE: **Note:** -If your DAST job exceeds the job timeout and you need to reduce the scan duration, we shared some tips for optimizing DAST scans in a [blog post](https://about.gitlab.com/blog/2020/08/31/how-to-configure-dast-full-scans-for-complex-web-applications/). +If your DAST job exceeds the job timeout and you need to reduce the scan duration, we shared some +tips for optimizing DAST scans in a [blog post](https://about.gitlab.com/blog/2020/08/31/how-to-configure-dast-full-scans-for-complex-web-applications/). #### Domain validation @@ -398,11 +396,9 @@ variables: DAST_API_HOST_OVERRIDE: api-test.host.com ``` -NOTE: **Note:** -Using a host override is ONLY supported when importing the API -specification from a URL. It does not work and will be ignored when importing -the specification from a file. This is due to a limitation in the ZAP OpenAPI -extension. +Note that using a host override is ONLY supported when importing the API specification from a URL. +It doesn't work and is ignored when importing the specification from a file. This is due to a +limitation in the ZAP OpenAPI extension. #### Authentication using headers @@ -427,7 +423,8 @@ A URL scan allows you to specify which parts of a website are scanned by DAST. #### Define the URLs to scan -To specify the paths to be scanned, add a comma-separated list of the paths to the `DAST_PATHS` environment variable. Note that you can only scan paths of a single host. +To specify the paths to scan, add a comma-separated list of the paths to the `DAST_PATHS` +environment variable. Note that you can only scan paths of a single host. ```yaml include: @@ -437,8 +434,10 @@ variables: DAST_PATHS=/page1.html,/category1/page1.html,/page3.html ``` -NOTE: **Note:** -`DAST_AUTH_EXCLUDE_URLS` are ignored when `DAST_PATHS` is set. +When using `DAST_PATHS`, note the following: + +- The `DAST_PATHS` environment variable has a limit of about 130kb. If you have a list or paths + greater than this, you should create multiple DAST jobs and split the paths over each job. #### Full Scan @@ -590,8 +589,7 @@ To use DAST in an offline environment, you need: [container image](https://gitlab.com/gitlab-org/security-products/dast), found in the [DAST container registry](https://gitlab.com/gitlab-org/security-products/dast/container_registry). -NOTE: **Note:** -GitLab Runner has a [default `pull policy` of `always`](https://docs.gitlab.com/runner/executors/docker.html#using-the-always-pull-policy), +Note that GitLab Runner has a [default `pull policy` of `always`](https://docs.gitlab.com/runner/executors/docker.html#using-the-always-pull-policy), meaning the runner tries to pull Docker images from the GitLab container registry even if a local copy is available. The GitLab Runner [`pull_policy` can be set to `if-not-present`](https://docs.gitlab.com/runner/executors/docker.html#using-the-if-not-present-pull-policy) in an offline environment if you prefer using only locally available Docker images. However, we @@ -672,11 +670,6 @@ To delete an existing site profile: ## Scanner profile > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/222767) in GitLab 13.4. -> - [Deployed behind a feature flag](../../feature_flags.md), enabled by default. -> - Enabled on GitLab.com. -> - Can be enabled or disabled per-project. -> - Recommended for production use. -> - For GitLab self-managed instances, GitLab administrators can [disable this feature](#enable-or-disable-dast-scanner-profiles). A scanner profile defines the scanner settings used to run an on-demand scan: @@ -684,6 +677,11 @@ A scanner profile defines the scanner settings used to run an on-demand scan: - **Spider timeout:** The maximum number of minutes allowed for the spider to traverse the site. - **Target timeout:** The maximum number of seconds DAST waits for the site to be available before starting the scan. +- **Scan mode:** A passive scan monitors all HTTP messages (requests and responses) sent to the target. An active scan attacks the target to find potential vulnerabilities. +- **AJAX spider:** Run the AJAX spider, in addition to the traditional spider, to crawl the target site. +- **Debug messages:** Include debug messages in the DAST console output. + +Scan mode, AJAX spider, Debug messages are [added in GitLab 13.5](https://gitlab.com/gitlab-org/gitlab/-/issues/225804) ### Create a scanner profile @@ -711,29 +709,6 @@ To delete a scanner profile: 1. Click **Manage** in the **DAST Profiles** row. 1. Click **{remove}** in the scanner profile's row. -### Enable or disable DAST scanner profiles - -The scanner profile feature is ready for production use. It's deployed behind a feature flag that -is **enabled by default**. [GitLab administrators with access to the GitLab Rails console](../../../administration/feature_flags.md) can opt to disable it. - -To disable it: - -```ruby -# Instance-wide -Feature.disable(:security_on_demand_scans_scanner_profiles) -# or by project -Feature.disable(:security_on_demand_scans_scanner_profiles, Project.find(<project id>)) -``` - -To enable it: - -```ruby -# Instance-wide -Feature.enable(:security_on_demand_scans_scanner_profiles) -# or by project -Feature.enable(:security_on_demand_scans_scanner_profiles, Project.find(<project ID>)) -``` - ## On-demand scans > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/218465) in GitLab 13.2. @@ -756,7 +731,8 @@ An on-demand DAST scan: NOTE: **Note:** You must have permission to run an on-demand DAST scan against a protected branch. -The default branch is automatically protected. For more details, see [Pipeline security on protected branches](../../../ci/pipelines/index.md#pipeline-security-on-protected-branches). +The default branch is automatically protected. For more information, see +[Pipeline security on protected branches](../../../ci/pipelines/index.md#pipeline-security-on-protected-branches). To run an on-demand DAST scan, you need: @@ -765,8 +741,8 @@ To run an on-demand DAST scan, you need: 1. From your project's home page, go to **Security & Compliance > On-demand Scans** in the left sidebar. 1. Click **Create new DAST scan**. -1. In **Scanner settings**, select a scanner profile from the dropdown. -1. In **Site profiles**, select a site profile from the dropdown. +1. In **Scanner profile**, select a scanner profile from the dropdown. +1. In **Site profile**, select a site profile from the dropdown. 1. Click **Run scan**. The on-demand DAST scan runs and the project's dashboard shows the results. @@ -866,7 +842,7 @@ include: template: DAST.gitlab-ci.yml variables: - DAST_INCLUDE_ALPHA_VULNERABILITIES: true + DAST_INCLUDE_ALPHA_VULNERABILITIES: "true" ``` ## Interacting with the vulnerabilities @@ -923,6 +899,10 @@ Change the number after `-Xmx` to the required memory amount. If your DAST job exceeds the job timeout and you need to reduce the scan duration, we shared some tips for optimizing DAST scans in a [blog post](https://about.gitlab.com/blog/2020/08/31/how-to-configure-dast-full-scans-for-complex-web-applications/). +### Getting warning message `gl-dast-report.json: no matching files` + +For information on this, see the [general Application Security troubleshooting section](../../../ci/pipelines/job_artifacts.md#error-message-no-files-to-upload). + <!-- ## Troubleshooting Include any troubleshooting steps that you can foresee. If you know beforehand what issues diff --git a/doc/user/application_security/dependency_scanning/index.md b/doc/user/application_security/dependency_scanning/index.md index 5cce336d04c..b90bb37c60f 100644 --- a/doc/user/application_security/dependency_scanning/index.md +++ b/doc/user/application_security/dependency_scanning/index.md @@ -9,25 +9,26 @@ info: To determine the technical writer assigned to the Stage/Group associated w > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/5105) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 10.7. -Dependency Scanning helps to find security vulnerabilities in your dependencies automatically -while you're developing and testing your applications, such as when your -application is using an external (open source) library that is known to be vulnerable. +GitLab's Dependency Scanning feature can automatically find security vulnerabilities in your +dependencies while you're developing and testing your applications. For example, dependency scanning +lets you know if your application uses an external (open source) library that is known to be +vulnerable. You can then take action to protect your application. ## Overview -If you're using [GitLab CI/CD](../../../ci/README.md), you can analyze your dependencies for known -vulnerabilities using Dependency Scanning. -All dependencies are scanned, including the transitive dependencies (also known as nested dependencies). -You can take advantage of Dependency Scanning by either [including the Dependency Scanning template](#configuration) -in your existing `.gitlab-ci.yml` file or by implicitly using -the [Auto Dependency Scanning](../../../topics/autodevops/stages.md#auto-dependency-scanning) +If you're using [GitLab CI/CD](../../../ci/README.md), you can use dependency scanning to analyze +your dependencies for known vulnerabilities. GitLab scans all dependencies, including transitive +dependencies (also known as nested dependencies). You can take advantage of dependency scanning by +either [including the dependency scanning template](#configuration) +in your existing `.gitlab-ci.yml` file, or by implicitly using +the [auto dependency scanning](../../../topics/autodevops/stages.md#auto-dependency-scanning) provided by [Auto DevOps](../../../topics/autodevops/index.md). -GitLab checks the Dependency Scanning report, compares the found vulnerabilities +GitLab checks the dependency scanning report, compares the found vulnerabilities between the source and target branches, and shows the information on the merge request. - + The results are sorted by the severity of the vulnerability: @@ -40,7 +41,7 @@ The results are sorted by the severity of the vulnerability: ## Requirements -To run Dependency Scanning jobs, by default, you need GitLab Runner with the +To run dependency scanning jobs, by default, you need GitLab Runner with the [`docker`](https://docs.gitlab.com/runner/executors/docker.html) or [`kubernetes`](https://docs.gitlab.com/runner/install/kubernetes.html) executor. If you're using the shared runners on GitLab.com, this is enabled by default. @@ -56,24 +57,25 @@ The current detection logic limits the maximum search depth to two levels. For e The following languages and dependency managers are supported: -| Language (package managers) | Supported files | Scan tool(s) | -|----------------------------- | --------------- | ------------ | -| C# .NET ([NuGet](https://www.nuget.org/) 4.9+) | [`packages.lock.json`](https://docs.microsoft.com/en-us/nuget/consume-packages/package-references-in-project-files#enabling-lock-file) | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | -| C/C++ ([Conan](https://conan.io/)) | [`conan.lock`](https://docs.conan.io/en/latest/versioning/lockfiles.html) | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | -| Java ([Gradle](https://gradle.org/), [Maven](https://maven.apache.org/)) | `build.gradle`, `build.gradle.kts`, `pom.xml` | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | -| JavaScript ([npm](https://www.npmjs.com/), [yarn](https://classic.yarnpkg.com/en/)) | `package-lock.json`, `npm-shrinkwrap.json`, `yarn.lock` | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium), [Retire.js](https://retirejs.github.io/retire.js/) | -| Go ([Golang](https://golang.org/)) | `go.sum` | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | -| PHP ([Composer](https://getcomposer.org/)) | `composer.lock` | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | -| Python ([setuptools](https://setuptools.readthedocs.io/en/latest/), [pip](https://pip.pypa.io/en/stable/), [Pipenv](https://pipenv.pypa.io/en/latest/)) | `setup.py`, `requirements.txt`, `requirements.pip`, `requires.txt`, `Pipfile` | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | -| Ruby ([Bundler](https://bundler.io/)) | `Gemfile.lock`, `gems.locked` | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium), [bundler-audit](https://github.com/rubysec/bundler-audit) | -| Scala ([sbt](https://www.scala-sbt.org/)) | `build.sbt` | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | +| Package Managers | Languages | Supported files | Scan tools | +| ------------------- | --------- | --------------- | ------------ | +| [Bundler](https://bundler.io/) | Ruby | `Gemfile.lock`, `gems.locked` | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium), [bundler-audit](https://github.com/rubysec/bundler-audit) | +| [Composer](https://getcomposer.org/) | PHP | `composer.lock` | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | +| [Conan](https://conan.io/) | C, C++ | [`conan.lock`](https://docs.conan.io/en/latest/versioning/lockfiles.html) | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | +| [Golang](https://golang.org/) | Go | `go.sum` | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | +| [Gradle](https://gradle.org/), [Maven](https://maven.apache.org/) | Java | `build.gradle`, `build.gradle.kts`, `pom.xml` | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | +| [npm](https://www.npmjs.com/), [yarn](https://classic.yarnpkg.com/en/) | JavaScript | `package-lock.json`, `npm-shrinkwrap.json`, `yarn.lock` | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium), [Retire.js](https://retirejs.github.io/retire.js/) | +| [NuGet](https://www.nuget.org/) 4.9+ | .NET, C# | [`packages.lock.json`](https://docs.microsoft.com/en-us/nuget/consume-packages/package-references-in-project-files#enabling-lock-file) | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | +| [setuptools](https://setuptools.readthedocs.io/en/latest/), [pip](https://pip.pypa.io/en/stable/), [Pipenv](https://pipenv.pypa.io/en/latest/) | Python | `setup.py`, `requirements.txt`, `requirements.pip`, `requires.txt`, `Pipfile` | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | +| [sbt](https://www.scala-sbt.org/) 1.2 and below ([Ivy](http://ant.apache.org/ivy/)) | Scala | `build.sbt` | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | Plans are underway for supporting the following languages, dependency managers, and dependency files. For details, see the issue link for each. -| Language (package managers) | Supported files | Scan tool(s) | Issue | -|----------------------------- | --------------- | ------------ | ----- | -| Python ([Poetry](https://python-poetry.org/)) | `poetry.lock` | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | [GitLab#7006](https://gitlab.com/gitlab-org/gitlab/-/issues/7006) | -| Python ([Pipenv](https://pipenv.pypa.io/en/latest/)) | `Pipfile.lock` | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | [GitLab#11756](https://gitlab.com/gitlab-org/gitlab/-/issues/11756) | +| Package Managers | Languages | Supported files | Scan tools | +| ------------------- | --------- | --------------- | ------------ | +| [Pipenv](https://pipenv.pypa.io/en/latest/) | Python | `Pipfile.lock` | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | [GitLab#11756](https://gitlab.com/gitlab-org/gitlab/-/issues/11756) | +| [Poetry](https://python-poetry.org/) | Python | `poetry.lock` | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | [GitLab#7006](https://gitlab.com/gitlab-org/gitlab/-/issues/7006) | +| [sbt](https://www.scala-sbt.org/) 1.3+ ([Coursier](https://get-coursier.io/))| Scala | `build.sbt` | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | [GitLab#249526](https://gitlab.com/gitlab-org/gitlab/-/issues/249526) | ## Contribute your scanner @@ -81,7 +83,7 @@ The [Security Scanner Integration](../../../development/integrations/secure.md) ## Configuration -To enable Dependency Scanning for GitLab 11.9 and later, you must +To enable dependency scanning for GitLab 11.9 and later, you must [include](../../../ci/yaml/README.md#includetemplate) the [`Dependency-Scanning.gitlab-ci.yml` template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml) that is provided as a part of your GitLab installation. @@ -95,16 +97,16 @@ include: - template: Dependency-Scanning.gitlab-ci.yml ``` -The included template creates Dependency Scanning jobs in your CI/CD +The included template creates dependency scanning jobs in your CI/CD pipeline and scans your project's source code for possible vulnerabilities. The results are saved as a -[Dependency Scanning report artifact](../../../ci/pipelines/job_artifacts.md#artifactsreportsdependency_scanning) +[dependency scanning report artifact](../../../ci/pipelines/job_artifacts.md#artifactsreportsdependency_scanning) that you can later download and analyze. Due to implementation limitations, we -always take the latest Dependency Scanning artifact available. +always take the latest dependency scanning artifact available. -### Customizing the Dependency Scanning settings +### Customizing the dependency scanning settings -The Dependency Scanning settings can be changed through [environment variables](#available-variables) by using the +The dependency scanning settings can be changed through [environment variables](#available-variables) by using the [`variables`](../../../ci/yaml/README.md#variables) parameter in `.gitlab-ci.yml`. For example: @@ -119,7 +121,7 @@ variables: Because template is [evaluated before](../../../ci/yaml/README.md#include) the pipeline configuration, the last mention of the variable takes precedence. -### Overriding Dependency Scanning jobs +### Overriding dependency scanning jobs CAUTION: **Deprecation:** Beginning in GitLab 13.0, the use of [`only` and `except`](../../../ci/yaml/README.md#onlyexcept-basic) @@ -141,10 +143,10 @@ gemnasium-dependency_scanning: ### Available variables -Dependency Scanning can be [configured](#customizing-the-dependency-scanning-settings) +Dependency scanning can be [configured](#customizing-the-dependency-scanning-settings) using environment variables. -#### Configuring Dependency Scanning +#### Configuring dependency scanning The following variables allow configuration of global dependency scanning settings. @@ -156,7 +158,7 @@ The following variables allow configuration of global dependency scanning settin | `DS_EXCLUDED_PATHS` | Exclude vulnerabilities from output based on the paths. A comma-separated list of patterns. Patterns can be globs, or file or folder paths (for example, `doc,spec`). Parent directories also match patterns. Default: `"spec, test, tests, tmp"` | | `SECURE_LOG_LEVEL` | Set the minimum logging level. Messages of this logging level or higher are output. From highest to lowest severity, the logging levels are: `fatal`, `error`, `warn`, `info`, `debug`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/10880) in GitLab 13.1. Default: `info` | -#### Configuring specific analyzers used by Dependency Scanning +#### Configuring specific analyzers used by dependency scanning The following variables are used for configuring specific analyzers (used for a specific language/framework). @@ -176,7 +178,7 @@ The following variables are used for configuring specific analyzers (used for a | `MAVEN_CLI_OPTS` | `gemnasium-maven` | `"-DskipTests --batch-mode"` | List of command line arguments that are passed to `maven` by the analyzer. See an example for [using private repositories](../index.md#using-private-maven-repos). | | `GRADLE_CLI_OPTS` | `gemnasium-maven` | | List of command line arguments that are passed to `gradle` by the analyzer. | | `SBT_CLI_OPTS` | `gemnasium-maven` | | List of command-line arguments that the analyzer passes to `sbt`. | -| `BUNDLER_AUDIT_UPDATE_DISABLED` | `bundler-audit` | `"false"` | Disable automatic updates for the `bundler-audit` analyzer. Useful if you're running Dependency Scanning in an offline, air-gapped environment.| +| `BUNDLER_AUDIT_UPDATE_DISABLED` | `bundler-audit` | `"false"` | Disable automatic updates for the `bundler-audit` analyzer. Useful if you're running dependency scanning in an offline, air-gapped environment.| | `BUNDLER_AUDIT_ADVISORY_DB_URL` | `bundler-audit` | `https://github.com/rubysec/ruby-advisory-db` | URL of the advisory database used by bundler-audit. | | `BUNDLER_AUDIT_ADVISORY_DB_REF_NAME` | `bundler-audit` | `master` | Git ref for the advisory database specified by `BUNDLER_AUDIT_ADVISORY_DB_URL`. | | `RETIREJS_JS_ADVISORY_DB` | `retire.js` | `https://raw.githubusercontent.com/RetireJS/retire.js/master/repository/jsrepository.json` | Path or URL to `retire.js` JS vulnerability data file. Note that if the URL hosting the data file uses a custom SSL certificate, for example in an offline installation, you can pass the certificate in the `ADDITIONAL_CA_CERT_BUNDLE` environment variable. | @@ -214,16 +216,16 @@ For more information about the vulnerabilities database update, check the ## Dependency List -An additional benefit of Dependency Scanning is the ability to view your +An additional benefit of dependency scanning is the ability to view your project's dependencies and their known vulnerabilities. Read more about the [Dependency List](../dependency_list/index.md). ## Reports JSON format -The Dependency Scanning tool emits a JSON report file. For more information, see the +The dependency scanning tool emits a JSON report file. For more information, see the [schema for this report](https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/blob/master/dist/dependency-scanning-report-format.json). -Here's an example Dependency Scanning report: +Here's an example dependency scanning report: ```json-doc { @@ -342,36 +344,35 @@ You can search the [gemnasium-db](https://gitlab.com/gitlab-org/security-product to find a vulnerability in the Gemnasium database. You can also [submit new vulnerabilities](https://gitlab.com/gitlab-org/security-products/gemnasium-db/blob/master/CONTRIBUTING.md). -## Running Dependency Scanning in an offline environment +## Running dependency scanning in an offline environment For self-managed GitLab instances in an environment with limited, restricted, or intermittent access -to external resources through the internet, some adjustments are required for Dependency Scanning +to external resources through the internet, some adjustments are required for dependency scanning jobs to run successfully. For more information, see [Offline environments](../offline_deployments/index.md). -### Requirements for offline Dependency Scanning +### Requirements for offline dependency scanning -Here are the requirements for using Dependency Scanning in an offline environment: +Here are the requirements for using dependency scanning in an offline environment: - GitLab Runner with the [`docker` or `kubernetes` executor](#requirements). -- Docker Container Registry with locally available copies of Dependency Scanning [analyzer](https://gitlab.com/gitlab-org/security-products/analyzers) images. +- Docker Container Registry with locally available copies of dependency scanning [analyzer](https://gitlab.com/gitlab-org/security-products/analyzers) images. - Host an offline Git copy of the [gemnasium-db advisory database](https://gitlab.com/gitlab-org/security-products/gemnasium-db/). This is required because, in an offline environment, the Gemnasium analyzer can't fetch the latest advisories from the online repository. - _Only if scanning Ruby projects_: Host an offline Git copy of the [advisory database](https://github.com/rubysec/ruby-advisory-db). - _Only if scanning npm/yarn projects_: Host an offline copy of the [retire.js](https://github.com/RetireJS/retire.js/) [node](https://github.com/RetireJS/retire.js/blob/master/repository/npmrepository.json) and [js](https://github.com/RetireJS/retire.js/blob/master/repository/jsrepository.json) advisory databases. -NOTE: **Note:** -GitLab Runner has a [default `pull policy` of `always`](https://docs.gitlab.com/runner/executors/docker.html#using-the-always-pull-policy), +Note that GitLab Runner has a [default `pull policy` of `always`](https://docs.gitlab.com/runner/executors/docker.html#using-the-always-pull-policy), meaning the runner tries to pull Docker images from the GitLab container registry even if a local copy is available. The GitLab Runner [`pull_policy` can be set to `if-not-present`](https://docs.gitlab.com/runner/executors/docker.html#using-the-if-not-present-pull-policy) in an offline environment if you prefer using only locally available Docker images. However, we recommend keeping the pull policy setting to `always` if not in an offline environment, as this enables the use of updated scanners in your CI/CD pipelines. -### Make GitLab Dependency Scanning analyzer images available inside your Docker registry +### Make GitLab dependency scanning analyzer images available inside your Docker registry -For Dependency Scanning with all [supported languages and frameworks](#supported-languages-and-package-managers), -import the following default Dependency Scanning analyzer images from `registry.gitlab.com` into +For dependency scanning with all [supported languages and frameworks](#supported-languages-and-package-managers), +import the following default dependency scanning analyzer images from `registry.gitlab.com` into your [local Docker container registry](../../packages/container_registry/index.md): ```plaintext @@ -392,7 +393,19 @@ For details on saving and transporting Docker images as a file, see Docker's doc [`docker save`](https://docs.docker.com/engine/reference/commandline/save/), [`docker load`](https://docs.docker.com/engine/reference/commandline/load/), [`docker export`](https://docs.docker.com/engine/reference/commandline/export/), and [`docker import`](https://docs.docker.com/engine/reference/commandline/import/). -### Set Dependency Scanning CI job variables to use local Dependency Scanning analyzers +#### Support for Custom Certificate Authorities + +Support for custom certificate authorities was introduced in the following versions. + +| Analyzer | Version | +| -------- | ------- | +| `gemnasium` | [v2.8.0](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium/-/releases/v2.8.0) | +| `gemnasium-maven` | [v2.9.0](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven/-/releases/v2.9.0) | +| `gemnasium-python` | [v2.7.0](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-python/-/releases/v2.7.0) | +| `retire.js` | [v2.4.0](https://gitlab.com/gitlab-org/security-products/analyzers/retire.js/-/releases/v2.4.0) | +| `bundler-audit` | [v2.4.0](https://gitlab.com/gitlab-org/security-products/analyzers/bundler-audit/-/releases/v2.4.0) | + +### Set dependency scanning CI job variables to use local dependency scanning analyzers Add the following configuration to your `.gitlab-ci.yml` file. You must change the value of `SECURE_ANALYZERS_PREFIX` to refer to your local Docker container registry. You must also change the @@ -479,7 +492,19 @@ As a workaround, remove the [`retire.js`](analyzers.md#selecting-specific-analyz ### `Error response from daemon: error processing tar file: docker-tar: relocation error` -This error occurs when the Docker version that runs the Dependency Scanning job is `19.03.00`. +This error occurs when the Docker version that runs the dependency scanning job is `19.03.00`. Consider updating to Docker `19.03.1` or greater. Older versions are not affected. Read more in [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/13830#note_211354992 "Current SAST container fails"). + +### Getting warning message `gl-dependency-scanning-report.json: no matching files` + +For information on this, see the [general Application Security troubleshooting section](../../../ci/pipelines/job_artifacts.md#error-message-no-files-to-upload). + +### Limitation when using rules:exists + +The [dependency scanning CI template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml) +uses the [`rules:exists`](../../../ci/yaml/README.md#rulesexists) +syntax. This directive is limited to 10000 checks and always returns `true` after reaching this +number. Because of this, and depending on the number of files in your repository, a dependency +scanning job might be triggered even if the scanner doesn't support your project. diff --git a/doc/user/application_security/img/cve_request_communication.png b/doc/user/application_security/img/cve_request_communication.png Binary files differindex 0766b371c11..5c58df463ef 100644 --- a/doc/user/application_security/img/cve_request_communication.png +++ b/doc/user/application_security/img/cve_request_communication.png diff --git a/doc/user/application_security/img/cve_request_communication_publication.png b/doc/user/application_security/img/cve_request_communication_publication.png Binary files differindex 9e34c217e13..9eb6f2f8d9f 100644 --- a/doc/user/application_security/img/cve_request_communication_publication.png +++ b/doc/user/application_security/img/cve_request_communication_publication.png diff --git a/doc/user/application_security/img/new_cve_request_issue.png b/doc/user/application_security/img/new_cve_request_issue.png Binary files differindex a342c73992e..6ea7ca4a2ab 100644 --- a/doc/user/application_security/img/new_cve_request_issue.png +++ b/doc/user/application_security/img/new_cve_request_issue.png diff --git a/doc/user/application_security/img/unconfigured_security_approval_rules_and_enabled_jobs_v13_4.png b/doc/user/application_security/img/unconfigured_security_approval_rules_and_enabled_jobs_v13_4.png Binary files differindex f497b0fbc4e..7b04988afdb 100644 --- a/doc/user/application_security/img/unconfigured_security_approval_rules_and_enabled_jobs_v13_4.png +++ b/doc/user/application_security/img/unconfigured_security_approval_rules_and_enabled_jobs_v13_4.png diff --git a/doc/user/application_security/img/unconfigured_security_approval_rules_and_jobs_v13_4.png b/doc/user/application_security/img/unconfigured_security_approval_rules_and_jobs_v13_4.png Binary files differindex fc847b578f5..b9b6dd13294 100644 --- a/doc/user/application_security/img/unconfigured_security_approval_rules_and_jobs_v13_4.png +++ b/doc/user/application_security/img/unconfigured_security_approval_rules_and_jobs_v13_4.png diff --git a/doc/user/application_security/img/vulnerability-check_v13_4.png b/doc/user/application_security/img/vulnerability-check_v13_4.png Binary files differindex e0b53059b45..3e38f6eebe7 100644 --- a/doc/user/application_security/img/vulnerability-check_v13_4.png +++ b/doc/user/application_security/img/vulnerability-check_v13_4.png diff --git a/doc/user/application_security/img/vulnerability_solution.png b/doc/user/application_security/img/vulnerability_solution.png Binary files differindex d86b89a5f99..63e9c1473b6 100644 --- a/doc/user/application_security/img/vulnerability_solution.png +++ b/doc/user/application_security/img/vulnerability_solution.png diff --git a/doc/user/application_security/index.md b/doc/user/application_security/index.md index d509176f2b2..413a9f894e2 100644 --- a/doc/user/application_security/index.md +++ b/doc/user/application_security/index.md @@ -22,10 +22,10 @@ Testing (SAST), and Secret Detection by adding the following to your `.gitlab-ci ```yaml include: - - template: Dependency-Scanning.gitlab-ci.yml - - template: License-Scanning.gitlab-ci.yml - - template: SAST.gitlab-ci.yml - - template: Secret-Detection.gitlab-ci.yml + - template: Security/Dependency-Scanning.gitlab-ci.yml + - template: Security/License-Scanning.gitlab-ci.yml + - template: Security/SAST.gitlab-ci.yml + - template: Security/Secret-Detection.gitlab-ci.yml ``` To add Dynamic Application Security Testing (DAST) scanning, add the following to your @@ -33,7 +33,7 @@ To add Dynamic Application Security Testing (DAST) scanning, add the following t ```yaml include: - - template: DAST.gitlab-ci.yml + - template: Security/DAST.gitlab-ci.yml variables: DAST_WEBSITE: https://staging.example.com @@ -449,7 +449,7 @@ To fix this issue, you can either: ```yaml include: - template: SAST.gitlab-ci.yml + template: Security/SAST.gitlab-ci.yml spotbugs-sast: stage: unit-tests @@ -458,6 +458,15 @@ To fix this issue, you can either: [Learn more on overriding SAST jobs](sast/index.md#overriding-sast-jobs). All the security scanning tools define their stage, so this error can occur with all of them. +### Getting warning messages `… report.json: no matching files` + +This is often followed by the [error `No files to upload`](../../ci/pipelines/job_artifacts.md#error-message-no-files-to-upload), +and preceded by other errors or warnings that indicate why the JSON report wasn't generated. Please +check the entire job log for such messages. If you don't find these messages, retry the failed job +after setting `SECURE_LOG_LEVEL: "debug"` as a +[custom environment variable](../../ci/variables/README.md#custom-environment-variables). +This provides useful information to investigate further. + ### Getting error message `sast job: config key may not be used with 'rules': only/except` When [including](../../ci/yaml/README.md#includetemplate) a `.gitlab-ci.yml` template @@ -490,7 +499,7 @@ would look similar to: ```yaml include: - - template: SAST.gitlab-ci.yml + - template: Security/SAST.gitlab-ci.yml # Ensure that the scanning is only executed on master or merge requests spotbugs-sast: @@ -505,7 +514,7 @@ would be written as follows: ```yaml include: - - template: SAST.gitlab-ci.yml + - template: Security/SAST.gitlab-ci.yml # Ensure that the scanning is only executed on master or merge requests spotbugs-sast: @@ -519,7 +528,7 @@ it would look similar to: ```yaml include: - - template: SAST.gitlab-ci.yml + - template: Security/SAST.gitlab-ci.yml # Ensure that the scanning is not executed on tags spotbugs-sast: @@ -531,7 +540,7 @@ To transition to the new `rules` syntax, the override would be rewritten as: ```yaml include: - - template: SAST.gitlab-ci.yml + - template: Security/SAST.gitlab-ci.yml # Ensure that the scanning is not executed on tags spotbugs-sast: diff --git a/doc/user/application_security/sast/analyzers.md b/doc/user/application_security/sast/analyzers.md index 727f077aa09..6167c0445f9 100644 --- a/doc/user/application_security/sast/analyzers.md +++ b/doc/user/application_security/sast/analyzers.md @@ -25,6 +25,7 @@ SAST supports the following official analyzers: - [`flawfinder`](https://gitlab.com/gitlab-org/security-products/analyzers/flawfinder) (Flawfinder) - [`gosec`](https://gitlab.com/gitlab-org/security-products/analyzers/gosec) (Gosec) - [`kubesec`](https://gitlab.com/gitlab-org/security-products/analyzers/kubesec) (Kubesec) +- [`mobsf`](https://gitlab.com/gitlab-org/security-products/analyzers/mobsf) (MobSF (beta)) - [`nodejs-scan`](https://gitlab.com/gitlab-org/security-products/analyzers/nodejs-scan) (NodeJsScan) - [`phpcs-security-audit`](https://gitlab.com/gitlab-org/security-products/analyzers/phpcs-security-audit) (PHP CS security-audit) - [`pmd-apex`](https://gitlab.com/gitlab-org/security-products/analyzers/pmd-apex) (PMD (Apex only)) @@ -53,7 +54,7 @@ In `.gitlab-ci.yml` define: ```yaml include: - - template: SAST.gitlab-ci.yml + - template: Security/SAST.gitlab-ci.yml variables: SECURE_ANALYZERS_PREFIX: my-docker-registry/gl-images @@ -70,7 +71,7 @@ In `.gitlab-ci.yml` define: ```yaml include: - - template: SAST.gitlab-ci.yml + - template: Security/SAST.gitlab-ci.yml variables: SAST_DEFAULT_ANALYZERS: "bandit,flawfinder" @@ -86,7 +87,7 @@ default analyzers. In `.gitlab-ci.yml` define: ```yaml include: - - template: SAST.gitlab-ci.yml + - template: Security/SAST.gitlab-ci.yml variables: SAST_DEFAULT_ANALYZERS: "" @@ -118,24 +119,24 @@ The [Security Scanner Integration](../../../development/integrations/secure.md) ## Analyzers Data -| Property / Tool | Apex | Bandit | Brakeman | ESLint security | SpotBugs | Flawfinder | Gosec | Kubesec Scanner | NodeJsScan | PHP CS Security Audit | Security code Scan (.NET) | Sobelow | -| --------------------------------------- | :------------------: | :------------------: | :------------------: | :------------------: | :------------------: | :------------------: | :------------------: | :------------------: | :------------------: | :---------------------: | :-------------------------: | :----------------: | -| Severity | ✓ | ✓ | 𐄂 | 𐄂 | ✓ | ✓ | ✓ | ✓ | 𐄂 | ✓ | 𐄂 | 𐄂 | -| Title | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | -| Description | ✓ | 𐄂 | 𐄂 | ✓ | ✓ | 𐄂 | 𐄂 | ✓ | ✓ | 𐄂 | 𐄂 | ✓ | -| File | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | -| Start line | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | 𐄂 | ✓ | ✓ | ✓ | ✓ | -| End line | ✓ | ✓ | 𐄂 | ✓ | ✓ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | -| Start column | ✓ | 𐄂 | 𐄂 | ✓ | ✓ | ✓ | ✓ | 𐄂 | 𐄂 | ✓ | ✓ | 𐄂 | -| End column | ✓ | 𐄂 | 𐄂 | ✓ | ✓ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | -| External ID (e.g. CVE) | 𐄂 | 𐄂 | ⚠ | 𐄂 | ⚠ | ✓ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | -| URLs | ✓ | 𐄂 | ✓ | 𐄂 | ⚠ | 𐄂 | ⚠ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | -| Internal doc/explanation | ✓ | ⚠ | ✓ | 𐄂 | ✓ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | ✓ | -| Solution | ✓ | 𐄂 | 𐄂 | 𐄂 | ⚠ | ✓ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | -| Affected item (e.g. class or package) | ✓ | 𐄂 | ✓ | 𐄂 | ✓ | ✓ | 𐄂 | ✓ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | -| Confidence | 𐄂 | ✓ | ✓ | 𐄂 | ✓ | x | ✓ | ✓ | 𐄂 | 𐄂 | 𐄂 | ✓ | -| Source code extract | 𐄂 | ✓ | ✓ | ✓ | 𐄂 | ✓ | ✓ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | -| Internal ID | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | 𐄂 | 𐄂 | ✓ | ✓ | ✓ | +| Property / Tool | Apex | Bandit | Brakeman | ESLint security | SpotBugs | Flawfinder | Gosec | Kubesec Scanner | MobSF | NodeJsScan | PHP CS Security Audit | Security code Scan (.NET) | Sobelow | +| --------------------------------------- | :------------------: | :------------------: | :------------------: | :------------------: | :------------------: | :------------------: | :------------------: | :------------------: | :------------------: | :------------------: | :---------------------: | :-------------------------: | :----------------: | +| Severity | ✓ | ✓ | 𐄂 | 𐄂 | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | 𐄂 | 𐄂 | +| Title | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| Description | ✓ | 𐄂 | 𐄂 | ✓ | ✓ | 𐄂 | 𐄂 | ✓ | ✓ | ✓ | 𐄂 | 𐄂 | ✓ | +| File | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| Start line | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | 𐄂 | ✓ | ✓ | ✓ | ✓ | ✓ | +| End line | ✓ | ✓ | 𐄂 | ✓ | ✓ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | +| Start column | ✓ | 𐄂 | 𐄂 | ✓ | ✓ | ✓ | ✓ | 𐄂 | 𐄂 | 𐄂 | ✓ | ✓ | 𐄂 | +| End column | ✓ | 𐄂 | 𐄂 | ✓ | ✓ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | +| External ID (e.g. CVE) | 𐄂 | 𐄂 | ⚠ | 𐄂 | ⚠ | ✓ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | +| URLs | ✓ | 𐄂 | ✓ | 𐄂 | ⚠ | 𐄂 | ⚠ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | +| Internal doc/explanation | ✓ | ⚠ | ✓ | 𐄂 | ✓ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | ✓ | +| Solution | ✓ | 𐄂 | 𐄂 | 𐄂 | ⚠ | ✓ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | +| Affected item (e.g. class or package) | ✓ | 𐄂 | ✓ | 𐄂 | ✓ | ✓ | 𐄂 | ✓ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | +| Confidence | 𐄂 | ✓ | ✓ | 𐄂 | ✓ | x | ✓ | ✓ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | ✓ | +| Source code extract | 𐄂 | ✓ | ✓ | ✓ | 𐄂 | ✓ | ✓ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | +| Internal ID | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | 𐄂 | 𐄂 | 𐄂 | ✓ | ✓ | ✓ | - ✓ => we have that data - ⚠ => we have that data but it's partially reliable, or we need to extract it from unstructured content diff --git a/doc/user/application_security/sast/index.md b/doc/user/application_security/sast/index.md index a4fc3c9e638..9e4d4112ae8 100644 --- a/doc/user/application_security/sast/index.md +++ b/doc/user/application_security/sast/index.md @@ -11,8 +11,8 @@ type: reference, howto NOTE: **Note:** The whitepaper ["A Seismic Shift in Application Security"](https://about.gitlab.com/resources/whitepaper-seismic-shift-application-security/) -explains how **4 of the top 6 attacks were application based**. Download it -to learn how to protect your organization. +explains how 4 of the top 6 attacks were application based. Download it to learn how to protect your +organization. If you're using [GitLab CI/CD](../../../ci/README.md), you can analyze your source code for known vulnerabilities using Static Application Security Testing (SAST). GitLab checks the SAST report and @@ -31,8 +31,10 @@ The results are sorted by the priority of the vulnerability: 1. Unknown 1. Everything else -NOTE: **Note:** -A pipeline consists of multiple jobs, including SAST and DAST scanning. If any job fails to finish for any reason, the security dashboard doesn't show SAST scanner output. For example, if the SAST job finishes but the DAST job fails, the security dashboard doesn't show SAST results. The analyzer outputs an [exit code](../../../development/integrations/secure.md#exit-code) on failure. +A pipeline consists of multiple jobs, including SAST and DAST scanning. If any job fails to finish +for any reason, the security dashboard doesn't show SAST scanner output. For example, if the SAST +job finishes but the DAST job fails, the security dashboard doesn't show SAST results. On failure, +the analyzer outputs an [exit code](../../../development/integrations/secure.md#exit-code). ## Use cases @@ -59,39 +61,41 @@ is **not** `19.03.0`. See [troubleshooting information](#error-response-from-dae GitLab SAST supports a variety of languages, package managers, and frameworks. Our SAST security scanners also feature automatic language detection which works even for mixed-language projects. If any supported language is detected in project source code we will automatically run the appropriate SAST analyzers. -You can also [view our language roadmap](https://about.gitlab.com/direction/secure/static-analysis/sast/#language-support) and [request other language support by opening an issue](https://gitlab.com/groups/gitlab-org/-/epics/297). +You can also [view our language roadmap](https://about.gitlab.com/direction/secure/static-analysis/sast/#language-support) and [request other language support by opening an issue](https://gitlab.com/groups/gitlab-org/-/epics/297). | Language (package managers) / framework | Scan tool | Introduced in GitLab Version | |--------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| .NET Core | [Security Code Scan](https://security-code-scan.github.io) | 11.0, [moved](https://gitlab.com/groups/gitlab-org/-/epics/2098) to [GitLab Core](https://about.gitlab.com/pricing/) in 13.3 | -| .NET Framework | [Security Code Scan](https://security-code-scan.github.io) | 13.0, [moved](https://gitlab.com/groups/gitlab-org/-/epics/2098) to [GitLab Core](https://about.gitlab.com/pricing/) in 13.3 | -| Apex (Salesforce) | [PMD](https://pmd.github.io/pmd/index.html) | 12.1, [moved](https://gitlab.com/groups/gitlab-org/-/epics/2098) to [GitLab Core](https://about.gitlab.com/pricing/) in 13.3 | -| C/C++ | [Flawfinder](https://github.com/david-a-wheeler/flawfinder) | 10.7, [moved](https://gitlab.com/groups/gitlab-org/-/epics/2098) to [GitLab Core](https://about.gitlab.com/pricing/) in 13.3 | -| Elixir (Phoenix) | [Sobelow](https://github.com/nccgroup/sobelow) | 11.10, [moved](https://gitlab.com/groups/gitlab-org/-/epics/2098) to [GitLab Core](https://about.gitlab.com/pricing/) in 13.3 | -| Go | [Gosec](https://github.com/securego/gosec) | 10.7, [moved](https://gitlab.com/groups/gitlab-org/-/epics/2098) to [GitLab Core](https://about.gitlab.com/pricing/) in 13.3 | -| Groovy ([Ant](https://ant.apache.org/), [Gradle](https://gradle.org/), [Maven](https://maven.apache.org/) and [SBT](https://www.scala-sbt.org/)) | [SpotBugs](https://spotbugs.github.io/) with the [find-sec-bugs](https://find-sec-bugs.github.io/) plugin | 11.3 (Gradle) & 11.9 (Ant, Maven, SBT), [moved](https://gitlab.com/groups/gitlab-org/-/epics/2098) to [GitLab Core](https://about.gitlab.com/pricing/) in 13.3 | -| Helm Charts | [Kubesec](https://github.com/controlplaneio/kubesec) | 13.1, [moved](https://gitlab.com/groups/gitlab-org/-/epics/2098) to [GitLab Core](https://about.gitlab.com/pricing/) in 13.3 | -| Java ([Ant](https://ant.apache.org/), [Gradle](https://gradle.org/), [Maven](https://maven.apache.org/) and [SBT](https://www.scala-sbt.org/)) | [SpotBugs](https://spotbugs.github.io/) with the [find-sec-bugs](https://find-sec-bugs.github.io/) plugin | 10.6 (Maven), 10.8 (Gradle) & 11.9 (Ant, SBT), [moved](https://gitlab.com/groups/gitlab-org/-/epics/2098) to [GitLab Core](https://about.gitlab.com/pricing/) in 13.3 | -| JavaScript | [ESLint security plugin](https://github.com/nodesecurity/eslint-plugin-security) | 11.8, [moved](https://gitlab.com/groups/gitlab-org/-/epics/2098) to [GitLab Core](https://about.gitlab.com/pricing/) in 13.2 | -| Kubernetes manifests | [Kubesec](https://github.com/controlplaneio/kubesec) | 12.6, [moved](https://gitlab.com/groups/gitlab-org/-/epics/2098) to [GitLab Core](https://about.gitlab.com/pricing/) in 13.3 | -| Node.js | [NodeJsScan](https://github.com/ajinabraham/NodeJsScan) | 11.1, [moved](https://gitlab.com/groups/gitlab-org/-/epics/2098) to [GitLab Core](https://about.gitlab.com/pricing/) in 13.3 | -| PHP | [phpcs-security-audit](https://github.com/FloeDesignTechnologies/phpcs-security-audit) | 10.8, [moved](https://gitlab.com/groups/gitlab-org/-/epics/2098) to [GitLab Core](https://about.gitlab.com/pricing/) in 13.3 | -| Python ([pip](https://pip.pypa.io/en/stable/)) | [bandit](https://github.com/PyCQA/bandit) | 10.3, [moved](https://gitlab.com/groups/gitlab-org/-/epics/2098) to [GitLab Core](https://about.gitlab.com/pricing/) in 13.3 | -| React | [ESLint react plugin](https://github.com/yannickcr/eslint-plugin-react) | 12.5, [moved](https://gitlab.com/groups/gitlab-org/-/epics/2098) to [GitLab Core](https://about.gitlab.com/pricing/) in 13.2 | -| Ruby on Rails | [brakeman](https://brakemanscanner.org) | 10.3, [moved](https://gitlab.com/groups/gitlab-org/-/epics/2098) to [GitLab Core](https://about.gitlab.com/pricing/) in 13.1 | -| Scala ([Ant](https://ant.apache.org/), [Gradle](https://gradle.org/), [Maven](https://maven.apache.org/) and [SBT](https://www.scala-sbt.org/)) | [SpotBugs](https://spotbugs.github.io/) with the [find-sec-bugs](https://find-sec-bugs.github.io/) plugin | 11.0 (SBT) & 11.9 (Ant, Gradle, Maven), [moved](https://gitlab.com/groups/gitlab-org/-/epics/2098) to [GitLab Core](https://about.gitlab.com/pricing/) in 13.3 | +| .NET Core | [Security Code Scan](https://security-code-scan.github.io) | 11.0 | +| .NET Framework | [Security Code Scan](https://security-code-scan.github.io) | 13.0 | +| Apex (Salesforce) | [PMD](https://pmd.github.io/pmd/index.html) | 12.1 | +| C/C++ | [Flawfinder](https://github.com/david-a-wheeler/flawfinder) | 10.7 | +| Elixir (Phoenix) | [Sobelow](https://github.com/nccgroup/sobelow) | 11.1 | +| Go | [Gosec](https://github.com/securego/gosec) | 10.7 | +| Groovy ([Ant](https://ant.apache.org/), [Gradle](https://gradle.org/), [Maven](https://maven.apache.org/) and [SBT](https://www.scala-sbt.org/)) | [SpotBugs](https://spotbugs.github.io/) with the [find-sec-bugs](https://find-sec-bugs.github.io/) plugin | 11.3 (Gradle) & 11.9 (Ant, Maven, SBT) | +| Helm Charts | [Kubesec](https://github.com/controlplaneio/kubesec) | 13.1 | +| Java ([Ant](https://ant.apache.org/), [Gradle](https://gradle.org/), [Maven](https://maven.apache.org/) and [SBT](https://www.scala-sbt.org/)) | [SpotBugs](https://spotbugs.github.io/) with the [find-sec-bugs](https://find-sec-bugs.github.io/) plugin | 10.6 (Maven), 10.8 (Gradle) & 11.9 (Ant, SBT) | +| Java (Android) | [MobSF (beta)](https://github.com/MobSF/Mobile-Security-Framework-MobSF) | 13.5 | +| JavaScript | [ESLint security plugin](https://github.com/nodesecurity/eslint-plugin-security) | 11.8 | +| Kotlin (Android) | [MobSF (beta)](https://github.com/MobSF/Mobile-Security-Framework-MobSF) | 13.5 | +| Kubernetes manifests | [Kubesec](https://github.com/controlplaneio/kubesec) | 12.6 | +| Node.js | [NodeJsScan](https://github.com/ajinabraham/NodeJsScan) | 11.1 | +| Objective-C (iOS) | [MobSF (beta)](https://github.com/MobSF/Mobile-Security-Framework-MobSF) | 13.5 | +| PHP | [phpcs-security-audit](https://github.com/FloeDesignTechnologies/phpcs-security-audit) | 10.8 | +| Python ([pip](https://pip.pypa.io/en/stable/)) | [bandit](https://github.com/PyCQA/bandit) | 10.3 | +| React | [ESLint react plugin](https://github.com/yannickcr/eslint-plugin-react) | 12.5 | +| Ruby on Rails | [brakeman](https://brakemanscanner.org) | 10.3 | +| Scala ([Ant](https://ant.apache.org/), [Gradle](https://gradle.org/), [Maven](https://maven.apache.org/) and [SBT](https://www.scala-sbt.org/)) | [SpotBugs](https://spotbugs.github.io/) with the [find-sec-bugs](https://find-sec-bugs.github.io/) plugin | 11.0 (SBT) & 11.9 (Ant, Gradle, Maven) | +| Swift (iOS) | [MobSF (beta)](https://github.com/MobSF/Mobile-Security-Framework-MobSF) | 13.5 | | TypeScript | [ESLint security plugin](https://github.com/nodesecurity/eslint-plugin-security) | 11.9, [merged](https://gitlab.com/gitlab-org/gitlab/-/issues/36059) with ESLint in 13.2 | -NOTE: **Note:** -The Java analyzers can also be used for variants like the +Note that the Java analyzers can also be used for variants like the [Gradle wrapper](https://docs.gradle.org/current/userguide/gradle_wrapper.html), -[Grails](https://grails.org/) and the [Maven wrapper](https://github.com/takari/maven-wrapper). +[Grails](https://grails.org/), +and the [Maven wrapper](https://github.com/takari/maven-wrapper). ### Making SAST analyzers available to all GitLab tiers -All open source (OSS) analyzers have been moved to the GitLab Core tier. Progress can be -tracked in the corresponding -[epic](https://gitlab.com/groups/gitlab-org/-/epics/2098). +All open source (OSS) analyzers have been moved to the GitLab Core tier as of GitLab 13.3. #### Summary of features per tier @@ -147,16 +151,19 @@ always take the latest SAST artifact available. > - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3659) in GitLab Ultimate 13.3. > - [Improved](https://gitlab.com/gitlab-org/gitlab/-/issues/232862) in GitLab Ultimate 13.4. +> - [Improved](https://gitlab.com/groups/gitlab-org/-/epics/3635) in GitLab Ultimate 13.5. You can enable and configure SAST with a basic configuration using the **SAST Configuration** page: 1. From the project's home page, go to **Security & Compliance** > **Configuration** in the left sidebar. -1. If the project does not have a `gitlab-ci.yml` file, click **Enable** in the Static Application Security Testing (SAST) row, otherwise click **Configure**. -1. Enter the custom SAST values, then click **Create Merge Request**. +1. If the project does not have a `.gitlab-ci.yml` file, click **Enable** in the Static Application Security Testing (SAST) row, otherwise click **Configure**. +1. Enter the custom SAST values. Custom values are stored in the `.gitlab-ci.yml` file. For variables not in the SAST Configuration page, their values are left unchanged. Default values are inherited from the GitLab SAST template. +1. Optionally, expand the **SAST analyzers** section, select individual [SAST analyzers](./analyzers.md) and enter custom analyzer values. +1. Click **Create Merge Request**. 1. Review and merge the merge request. ### Customizing the SAST settings @@ -169,7 +176,7 @@ set the `SAST_GOSEC_LEVEL` variable to `2`: ```yaml include: - - template: SAST.gitlab-ci.yml + - template: Security/SAST.gitlab-ci.yml variables: SAST_GOSEC_LEVEL: 2 @@ -191,13 +198,78 @@ inclusion and specify any additional keys under it. For example, this enables `F ```yaml include: - - template: SAST.gitlab-ci.yml + - template: Security/SAST.gitlab-ci.yml spotbugs-sast: variables: FAIL_NEVER: 1 ``` +### Custom rulesets + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/235382) in GitLab 13.5. + +You can customize the default scanning rules provided with SAST's NodeJS-Scan and Gosec analyzers. +Customization allows you to exclude rules and modify the behavior of existing rules. + +To customize the default scanning rules, create a file containing custom rules. These rules +are passed through to the analyzer's underlying scanner tool. + +To create a custom ruleset: + +1. Create a `.gitlab` directory at the root of your project, if one doesn't already exist. +1. Create a custom ruleset file named `sast-ruleset.toml` in the `.gitlab` directory. +1. In the `sast-ruleset.toml` file, do one of the following: + + - Define a custom analyzer configuration. In this example, customized rules are defined for the + `nodejs-scan` scanner: + + ```toml + [nodejs-scan] + description = 'custom ruleset for nodejs-scan' + + [[nodejs-scan.passthrough]] + type = "raw" + value = ''' + - nodejs-extensions: + - .js + + template-extensions: + - .new + - .hbs + - '' + + ignore-filenames: + - skip.js + + ignore-paths: + - __MACOSX + - skip_dir + - node_modules + + ignore-extensions: + - .hbs + + ignore-rules: + - regex_injection_dos + - pug_jade_template + - express_xss + + ''' + ``` + + - Provide the name of the file containing a custom analyzer configuration. In this example, + customized rules for the `gosec` scanner are contained in the file `gosec-config.json`: + + ```toml + [gosec] + description = 'custom ruleset for gosec' + + [[gosec.passthrough]] + type = "file" + value = "gosec-config.json" + ``` + ### Using environment variables to pass credentials for private repositories Some analyzers require downloading the project's dependencies in order to @@ -222,7 +294,7 @@ Kubesec analyzer. In `.gitlab-ci.yml`, define: ```yaml include: - - template: SAST.gitlab-ci.yml + - template: Security/SAST.gitlab-ci.yml variables: SCAN_KUBERNETES_MANIFESTS: "true" @@ -248,7 +320,7 @@ stages: - test include: - - template: SAST.gitlab-ci.yml + - template: Security/SAST.gitlab-ci.yml build: stage: build @@ -264,17 +336,16 @@ spotbugs-sast: - build variables: MAVEN_REPO_PATH: ./.m2/repository - COMPILE: false + COMPILE: "false" artifacts: reports: sast: gl-sast-report.json ``` -NOTE: **Note:** -The path to the vendored directory must be specified explicitly to allow -the analyzer to recognize the compiled artifacts. This configuration can vary per -analyzer but in the case of Java above, `MAVEN_REPO_PATH` can be used. -See [Analyzer settings](#analyzer-settings) for the complete list of available options. +To allow the analyzer to recognize the compiled artifacts, you must explicitly specify the path to +the vendored directory. This configuration can vary per analyzer but in the case of Java above, you +can use `MAVEN_REPO_PATH`. See +[Analyzer settings](#analyzer-settings) for the complete list of available options. ### Available variables @@ -358,13 +429,29 @@ CAUTION: **Caution:** Variables having names starting with these prefixes will **not** be propagated to the SAST Docker container and/or analyzer containers: `DOCKER_`, `CI`, `GITLAB_`, `FF_`, `HOME`, `PWD`, `OLDPWD`, `PATH`, `SHLVL`, `HOSTNAME`. +### Experimental features + +Receive early access to experimental features. + +Currently, this will enable scanning of iOS and Android apps via the [MobSF analyzer](https://gitlab.com/gitlab-org/security-products/analyzers/mobsf/). + +To enable experimental features, add the following to your `.gitlab-ci.yml` file: + +```yaml +include: + - template: Security/SAST.gitlab-ci.yml + +variables: + SAST_EXPERIMENTAL_FEATURES: "true" +``` + ## Reports JSON format The SAST tool emits a JSON report file. For more information, see the [schema for this report](https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/blob/master/dist/sast-report-format.json). -The JSON report file can be downloaded from the CI pipelines page, for more -information see [Downloading artifacts](../../../ci/pipelines/job_artifacts.md). +The JSON report file can be downloaded from the CI pipelines page, or the +pipelines tab on merge requests. For more information see [Downloading artifacts](../../../ci/pipelines/job_artifacts.md). Here's an example SAST report: @@ -480,7 +567,6 @@ To use SAST in an offline environment, you need: - A Docker Container Registry with locally available copies of SAST [analyzer](https://gitlab.com/gitlab-org/security-products/analyzers) images. - Configure certificate checking of packages (optional). -NOTE: **Note:** GitLab Runner has a [default `pull policy` of `always`](https://docs.gitlab.com/runner/executors/docker.html#using-the-always-pull-policy), meaning the runner tries to pull Docker images from the GitLab container registry even if a local copy is available. The GitLab Runner [`pull_policy` can be set to `if-not-present`](https://docs.gitlab.com/runner/executors/docker.html#using-the-if-not-present-pull-policy) @@ -518,6 +604,25 @@ For details on saving and transporting Docker images as a file, see Docker's doc [`docker save`](https://docs.docker.com/engine/reference/commandline/save/), [`docker load`](https://docs.docker.com/engine/reference/commandline/load/), [`docker export`](https://docs.docker.com/engine/reference/commandline/export/), and [`docker import`](https://docs.docker.com/engine/reference/commandline/import/). +#### If support for Custom Certificate Authorities are needed + +Support for custom certificate authorities was introduced in the following versions. + +| Analyzer | Version | +| -------- | ------- | +| `bandit` | [v2.3.0](https://gitlab.com/gitlab-org/security-products/analyzers/bandit/-/releases/v2.3.0) | +| `brakeman` | [v2.1.0](https://gitlab.com/gitlab-org/security-products/analyzers/brakeman/-/releases/v2.1.0) | +| `eslint` | [v2.9.2](https://gitlab.com/gitlab-org/security-products/analyzers/eslint/-/releases/v2.9.2) | +| `flawfinder` | [v2.3.0](https://gitlab.com/gitlab-org/security-products/analyzers/flawfinder/-/releases/v2.3.0) | +| `gosec` | [v2.5.0](https://gitlab.com/gitlab-org/security-products/analyzers/gosec/-/releases/v2.5.0) | +| `kubesec` | [v2.1.0](https://gitlab.com/gitlab-org/security-products/analyzers/kubesec/-/releases/v2.1.0) | +| `nodejs-scan` | [v2.9.5](https://gitlab.com/gitlab-org/security-products/analyzers/nodejs-scan/-/releases/v2.9.5) | +| `phpcs-security-audit` | [v2.8.2](https://gitlab.com/gitlab-org/security-products/analyzers/phpcs-security-audit/-/releases/v2.8.2) | +| `pmd-apex` | [v2.1.0](https://gitlab.com/gitlab-org/security-products/analyzers/pmd-apex/-/releases/v2.1.0) | +| `security-code-scan` | [v2.7.3](https://gitlab.com/gitlab-org/security-products/analyzers/security-code-scan/-/releases/v2.7.3) | +| `sobelow` | [v2.2.0](https://gitlab.com/gitlab-org/security-products/analyzers/sobelow/-/releases/v2.2.0) | +| `spotbugs` | [v2.7.1](https://gitlab.com/gitlab-org/security-products/analyzers/spotbugs/-/releases/v2.7.1) | + ### Set SAST CI job variables to use local SAST analyzers Add the following configuration to your `.gitlab-ci.yml` file. You must replace @@ -525,7 +630,7 @@ Add the following configuration to your `.gitlab-ci.yml` file. You must replace ```yaml include: - - template: SAST.gitlab-ci.yml + - template: Security/SAST.gitlab-ci.yml variables: SECURE_ANALYZERS_PREFIX: "localhost:5000/analyzers" @@ -549,3 +654,16 @@ This error occurs when the Docker version that runs the SAST job is `19.03.0`. Consider updating to Docker `19.03.1` or greater. Older versions are not affected. Read more in [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/13830#note_211354992 "Current SAST container fails"). + +### Getting warning message `gl-sast-report.json: no matching files` + +For information on this, see the [general Application Security troubleshooting section](../../../ci/pipelines/job_artifacts.md#error-message-no-files-to-upload). + +### Limitation when using rules:exists + +The [SAST CI template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml) +uses the `rules:exists` parameter. For performance reasons, a maximum number of matches are made +against the given glob pattern. If the number of matches exceeds the maximum, the `rules:exists` +parameter returns `true`. Depending on the number of files in your repository, a SAST job might be +triggered even if the scanner doesn't support your project. For more details about this issue, see +the [`rules:exists` documentation](../../../ci/yaml/README.md#rulesexists). diff --git a/doc/user/application_security/secret_detection/index.md b/doc/user/application_security/secret_detection/index.md index f3e411cdc16..bb10e9d7315 100644 --- a/doc/user/application_security/secret_detection/index.md +++ b/doc/user/application_security/secret_detection/index.md @@ -9,8 +9,6 @@ info: To determine the technical writer assigned to the Stage/Group associated w > [Introduced](https://about.gitlab.com/releases/2019/03/22/gitlab-11-9-released/#detect-secrets-and-credentials-in-the-repository) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.9. -## Overview - A recurring problem when developing applications is that developers may unintentionally commit secrets and credentials to their remote repositories. If other people have access to the source, or if the project is public, the sensitive information is then exposed and can be leveraged by @@ -40,7 +38,7 @@ To run Secret Detection jobs, by default, you need GitLab Runner with the If you're using the shared runners on GitLab.com, this is enabled by default. CAUTION: **Caution:** -Our Secret Detection jobs currently expect a Linux container type. Windows containers are not yet supported. +Our Secret Detection jobs expect a Linux container type. Windows containers are not supported. CAUTION: **Caution:** If you use your own runners, make sure the Docker version installed @@ -67,26 +65,27 @@ as shown in the following table: ## Configuration -NOTE: **Note:** -With GitLab 13.1 Secret Detection was split into its own CI/CD template. +> GitLab 13.1 splits Secret Detection from the [SAST configuration](../sast#configuration) into its own CI/CD template. If you're using GitLab 13.0 or earlier and SAST is enabled, then Secret Detection is already enabled. Secret Detection is performed by a [specific analyzer](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml) -during the `secret-detection` job. It runs regardless of the programming -language of your app. +during the `secret-detection` job. It runs regardless of your app's programming language. -The Secret Detection analyzer includes [Gitleaks](https://github.com/zricethezav/gitleaks) and [TruffleHog](https://github.com/dxa4481/truffleHog) checks. +The Secret Detection analyzer includes [Gitleaks](https://github.com/zricethezav/gitleaks) and +[TruffleHog](https://github.com/dxa4481/truffleHog) checks. -NOTE: **Note:** -The Secret Detection analyzer will ignore "Password in URL" vulnerabilities if the password begins -with a dollar sign (`$`) as this likely indicates the password being used is an environment -variable. For example, `https://username:$password@example.com/path/to/repo` won't be -detected, whereas `https://username:password@example.com/path/to/repo` would be detected. +Note that the Secret Detection analyzer ignores Password-in-URL vulnerabilities if the password +begins with a dollar sign (`$`), as this likely indicates the password is an environment variable. +For example, `https://username:$password@example.com/path/to/repo` isn't detected, while +`https://username:password@example.com/path/to/repo` is. NOTE: **Note:** -You don't have to configure Secret Detection manually as shown in this section if you're using [Auto Secret Detection](../../../topics/autodevops/stages.md#auto-secret-detection) +You don't have to configure Secret Detection manually as shown in this section if you're using +[Auto Secret Detection](../../../topics/autodevops/stages.md#auto-secret-detection) provided by [Auto DevOps](../../../topics/autodevops/index.md). -To enable Secret Detection for GitLab 13.1 and later, you must include the `Secret-Detection.gitlab-ci.yml` template that’s provided as a part of your GitLab installation. For GitLab versions earlier than 11.9, you can copy and use the job as defined in that template. +To enable Secret Detection for GitLab 13.1 and later, you must include the +`Secret-Detection.gitlab-ci.yml` template that's provided as a part of your GitLab installation. For +GitLab versions earlier than 11.9, you can copy and use the job as defined in that template. Add the following to your `.gitlab-ci.yml` file: @@ -103,30 +102,6 @@ The results are saved as a that you can later download and analyze. Due to implementation limitations, we always take the latest Secret Detection artifact available. -### Using the SAST Template - -Prior to GitLab 13.1, Secret Detection was part of [SAST configuration](../sast#configuration). -If you already have SAST enabled for your app configured before GitLab 13.1, -you don't need to manually configure it. - -CAUTION: **Planned Deprecation:** -In a future GitLab release, configuring Secret Detection with the SAST template will be deprecated. Please begin using `Secret-Detection.gitlab-ci.yml` -to prevent future issues. We have made a -[video to guide you through the process of transitioning](https://www.youtube.com/watch?v=W2tjcQreDwQ) -to this new template. - -<div class="video-fallback"> - See the video: <a href="https://www.youtube.com/watch?v=W2tjcQreDwQ">Walkthrough of historical secret scan</a>. -</div> -<figure class="video-container"> - <iframe src="https://www.youtube.com/embed/W2tjcQreDwQ" frameborder="0" allowfullscreen="true"> </iframe> -</figure> - -When using the SAST template, Secret Detection is performed by a [specific analyzer](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml#L180) -during the `sast` job. It runs regardless of the programming -language of your app, and you don't need to change your -CI/CD configuration file to enable it. Results are available in the SAST report. - ### Customizing settings The Secret Detection scan settings can be changed through [environment variables](#available-variables) @@ -164,9 +139,52 @@ Secret Detection can be customized by defining available variables: |-------------------------|---------------|-------------| | `SECRET_DETECTION_COMMIT_FROM` | - | The commit a Gitleaks scan starts at. | | `SECRET_DETECTION_COMMIT_TO` | - | The commit a Gitleaks scan ends at. | -| `SECRET_DETECTION_EXCLUDED_PATHS` | "" | Exclude vulnerabilities from output based on the paths. This is a comma-separated list of patterns. Patterns can be globs, or file or folder paths (for example, `doc,spec` ). Parent directories will also match patterns. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/225273) in GitLab 13.3. | +| `SECRET_DETECTION_EXCLUDED_PATHS` | "" | Exclude vulnerabilities from output based on the paths. This is a comma-separated list of patterns. Patterns can be globs, or file or folder paths (for example, `doc,spec` ). Parent directories also match patterns. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/225273) in GitLab 13.3. | | `SECRET_DETECTION_HISTORIC_SCAN` | false | Flag to enable a historic Gitleaks scan. | +### Custom rulesets + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/211387) in GitLab 13.5. + +You can customize the default secret detection rules provided with GitLab. +Customization allows you to exclude rules and add new rules. + +To create a custom ruleset: + +1. Create a `.gitlab` directory at the root of your project, if one doesn't already exist. +1. Create a custom ruleset file named `secret-detection-ruleset.toml` in the `.gitlab` directory. +1. In the `secret-detection-ruleset.toml` file, do one of the following: + + - Define a custom ruleset: + + ```toml + [secrets] + description = 'secrets custom rules configuration' + + [[secrets.passthrough]] + type = "raw" + target = "gitleaks.toml" + value = """\ + title = "gitleaks config" + # add regexes to the regex table + [[rules]] + description = "Test for Raw Custom Rulesets" + regex = '''Custom Raw Ruleset T[est]{3}''' + """ + ``` + + - Provide the name of the file containing a custom ruleset: + + ```toml + [secrets] + description = 'secrets custom rules configuration' + + [[secrets.passthrough]] + type = "file" + target = "gitleaks.toml" + value = "config/gitleaks.toml" + ``` + ### Logging level To control the verbosity of logs set the `SECURE_LOG_LEVEL` environment variable. Messages of this logging level or higher are output. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/10880) in GitLab 13.1. @@ -197,3 +215,35 @@ We have created a [short video walkthrough](https://youtu.be/wDtc_K00Y0A) showca <figure class="video-container"> <iframe src="https://www.youtube.com/embed/wDtc_K00Y0A" frameborder="0" allowfullscreen="true"> </iframe> </figure> + +### Make GitLab Secret Detection analyzer image available inside your Docker registry + +Import the following default Secret Detection analyzer images from `registry.gitlab.com` into your +[local Docker container registry](../../packages/container_registry/index.md): + +```plaintext +registry.gitlab.com/gitlab-org/security-products/analyzers/secrets:3 +``` + +The process for importing Docker images into a local offline Docker registry depends on +**your network security policy**. Please consult your IT staff to find an accepted and approved +process by which external resources can be imported or temporarily accessed. Note that these scanners are [updated periodically](../index.md#maintenance-and-update-of-the-vulnerabilities-database) +with new definitions, so consider if you're able to make periodic updates yourself. + +For details on saving and transporting Docker images as a file, see Docker's documentation on +[`docker save`](https://docs.docker.com/engine/reference/commandline/save/), [`docker load`](https://docs.docker.com/engine/reference/commandline/load/), +[`docker export`](https://docs.docker.com/engine/reference/commandline/export/), and [`docker import`](https://docs.docker.com/engine/reference/commandline/import/). + +#### If support for Custom Certificate Authorities are needed + +Support for custom certificate authorities was introduced in the following versions. + +| Analyzer | Version | +| -------- | ------- | +| secrets | [v3.0.0](https://gitlab.com/gitlab-org/security-products/analyzers/secrets/-/releases/v3.0.0) | + +## Troubleshooting + +### Getting warning message `gl-secret-detection-report.json: no matching files` + +For information on this, see the [general Application Security troubleshooting section](../../../ci/pipelines/job_artifacts.md#error-message-no-files-to-upload). diff --git a/doc/user/application_security/security_dashboard/img/group_vulnerability_report_v13_4.png b/doc/user/application_security/security_dashboard/img/group_vulnerability_report_v13_4.png Binary files differindex 67a7bb5f368..0310ef3ea0a 100644 --- a/doc/user/application_security/security_dashboard/img/group_vulnerability_report_v13_4.png +++ b/doc/user/application_security/security_dashboard/img/group_vulnerability_report_v13_4.png diff --git a/doc/user/application_security/security_dashboard/img/instance_security_center_settings_v13_4.png b/doc/user/application_security/security_dashboard/img/instance_security_center_settings_v13_4.png Binary files differnew file mode 100644 index 00000000000..4223494c294 --- /dev/null +++ b/doc/user/application_security/security_dashboard/img/instance_security_center_settings_v13_4.png diff --git a/doc/user/application_security/security_dashboard/img/instance_security_dashboard_empty_v13_4.png b/doc/user/application_security/security_dashboard/img/instance_security_dashboard_empty_v13_4.png Binary files differindex 3c618090be8..5edceb32e5c 100644 --- a/doc/user/application_security/security_dashboard/img/instance_security_dashboard_empty_v13_4.png +++ b/doc/user/application_security/security_dashboard/img/instance_security_dashboard_empty_v13_4.png diff --git a/doc/user/application_security/security_dashboard/img/instance_security_dashboard_v13_4.png b/doc/user/application_security/security_dashboard/img/instance_security_dashboard_v13_4.png Binary files differindex d010adcc90c..5379b5c6e5d 100644 --- a/doc/user/application_security/security_dashboard/img/instance_security_dashboard_v13_4.png +++ b/doc/user/application_security/security_dashboard/img/instance_security_dashboard_v13_4.png diff --git a/doc/user/application_security/security_dashboard/img/project_security_dashboard_dismissal_v13_4.png b/doc/user/application_security/security_dashboard/img/project_security_dashboard_dismissal_v13_4.png Binary files differnew file mode 100644 index 00000000000..eb1dfe6c6f4 --- /dev/null +++ b/doc/user/application_security/security_dashboard/img/project_security_dashboard_dismissal_v13_4.png diff --git a/doc/user/application_security/security_dashboard/img/project_security_dashboard_v13_0.png b/doc/user/application_security/security_dashboard/img/project_security_dashboard_v13_0.png Binary files differdeleted file mode 100644 index 878bb83c2a2..00000000000 --- a/doc/user/application_security/security_dashboard/img/project_security_dashboard_v13_0.png +++ /dev/null diff --git a/doc/user/application_security/security_dashboard/img/project_security_dashboard_v13_2.png b/doc/user/application_security/security_dashboard/img/project_security_dashboard_v13_2.png Binary files differdeleted file mode 100644 index 7cab7b0a61f..00000000000 --- a/doc/user/application_security/security_dashboard/img/project_security_dashboard_v13_2.png +++ /dev/null diff --git a/doc/user/application_security/security_dashboard/img/project_security_dashboard_v13_3.png b/doc/user/application_security/security_dashboard/img/project_security_dashboard_v13_3.png Binary files differdeleted file mode 100644 index 34c64f830ba..00000000000 --- a/doc/user/application_security/security_dashboard/img/project_security_dashboard_v13_3.png +++ /dev/null diff --git a/doc/user/application_security/security_dashboard/img/project_security_dashboard_v13_5.png b/doc/user/application_security/security_dashboard/img/project_security_dashboard_v13_5.png Binary files differnew file mode 100644 index 00000000000..c46a8295a53 --- /dev/null +++ b/doc/user/application_security/security_dashboard/img/project_security_dashboard_v13_5.png diff --git a/doc/user/application_security/security_dashboard/img/vulnerability_list_table_v13_4.png b/doc/user/application_security/security_dashboard/img/vulnerability_list_table_v13_4.png Binary files differindex eb91cfc47ad..760942c3239 100644 --- a/doc/user/application_security/security_dashboard/img/vulnerability_list_table_v13_4.png +++ b/doc/user/application_security/security_dashboard/img/vulnerability_list_table_v13_4.png diff --git a/doc/user/application_security/security_dashboard/index.md b/doc/user/application_security/security_dashboard/index.md index 51d9b4f45cd..5fa8ebb80e0 100644 --- a/doc/user/application_security/security_dashboard/index.md +++ b/doc/user/application_security/security_dashboard/index.md @@ -5,21 +5,26 @@ group: Threat Insights info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# GitLab Security Dashboard **(ULTIMATE)** +# GitLab Security Dashboard, Security Center, and Vulnerability Reports **(ULTIMATE)** -The Security Dashboard is a good place to get an overview of all the security -vulnerabilities in your groups, projects, and pipelines. +GitLab provides a comprehensive set of features for viewing and managing vulnerabilities: + +- Security dashboards: An overview of the security status in your instance, groups, and projects. +- Vulnerability reports: Detailed lists of all vulnerabilities for the instance, group, project, or + pipeline. This is where you triage and manage vulnerabilities. +- Security Center: A dedicated area for vulnerability management at the instance level. This + includes a security dashboard, vulnerability report, and settings. You can also drill down into a vulnerability and get extra information. This includes the project it comes from, any related file(s), and metadata that helps you analyze the risk it poses. You can also dismiss a vulnerability or create an issue for it. -To benefit from the Security Dashboard you must first configure one of the +To benefit from these features, you must first configure one of the [security scanners](../index.md). ## Supported reports -The Security Dashboard displays vulnerabilities detected by scanners such as: +The vulnerability report displays vulnerabilities detected by scanners such as: - [Container Scanning](../container_scanning/index.md) - [Dynamic Application Security Testing](../dast/index.md) @@ -29,7 +34,7 @@ The Security Dashboard displays vulnerabilities detected by scanners such as: ## Requirements -To use the instance, group, project, or pipeline security dashboard: +To use the security dashboards and vulnerability reports: 1. At least one project inside a group must be configured with at least one of the [supported reports](#supported-reports). @@ -41,15 +46,19 @@ To use the instance, group, project, or pipeline security dashboard: > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/13496) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.3. -At the pipeline level, the Security section displays the vulnerabilities present in the branch of the project the pipeline was run against. +At the pipeline level, the Security section displays the vulnerabilities present in the branch of +the project the pipeline ran against.  Visit the page for any pipeline that ran any of the [supported reports](#supported-reports). To view the pipeline's security findings, select the **Security** tab when viewing the pipeline. -NOTE: **Note:** -A pipeline consists of multiple jobs, including SAST and DAST scanning. If any job fails to finish for any reason, the security dashboard will not show SAST scanner output. For example, if the SAST job finishes but the DAST job fails, the security dashboard will not show SAST results. The analyzer will output an [exit code](../../../development/integrations/secure.md#exit-code) on failure. +A pipeline consists of multiple jobs, including SAST and DAST scanning. If any job fails to finish +for any reason, the security dashboard doesn't show SAST scanner output. For example, if the SAST +job finishes but the DAST job fails, the security dashboard doesn't show SAST results. On failure, +the analyzer outputs an +[exit code](../../../development/integrations/secure.md#exit-code). ## Project Security Dashboard @@ -60,12 +69,15 @@ At the project level, the Security Dashboard displays the vulnerabilities merged to **Security & Compliance > Security Dashboard**. By default, the Security Dashboard displays all detected and confirmed vulnerabilities. -The Security Dashboard first displays the total number of vulnerabilities by severity (for example, +The Security Dashboard first displays the time at which the last pipeline completed on the project's +default branch. There's also a link to view this in more detail. + +The Security Dashboard next displays the total number of vulnerabilities by severity (for example, Critical, High, Medium, Low, Info, Unknown). Below this, a table shows each vulnerability's status, severity, and description. Clicking a vulnerability takes you to its [Vulnerability Details](../vulnerabilities) page to view more information about that vulnerability. - + You can filter the vulnerabilities by one or more of the following: @@ -78,7 +90,7 @@ You can also dismiss vulnerabilities in the table: 1. Select the checkbox for each vulnerability you want to dismiss. 1. In the menu that appears, select the reason for dismissal and click **Dismiss Selected**. - + ## Group Security Dashboard @@ -86,79 +98,99 @@ You can also dismiss vulnerabilities in the table: The group Security Dashboard gives an overview of the vulnerabilities in the default branches of the projects in a group and its subgroups. Access it by navigating to **Security > Security Dashboard** -for your group. By default, the Security Dashboard displays all detected and confirmed -vulnerabilities. +after selecting your group. By default, the Security Dashboard displays all detected and confirmed +vulnerabilities. If you don't see the vulnerabilities over time graph, the likely cause is that you +have not selected a group. -NOTE: **Note:** -The Security Dashboard only shows projects with [security reports](#supported-reports) enabled in a -group. +Note that the Security Dashboard only shows projects with +[security reports](#supported-reports) +enabled in a group.  There is a timeline chart that shows how many open -vulnerabilities your projects had at various points in time. You can filter among 30, 60, and -90 days, with the default being 90. Hover over the chart to get more details about -the open vulnerabilities at a specific time. +vulnerabilities your projects had at various points in time. You can display the vulnerability +trends over a 30, 60, or 90-day time frame (the default is 90 days). Hover over the chart to get +more details about the open vulnerabilities at a specific time. Next to the timeline chart is a list of projects, grouped and sorted by the severity of the vulnerability found: -- F: 1 or more "critical" -- D: 1 or more "high" or "unknown" -- C: 1 or more "medium" -- B: 1 or more "low" -- A: 0 vulnerabilities +- F: One or more "critical" +- D: One or more "high" or "unknown" +- C: One or more "medium" +- B: One or more "low" +- A: Zero vulnerabilities Projects with no vulnerability tests configured will not appear in the list. Additionally, dismissed -vulnerabilities are not included either. +vulnerabilities are excluded. -Navigate to the group's [Vulnerability Report](#vulnerability-list) to view the vulnerabilities found. +Navigate to the group's [vulnerability report](#vulnerability-report) to view the vulnerabilities found. -## Instance Security Dashboard +## Instance Security Center -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/6953) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.8. +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3426) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.4. -At the instance level, the Security Dashboard displays the vulnerabilities present in the default -branches of all the projects you configure to display on the dashboard. It includes all the -[group Security Dashboard's](#group-security-dashboard) -features. +The Security Center is where you manage vulnerabilities for your instance. It displays the +vulnerabilities present in the default branches of all the projects you configure. It includes the +following: + +- The [group security dashboard's](#group-security-dashboard) features. +- A [vulnerability report](#vulnerability-report). +- A dedicated settings area to configure which projects to display.  -You can access the Instance Security Dashboard from the menu +You can access the Instance Security Center from the menu bar at the top of the page. Under **More**, select **Security**. - + -The dashboard is empty before you add projects to it. +The dashboard and vulnerability report are empty before you add projects. - + -### Adding projects to the dashboard +### Adding projects to the Security Center -To add projects to the dashboard: +To add projects to the Security Center: 1. Click **Settings** in the left navigation bar or click the **Add projects** button. 1. Search for and add one or more projects using the **Search your projects** field. 1. Click the **Add projects** button. -After you add projects, the Security Dashboard displays the vulnerabilities found in those projects' -default branches. + + +After you add projects, the security dashboard and vulnerability report display the vulnerabilities +found in those projects' default branches. ## Export vulnerabilities > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/213014) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.10. -You can export all your vulnerabilities in CSV format by clicking the **{upload}** **Export** -button located at top right of the **Security Dashboard**. After the report -is built, the CSV report downloads to your local machine. The report contains all -vulnerabilities for the projects defined in the **Security Dashboard**, -as filters don't apply to the export function. - - +You can export all your vulnerabilities in CSV (comma separated values) format by clicking the +**{upload}** **Export** button located at top right of the Security Dashboard. When the report is +ready, the CSV report downloads to your local machine. The report contains all vulnerabilities for +the projects defined in the Security Dashboard, as filters don't apply to the export function. NOTE: **Note:** It may take several minutes for the download to start if your project contains -thousands of vulnerabilities. Do not close the page until the download finishes. +thousands of vulnerabilities. Don't close the page until the download finishes. + +The fields in the export include: + +- Group Name +- Project Name +- Scanner Type +- Scanner Name +- Status +- Vulnerability +- Details +- Additional Info +- Severity +- [CVE](https://cve.mitre.org/) (Common Vulnerabilities and Exposures) +- [CWE](https://cwe.mitre.org/) (Common Weakness Enumeration) +- Other Identifiers + + ## Keeping the dashboards up to date @@ -191,14 +223,14 @@ When using [Auto DevOps](../../../topics/autodevops/index.md), use [special environment variables](../../../topics/autodevops/customize.md#environment-variables) to configure daily security scans. -## Vulnerability list +## Vulnerability report -Each dashboard's vulnerability list contains vulnerabilities from the latest scans that were merged +Each vulnerability report contains vulnerabilities from the latest scans that were merged into the default branch.  -You can filter which vulnerabilities the Security Dashboard displays by: +You can filter which vulnerabilities the vulnerability report displays by: - Status - Severity @@ -211,8 +243,10 @@ To create an issue associated with the vulnerability, click the **Create Issue**  -Once you create the issue, the vulnerability list contains a link to the issue and an icon whose -color indicates the issue's status (green for open issues, blue for closed issues). +Once you create the issue, the linked issue icon in the vulnerability list: + +- Indicates that an issue has been created for that vulnerability. +- Shows a tooltip that contains a link to the issue.  diff --git a/doc/user/application_security/terminology/index.md b/doc/user/application_security/terminology/index.md index 8006a49ba35..f975de213ef 100644 --- a/doc/user/application_security/terminology/index.md +++ b/doc/user/application_security/terminology/index.md @@ -13,7 +13,6 @@ This terminology list for GitLab Secure and Defend aims to: - Improve the effectiveness of communication regarding GitLab's application security features. - Get new contributors up to speed faster. -NOTE: **Note:** This document defines application security terms in the specific context of GitLab's Secure and Defend products. Terms may therefore have different meanings outside of GitLab Secure and Defend. diff --git a/doc/user/application_security/threat_monitoring/index.md b/doc/user/application_security/threat_monitoring/index.md index 5414800b290..391666a077e 100644 --- a/doc/user/application_security/threat_monitoring/index.md +++ b/doc/user/application_security/threat_monitoring/index.md @@ -105,11 +105,10 @@ disabled state. Once enabled,a predefined policy deploys to the selected environment's deployment platform and you can manage it like the regular policies. -NOTE: **Note:** -If you're using [Auto DevOps](../../../topics/autodevops/index.md) and -change a policy in this section, your `auto-deploy-values.yaml` file -doesn't update. Auto DevOps users must make changes by following -the [Container Network Policy documentation](../../../topics/autodevops/stages.md#network-policy). +Note that if you're using [Auto DevOps](../../../topics/autodevops/index.md) +and change a policy in this section, your `auto-deploy-values.yaml` file doesn't update. Auto DevOps +users must make changes by following the +[Container Network Policy documentation](../../../topics/autodevops/stages.md#network-policy). ### Changing enforcement status @@ -119,12 +118,9 @@ To change a network policy's enforcement status: - Click the **Enforcement status** toggle to update the selected policy. - Click the **Apply changes** button to deploy network policy changes. -NOTE: **Note:** -Disabled network policies have the -`network-policy.gitlab.com/disabled_by: gitlab` selector inside the -`podSelector` block. This narrows the scope of such a policy and as a -result it doesn't affect any pods. The policy itself is still deployed -to the corresponding deployment namespace. +Disabled network policies have the `network-policy.gitlab.com/disabled_by: gitlab` selector inside +the `podSelector` block. This narrows the scope of such a policy and as a result it doesn't affect +any pods. The policy itself is still deployed to the corresponding deployment namespace. ### Container Network Policy editor @@ -135,10 +131,8 @@ create a new policy click the **New policy** button located in the **Policy** tab's header. To edit an existing policy, click**Edit policy** in the selected policy drawer. -NOTE: **Note:** -The policy editor only supports the -[CiliumNetworkPolicy](https://docs.cilium.io/en/v1.8/policy/)specification. Regular -Kubernetes +Note that the policy editor only supports the +[CiliumNetworkPolicy](https://docs.cilium.io/en/v1.8/policy/)specification. Regular Kubernetes [NetworkPolicy](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#networkpolicy-v1-networking-k8s-io) resources aren't supported. diff --git a/doc/user/application_security/vulnerabilities/index.md b/doc/user/application_security/vulnerabilities/index.md index ff383fdf553..ee3fd6c4dd4 100644 --- a/doc/user/application_security/vulnerabilities/index.md +++ b/doc/user/application_security/vulnerabilities/index.md @@ -14,6 +14,7 @@ Each security vulnerability in a project's [Security Dashboard](../security_dash - Details of the vulnerability. - The status of the vulnerability within the project. - Available actions for the vulnerability. +- Issues related to the vulnerability. On the vulnerability page, you can interact with the vulnerability in several different ways: @@ -23,6 +24,7 @@ several different ways: - [Create issue](#creating-an-issue-for-a-vulnerability) - Create a new issue with the title and description pre-populated with information from the vulnerability report. By default, such issues are [confidential](../../project/issues/confidential_issues.md). +- [Link issues](#link-issues-to-the-vulnerability) - Link existing issues to vulnerability. - [Solution](#automatic-remediation-for-vulnerabilities) - For some vulnerabilities, a solution is provided for how to fix the vulnerability. @@ -38,6 +40,9 @@ the following values: | Dismissed | A user has seen this vulnerability and dismissed it | | Resolved | The vulnerability has been fixed and is no longer in the codebase | +A timeline shows you when the vulnerability status has changed, +and allows you to comment on a change. + ## Creating an issue for a vulnerability You can create an issue for a vulnerability by selecting the **Create issue** button. @@ -47,6 +52,12 @@ project the vulnerability came from, and pre-populates it with useful informatio the vulnerability report. After the issue is created, GitLab redirects you to the issue page so you can edit, assign, or comment on the issue. +## Link issues to the vulnerability + +You can link one or more existing issues to the vulnerability. This allows you to +indicate that this vulnerability affects multiple issues. It also allows you to indicate +that the resolution of one issue would resolve multiple vulnerabilities. + ## Automatic remediation for vulnerabilities You can fix some vulnerabilities by applying the solution that GitLab automatically diff --git a/doc/user/clusters/agent/index.md b/doc/user/clusters/agent/index.md index 6521b71e9e6..196c3e9fb43 100644 --- a/doc/user/clusters/agent/index.md +++ b/doc/user/clusters/agent/index.md @@ -6,18 +6,23 @@ info: To determine the technical writer assigned to the Stage/Group associated w # GitLab Kubernetes Agent **(PREMIUM ONLY)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/223061) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.4. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/223061) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.4. +> - It's disabled on GitLab.com. Rolling this feature out to GitLab.com is [planned](https://gitlab.com/groups/gitlab-org/-/epics/3834). -## Goals +CAUTION: **Warning:** +This feature might not be available to you. Check the **version history** note above for details. -The [GitLab Kubernetes Agent](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent) is an active in-cluster component for solving GitLab and Kubernetes integration tasks in a secure and cloud native way. +The [GitLab Kubernetes Agent](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent) +is an active in-cluster component for solving GitLab and Kubernetes integration +tasks in a secure and cloud-native way. It enables: -Features: +- Integrating GitLab with a Kubernetes cluster behind a firewall or NAT + (network address translation). +- Pull-based GitOps deployments by leveraging the + [GitOps Engine](https://github.com/argoproj/gitops-engine). +- Real-time access to API endpoints within a cluster. -1. Makes it possible to integrate GitLab with a Kubernetes cluster behind a firewall or NAT -1. Enables pull-based GitOps deployments by leveraging the [GitOps Engine](https://github.com/argoproj/gitops-engine) -1. Allows for real-time access to API endpoints within a cluster. -1. Many more features are planned. Please [review our roadmap](https://gitlab.com/groups/gitlab-org/-/epics/3329). +Many more features are planned. Please [review our roadmap](https://gitlab.com/groups/gitlab-org/-/epics/3329). ## Architecture @@ -39,34 +44,52 @@ sequenceDiagram end ``` -Please refer to our [full architecture documentation in the Agent project](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/blob/master/doc/architecture.md#high-level-architecture). +Please refer to our [full architecture documentation](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/blob/master/doc/architecture.md#high-level-architecture) +in the Agent project. -## Getting started with GitOps using the GitLab Agent and the GitLab Cloud Native Helm chart +## Get started with GitOps and the GitLab Agent There are several components that work in concert for the Agent to accomplish GitOps deployments: -1. A Kubernetes cluster that is properly configured -1. A configuration repository that contains a `config.yaml` file. This `config.yaml` tells the Agent which repositories to synchronize with. -1. A manifest repository that contains a `manifest.yaml`. This `manifest.yaml` (which can be autogenerated) is tracked by the Agent and any changes to the file are automatically applied to the cluster. +- A properly-configured Kubernetes cluster. +- A configuration repository that contains a `config.yaml` file, which tells the + Agent which repositories to synchronize with. +- A manifest repository that contains a `manifest.yaml`, which is tracked by the + Agent and can be auto-generated. Any changes to `manifest.yaml` are applied to the cluster. -The setup process involves a few steps that, once completed, will enable GitOps deployments to work +The setup process involves a few steps to enable GitOps deployments: -1. Installing the Agent server via GitLab Helm chart -1. Defining a configuration directory -1. Creating an Agent record in GitLab -1. Generating and copying a Secret token used to connect to the Agent -1. Installing the Agent into the cluster -1. Creating a `manifest.yaml` +1. Installing the Agent server. +1. Defining a configuration directory. +1. Creating an Agent record in GitLab. +1. Generating and copying a Secret token used to connect to the Agent. +1. Installing the Agent into the cluster. +1. Creating a `manifest.yaml`. -### Installing the Agent server via Helm +### Install the Agent server -Currently the GitLab Kubernetes Agent can only be deployed via our [Helm chart](https://gitlab.com/gitlab-org/charts/gitlab). +The GitLab Kubernetes Agent can be deployed using [Omnibus +GitLab](https://docs.gitlab.com/omnibus/) or the [GitLab +chart](https://gitlab.com/gitlab-org/charts/gitlab). If you don't already have +GitLab installed, please refer to our [installation +documentation](https://docs.gitlab.com/ee/install/README.html). -NOTE: We are working quickly to [include the Agent in Official Linux Package](https://gitlab.com/gitlab-org/gitlab/-/issues/223060). +NOTE: **Note:** +GitLab plans to include the Agent on [GitLab.com](https://gitlab.com/groups/gitlab-org/-/epics/3834). -If you don't already have GitLab installed via Helm please refer to our [installation documentation](https://docs.gitlab.com/charts/installation/) +When using the [Omnibus GitLab](https://docs.gitlab.com/omnibus/) package: -When installing/upgrading the GitLab Helm chart please consider the following Helm 2 example (if using Helm 3 please modify): +1. Edit `/etc/gitlab/gitlab.rb`: + +```plaintext +gitlab_kas['enable'] = true +``` + +1. [Reconfigure GitLab](../../../administration/restart_gitlab.md#omnibus-gitlab-reconfigure). + +When installing or upgrading the GitLab Helm chart, consider the following Helm 2 example. +(If you're using Helm 3, you must modify this example.) You must set `global.kas.enabled=true` +for the Agent to be properly installed and configured: ```shell helm repo update @@ -79,15 +102,14 @@ helm upgrade --force --install gitlab gitlab/gitlab \ --set global.kas.enabled=true ``` -`global.kas.enabled=true` must be set in order for the Agent to be properly installed and configured. - -### Defining a configuration repository +### Define a configuration repository -Next you will need a GitLab repository that will contain your Agent configuration. +Next, you need a GitLab repository to contain your Agent configuration. The minimal +repository layout looks like this: -The minimal repository layout looks like this: - -`.gitlab/agents/<agent-name>/config.yaml` +```plaintext +.gitlab/agents/<agent-name>/config.yaml +``` The `config.yaml` file contents should look like this: @@ -97,31 +119,24 @@ gitops: - id: "path-to/your-awesome-project" ``` -### Creating an Agent record in GitLab - -Next you will need to create an GitLab Rails Agent record so that your GitLab project so that the Agent itself can associate with a GitLab project. This process will also yield a Secret that you will use to configure the Agent in subsequent steps. +### Create an Agent record in GitLab -There are two ways to accomplish this: +Next, create an GitLab Rails Agent record so the Agent can associate itself with +the configuration repository project. Creating this record also creates a Secret needed to configure +the Agent in subsequent steps. You can create an Agent record either: -1. Via the Rails console -1. Via GraphQL +- Through the Rails console, by running `rails c`: -To do this you could either run `rails c` or via GraphQL. From `rails c`: - -```ruby + ```ruby project = ::Project.find_by_full_path("path-to/your-awesome-project") agent = ::Clusters::Agent.create(name: "<agent-name>", project: project) token = ::Clusters::AgentToken.create(agent: agent) token.token # this will print out the token you need to use on the next step -``` + ``` -or using GraphQL: +- Through GraphQL: **(PREMIUM ONLY)** -with this approach, you'll need a premium license to use this feature. - -If you are new to using the GitLab GraphQL API please refer to the [Getting started with the GraphQL API page](../../../api/graphql/getting_started.md) or check out the [GraphQL Explorer](https://gitlab.com/-/graphql-explorer). - -```json + ```graphql mutation createAgent { createClusterAgent(input: { projectPath: "path-to/your-awesome-project", name: "<agent-name>" }) { clusterAgent { @@ -142,37 +157,77 @@ If you are new to using the GitLab GraphQL API please refer to the [Getting star errors } } -``` + ``` -Note that GraphQL will only show you the token once, after you've created it. + NOTE: **Note:** + GraphQL only displays the token once, after creating it. -### Creating the Kubernetes secret + If you are new to using the GitLab GraphQL API, refer to the + [Getting started with the GraphQL API page](../../../api/graphql/getting_started.md), + or the [GraphQL Explorer](https://gitlab.com/-/graphql-explorer). -Once the token has been generated it needs to be applied to the Kubernetes cluster. +### Create the Kubernetes secret -If you didn't previously define or create a namespace you need to do that first: +After generating the token, you must apply it to the Kubernetes cluster. -```shell -kubectl create namespace <YOUR-DESIRED-NAMESPACE> -``` +1. If you haven't previous defined or created a namespace, run the following command: + + ```shell + kubectl create namespace <YOUR-DESIRED-NAMESPACE> + ``` + +1. Run the following command to create your Secret: + + ```shell + kubectl create secret generic -n <YOUR-DESIRED-NAMESPACE> gitlab-agent-token --from-literal=token='YOUR_AGENT_TOKEN' + ``` -Run the following command to create your Secret: +### Install the Agent into the cluster + +Next, install the in-cluster component of the Agent. This example file contains the +Kubernetes resources required for the Agent to be installed. You can modify this +example [`resources.yml` file](#example-resourcesyml-file) in the following ways: + +- You can replace `gitlab-agent` with `<YOUR-DESIRED-NAMESPACE>`. +- For the `kas-address` (Kubernetes Agent Server), the agent can use the WebSockets + or gRPC protocols to connect to the Agent Server. Depending on your cluster + configuration and GitLab architecture, you may need to use one or the other. + For the `gitlab-kas` Helm chart, an Ingress is created for the Agent Server using + the `/-/kubernetes-agent` endpoint. This can be used for the WebSockets protocol connection. + - Specify the `grpc` scheme (such as `grpc://gitlab-kas:5005`) to use gRPC directly. + Encrypted gRPC is not supported yet. Follow the + [Support TLS for gRPC communication issue](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/issues/7) + for progress updates. + - Specify the `ws` scheme (such as `ws://gitlab-kas-ingress:80/-/kubernetes-agent`) + to use an unencrypted WebSockets connection. + - Specify the `wss` scheme (such as `wss://gitlab-kas-ingress:443/-/kubernetes-agent`) + to use an encrypted WebSockets connection. This is the recommended option if + installing the Agent into a separate cluster from your Agent Server. +- If you defined your own secret name, replace `gitlab-agent-token` with your secret name. + +To apply this file, run the following command: ```shell -kubectl create secret generic -n <YOUR-DESIRED-NAMESPACE> gitlab-agent-token --from-literal=token='YOUR_AGENT_TOKEN' +kubectl apply -n gitlab-agent -f ./resources.yml ``` -### Installing the Agent into the cluster +To review your configuration, run the following command: -Next you are now ready to install the in-cluster component of the Agent. The below is an example YAML file of the Kubernetes resources required for the Agent to be installed. - -Let's highlight a few of the details in the example below: +```shell +$ kubectl get pods --all-namespaces -1. You can replace `gitlab-agent` with <YOUR-DESIRED-NAMESPACE> -1. For the `kas-address` (Kubernetes Agent Server), you can replace `grpc://host.docker.internal:5005` with the address of the kas agent that was initialized via your Helm install. -1. If you defined your own secret name, then replace `gitlab-agent-token` with your secret name. +NAMESPACE NAME READY STATUS RESTARTS AGE +gitlab-agent gitlab-agent-77689f7dcb-5skqk 1/1 Running 0 51s +kube-system coredns-f9fd979d6-n6wcw 1/1 Running 0 14m +kube-system etcd-minikube 1/1 Running 0 14m +kube-system kube-apiserver-minikube 1/1 Running 0 14m +kube-system kube-controller-manager-minikube 1/1 Running 0 14m +kube-system kube-proxy-j6zdh 1/1 Running 0 14m +kube-system kube-scheduler-minikube 1/1 Running 0 14m +kube-system storage-provisioner 1/1 Running 0 14m +``` -`./resources.yml` +#### Example `resources.yml` file ```yaml apiVersion: v1 @@ -201,7 +256,7 @@ spec: args: - --token-file=/config/token - --kas-address - - grpc://host.docker.internal:5005 # {"$openapi":"kas-address"} + - grpc://host.docker.internal:5005 # {"$openapi":"kas-address"} volumeMounts: - name: token-volume mountPath: /config @@ -269,31 +324,24 @@ subjects: - name: gitlab-agent kind: ServiceAccount namespace: gitlab-agent - ``` -```shell -kubectl apply -n gitlab-agent -f ./resources.yml -``` +### Create a `manifest.yaml` + +In a previous step, you configured a `config.yaml` to point to the GitLab projects +the Agent should synchronize. In each of those projects, you must create a `manifest.yaml` +file for the Agent to monitor. You can auto-generate this `manifest.yaml` with a +templating engine or other means. + +Each time you commit and push a change to this file, the Agent logs the change: ```plaintext -$ kubectl get pods --all-namespaces -NAMESPACE NAME READY STATUS RESTARTS AGE -gitlab-agent gitlab-agent-77689f7dcb-5skqk 1/1 Running 0 51s -kube-system coredns-f9fd979d6-n6wcw 1/1 Running 0 14m -kube-system etcd-minikube 1/1 Running 0 14m -kube-system kube-apiserver-minikube 1/1 Running 0 14m -kube-system kube-controller-manager-minikube 1/1 Running 0 14m -kube-system kube-proxy-j6zdh 1/1 Running 0 14m -kube-system kube-scheduler-minikube 1/1 Running 0 14m -kube-system storage-provisioner 1/1 Running 0 14m +2020-09-15_14:09:04.87946 gitlab-k8s-agent : time="2020-09-15T10:09:04-04:00" level=info msg="Config: new commit" agent_id=1 commit_id=e6a3651f1faa2e928fe6120e254c122451be4eea ``` -### Creating a `manifest.yaml` - -In the above step, you configured a `config.yaml` to point to which GitLab projects the Agent should synchronize. Within each one of those projects, you need to create a `manifest.yaml` file which the Agent will monitor. This `manifest.yaml` can be autogenerated by a templating engine or other means. +#### Example `manifest.yaml` file -Example `manifest.yaml`: +This file creates a simple NGINX deployment. ```yaml apiVersion: apps/v1 @@ -318,14 +366,9 @@ spec: - containerPort: 80 ``` -The above file creates a simple NGINX deployment. - -Each time you commit and push a change to the `manifest.yaml` the Agent will observe the change. Example log: - -```plaintext -2020-09-15_14:09:04.87946 gitlab-k8s-agent : time="2020-09-15T10:09:04-04:00" level=info msg="Config: new commit" agent_id=1 commit_id=e6a3651f1faa2e928fe6120e254c122451be4eea -``` - ## Example projects -Basic GitOps example deploying NGINX: [Configuration repository](https://gitlab.com/gitlab-org/configure/examples/kubernetes-agent), [Manifest repository](https://gitlab.com/gitlab-org/configure/examples/gitops-project) +This basic GitOps example deploys NGINX: + +- [Configuration repository](https://gitlab.com/gitlab-org/configure/examples/kubernetes-agent) +- [Manifest repository](https://gitlab.com/gitlab-org/configure/examples/gitops-project) diff --git a/doc/user/clusters/applications.md b/doc/user/clusters/applications.md index 2243ffa0cb1..4e0e2991acb 100644 --- a/doc/user/clusters/applications.md +++ b/doc/user/clusters/applications.md @@ -6,37 +6,30 @@ info: To determine the technical writer assigned to the Stage/Group associated w # GitLab Managed Apps -GitLab provides **GitLab Managed Apps**, a one-click install for various applications which can -be added directly to your configured cluster. +GitLab provides **GitLab Managed Apps** for various +applications which can be added directly to your configured cluster. These +applications are needed for [Review Apps](../../ci/review_apps/index.md) and +[deployments](../../ci/environments/index.md) when using [Auto DevOps](../../topics/autodevops/index.md). +You can install them after you [create a cluster](../project/clusters/add_remove_clusters.md). GitLab provides +GitLab Managed Apps that can installed with [one-click](#install-with-one-click) or [using CI/CD](#install-using-gitlab-cicd-alpha). -These applications are needed for [Review Apps](../../ci/review_apps/index.md) -and [deployments](../../ci/environments/index.md) when using [Auto DevOps](../../topics/autodevops/index.md). +## Install with one click -You can install them after you -[create a cluster](../project/clusters/add_remove_clusters.md). - -## Installing applications - -Applications managed by GitLab will be installed onto the `gitlab-managed-apps` namespace. - -This namespace: +Applications managed by GitLab are installed onto the `gitlab-managed-apps` +namespace. This namespace: - Is different from the namespace used for project deployments. - Is created once. - Has a non-configurable name. -To see a list of available applications to install. For a: +To view a list of available applications to install for a: - [Project-level cluster](../project/clusters/index.md), navigate to your project's **Operations > Kubernetes**. - [Group-level cluster](../group/clusters/index.md), navigate to your group's **Kubernetes** page. -NOTE: **Note:** -As of GitLab 11.6, Helm will be upgraded to the latest version supported -by GitLab before installing any of the applications. - -The following applications can be installed: +You can install the following applications with one click: - [Helm](#helm) - [Ingress](#ingress) @@ -49,27 +42,28 @@ The following applications can be installed: - [Elastic Stack](#elastic-stack) - [Fluentd](#fluentd) -With the exception of Knative, the applications will be installed in a dedicated +With the exception of Knative, the applications are installed in a dedicated namespace called `gitlab-managed-apps`. -NOTE: **Note:** Some applications are installable only for a project-level cluster. Support for installing these applications in a group-level cluster is planned for future releases. -For updates, see [the issue tracking -progress](https://gitlab.com/gitlab-org/gitlab/-/issues/24411). +For updates, see the [issue tracking progress](https://gitlab.com/gitlab-org/gitlab/-/issues/24411). CAUTION: **Caution:** If you have an existing Kubernetes cluster with Helm already installed, you should be careful as GitLab cannot detect it. In this case, installing -Helm via the applications will result in the cluster having it twice, which +Helm with the applications results in the cluster having it twice, which can lead to confusion during deployments. +In GitLab versions 11.6 and greater, Helm is upgraded to the latest version +supported by GitLab before installing any of the applications. + ### Helm > - Introduced in GitLab 10.2 for project-level clusters. > - Introduced in GitLab 11.6 for group-level clusters. -> - [Uses a local Tiller](https://gitlab.com/gitlab-org/gitlab/-/issues/209736) since GitLab 13.2. +> - [Uses a local Tiller](https://gitlab.com/gitlab-org/gitlab/-/issues/209736) in GitLab 13.2 and later. [Helm](https://helm.sh/docs/) is a package manager for Kubernetes and is used to install the GitLab-managed apps. GitLab runs each `helm` command @@ -81,7 +75,6 @@ applications. Prior to [GitLab 13.2](https://gitlab.com/gitlab-org/gitlab/-/issu GitLab used an in-cluster Tiller server in the `gitlab-managed-apps` namespace. This server can now be safely removed. -NOTE: **Note:** GitLab's Helm integration does not support installing applications behind a proxy, but a [workaround](../../topics/autodevops/index.md#install-applications-behind-a-proxy) is available. @@ -90,26 +83,25 @@ is available. > Introduced in GitLab 11.6 for project- and group-level clusters. -[cert-manager](https://cert-manager.io/docs/) is a native -Kubernetes certificate management controller that helps with issuing -certificates. Installing cert-manager on your cluster will issue a -certificate by [Let's Encrypt](https://letsencrypt.org/) and ensure that -certificates are valid and up-to-date. +[cert-manager](https://cert-manager.io/docs/) is a native Kubernetes certificate +management controller that helps with issuing certificates. Installing +cert-manager on your cluster issues a certificate by [Let's Encrypt](https://letsencrypt.org/) +and ensures that certificates are valid and up-to-date. The chart used to install this application depends on the version of GitLab used. In: -- GitLab 12.3 and newer, the [jetstack/cert-manager](https://github.com/jetstack/cert-manager) +- GitLab 12.3 and newer, the [`jetstack/cert-manager`](https://github.com/jetstack/cert-manager) chart is used with a [`values.yaml`](https://gitlab.com/gitlab-org/gitlab/blob/master/vendor/cert_manager/values.yaml) file. -- GitLab 12.2 and older, the [stable/cert-manager](https://github.com/helm/charts/tree/master/stable/cert-manager) +- GitLab 12.2 and older, the [`stable/cert-manager`](https://gi2wthub.com/helm/charts/tree/master/stable/cert-manager) chart was used. -If you have installed cert-manager prior to GitLab 12.3, Let's Encrypt will -[block requests from older versions of cert-manager](https://community.letsencrypt.org/t/blocking-old-cert-manager-versions/98753). - -To resolve this: +If you installed cert-manager prior to GitLab 12.3, Let's Encrypt +[blocks requests](https://community.letsencrypt.org/t/blocking-old-cert-manager-versions/98753) +from older versions of `cert-manager`. To resolve this: -1. Uninstall cert-manager (consider [backing up any additional configuration](https://cert-manager.io/docs/tutorials/backup/)). +1. [Back up any additional configuration](https://cert-manager.io/docs/tutorials/backup/). +1. Uninstall cert-manager. 1. Install cert-manager again. ### GitLab Runner @@ -117,52 +109,52 @@ To resolve this: > - Introduced in GitLab 10.6 for project-level clusters. > - Introduced in GitLab 11.10 for group-level clusters. -[GitLab Runner](https://docs.gitlab.com/runner/) is the open source -project that is used to run your jobs and send the results back to -GitLab. It is used in conjunction with [GitLab -CI/CD](../../ci/README.md), the open-source continuous integration -service included with GitLab that coordinates the jobs. - -If the project is on GitLab.com, shared runners are available -(the first 2000 minutes are free, you can -[buy more later](../../subscriptions/gitlab_com/index.md#purchase-additional-ci-minutes)) -and you do not have to deploy one if they are enough for your needs. If a -project-specific runner is desired, or there are no shared runners, it is easy -to deploy one. - -Note that the deployed runner will be set as **privileged**, which means it will essentially -have root access to the underlying machine. This is required to build Docker images, -so it is the default. Make sure you read the -[security implications](../project/clusters/index.md#security-implications) +[GitLab Runner](https://docs.gitlab.com/runner/) is the open source project that +is used to run your jobs and send the results back to GitLab. It's used in +conjunction with [GitLab CI/CD](../../ci/README.md), the open-source continuous +integration service included with GitLab that coordinates the jobs. + +If the project is on GitLab.com, [shared runners](../gitlab_com/index.md#shared-runners) +are available. You don't have to deploy one if they are enough for your +needs. If a project-specific runner is desired, or there are no shared runners, +you can deploy one. + +The deployed runner is set as **privileged**. Root access to the underlying +server is required to build Docker images, so it's the default. Be sure to read +the [security implications](../project/clusters/index.md#security-implications) before deploying one. -NOTE: **Note:** The [`runner/gitlab-runner`](https://gitlab.com/gitlab-org/charts/gitlab-runner) chart is used to install this application, using [a preconfigured `values.yaml`](https://gitlab.com/gitlab-org/charts/gitlab-runner/-/blob/master/values.yaml) -file. Customizing the installation by modifying this file is not supported. +file. Customizing the installation by modifying this file is not supported. This +also means you cannot modify `config.toml` file for this Runner. If you want to +have that possibility and still deploy Runner in Kubernetes, consider using the +[Cluster management project](management_project.md) or installing Runner manually +via [GitLab Runner Helm Chart](https://docs.gitlab.com/runner/install/kubernetes.html). ### Ingress > - Introduced in GitLab 10.2 for project-level clusters. > - Introduced in GitLab 11.6 for group-level clusters. -[Ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/) provides load balancing, SSL termination, and name-based virtual hosting +[Ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/) +provides load balancing, SSL termination, and name-based virtual hosting out of the box. It acts as a web proxy for your applications and is useful if you want to use [Auto DevOps](../../topics/autodevops/index.md) or deploy your own web apps. -The Ingress Controller installed is [Ingress-NGINX](https://kubernetes.io/docs/concepts/services-networking/ingress/), +The Ingress Controller installed is +[Ingress-NGINX](https://kubernetes.io/docs/concepts/services-networking/ingress/), which is supported by the Kubernetes community. -NOTE: **Note:** With the following procedure, a load balancer must be installed in your cluster to obtain the endpoint. You can use either Ingress, or Knative's own load balancer ([Istio](https://istio.io)) if using Knative. -In order to publish your web application, you first need to find the endpoint which will be either an IP +To publish your web application, you first need to find the endpoint, which is either an IP address or a hostname associated with your load balancer. -To install it, click on the **Install** button for Ingress. GitLab will attempt +To install it, click on the **Install** button for Ingress. GitLab attempts to determine the external endpoint and it should be available within a few minutes. #### Determining the external endpoint automatically @@ -178,22 +170,25 @@ using the `KUBE_INGRESS_BASE_DOMAIN` environment variable. If the endpoint doesn't appear and your cluster runs on Google Kubernetes Engine: -1. Check your [Kubernetes cluster on Google Kubernetes Engine](https://console.cloud.google.com/kubernetes) to ensure there are no errors on its nodes. -1. Ensure you have enough [Quotas](https://console.cloud.google.com/iam-admin/quotas) on Google Kubernetes Engine. For more information, see [Resource Quotas](https://cloud.google.com/compute/quotas). -1. Check [Google Cloud's Status](https://status.cloud.google.com/) to ensure they are not having any disruptions. +1. [Examine your Kubernetes cluster](https://console.cloud.google.com/kubernetes) + on Google Kubernetes Engine to ensure there are no errors on its nodes. +1. Ensure you have enough [Quotas](https://console.cloud.google.com/iam-admin/quotas) + on Google Kubernetes Engine. For more information, see + [Resource Quotas](https://cloud.google.com/compute/quotas). +1. Review [Google Cloud's Status](https://status.cloud.google.com/) for service + disruptions. -Once installed, you may see a `?` for "Ingress IP Address" depending on the -cloud provider. For EKS specifically, this is because the ELB is created -with a DNS name, not an IP address. If GitLab is still unable to -determine the endpoint of your Ingress or Knative application, you can -[determine it manually](#determining-the-external-endpoint-manually). - -NOTE: **Note:** The [`stable/nginx-ingress`](https://github.com/helm/charts/tree/master/stable/nginx-ingress) chart is used to install this application with a [`values.yaml`](https://gitlab.com/gitlab-org/gitlab/blob/master/vendor/ingress/values.yaml) file. +After installing, you may see a `?` for **Ingress IP Address** depending on the +cloud provider. For EKS specifically, this is because the ELB is created +with a DNS name, not an IP address. If GitLab is still unable to +determine the endpoint of your Ingress or Knative application, you can +[determine it manually](#determining-the-external-endpoint-manually). + #### Determining the external endpoint manually If the cluster is on GKE, click the **Google Kubernetes Engine** link in the @@ -204,62 +199,62 @@ the `gcloud` command in a local terminal or using the **Cloud Shell**. If the cluster is not on GKE, follow the specific instructions for your Kubernetes provider to configure `kubectl` with the right credentials. -The output of the following examples will show the external endpoint of your +The output of the following examples show the external endpoint of your cluster. This information can then be used to set up DNS entries and forwarding rules that allow external access to your deployed applications. -If you installed Ingress via the **Applications**, run the following command: +- If you installed Ingress using the **Applications**, run the following + command: -```shell -kubectl get service --namespace=gitlab-managed-apps ingress-nginx-ingress-controller -o jsonpath='{.status.loadBalancer.ingress[0].ip}' -``` + ```shell + kubectl get service --namespace=gitlab-managed-apps ingress-nginx-ingress-controller -o jsonpath='{.status.loadBalancer.ingress[0].ip}' + ``` -Some Kubernetes clusters return a hostname instead, like [Amazon EKS](https://aws.amazon.com/eks/). For these platforms, run: +- Some Kubernetes clusters return a hostname instead, like + [Amazon EKS](https://aws.amazon.com/eks/). For these platforms, run: -```shell -kubectl get service --namespace=gitlab-managed-apps ingress-nginx-ingress-controller -o jsonpath='{.status.loadBalancer.ingress[0].hostname}' -``` + ```shell + kubectl get service --namespace=gitlab-managed-apps ingress-nginx-ingress-controller -o jsonpath='{.status.loadBalancer.ingress[0].hostname}' + ``` -For Istio/Knative, the command will be different: + If EKS is used, an [Elastic Load Balancer](https://docs.aws.amazon.com/elasticloadbalancing/) + is also created, which will incur additional AWS costs. -```shell -kubectl get svc --namespace=istio-system istio-ingressgateway -o jsonpath='{.status.loadBalancer.ingress[0].ip} ' -``` +- For Istio/Knative, the command is different: -Otherwise, you can list the IP addresses of all load balancers: + ```shell + kubectl get svc --namespace=istio-system istio-ingressgateway -o jsonpath='{.status.loadBalancer.ingress[0].ip} ' + ``` -```shell -kubectl get svc --all-namespaces -o jsonpath='{range.items[?(@.status.loadBalancer.ingress)]}{.status.loadBalancer.ingress[*].ip} ' -``` +- Otherwise, you can list the IP addresses of all load balancers: -NOTE: **Note:** -If EKS is used, an [Elastic Load Balancer](https://docs.aws.amazon.com/elasticloadbalancing/) -will also be created, which will incur additional AWS costs. + ```shell + kubectl get svc --all-namespaces -o jsonpath='{range.items[?(@.status.loadBalancer.ingress)]}{.status.loadBalancer.ingress[*].ip} ' + ``` -NOTE: **Note:** -You may see a trailing `%` on some Kubernetes versions, **do not include it**. +You may see a trailing `%` on some Kubernetes versions. Do not include it. -The Ingress is now available at this address and will route incoming requests to -the proper service based on the DNS name in the request. To support this, a -wildcard DNS CNAME record should be created for the desired domain name. For example, +The Ingress is now available at this address, and routes incoming requests to +the proper service based on the DNS name in the request. To support this, create +a wildcard DNS CNAME record for the desired domain name. For example, `*.myekscluster.com` would point to the Ingress hostname obtained earlier. #### Using a static IP By default, an ephemeral external IP address is associated to the cluster's load balancer. If you associate the ephemeral IP with your DNS and the IP changes, -your apps will not be able to be reached, and you'd have to change the DNS -record again. In order to avoid that, you should change it into a static -reserved IP. +your apps won't be reachable, and you'd have to change the DNS record again. +To avoid that, change it into a static reserved IP. Read how to [promote an ephemeral external IP address in GKE](https://cloud.google.com/compute/docs/ip-addresses/reserve-static-external-ip-address#promote_ephemeral_ip). #### Pointing your DNS at the external endpoint -Once you've set up the external endpoint, you should associate it with a [wildcard DNS -record](https://en.wikipedia.org/wiki/Wildcard_DNS_record) such as `*.example.com.` -in order to be able to reach your apps. If your external endpoint is an IP address, -use an A record. If your external endpoint is a hostname, use a CNAME record. +After you have set up the external endpoint, associate it with a +[wildcard DNS record](https://en.wikipedia.org/wiki/Wildcard_DNS_record) (such +as `*.example.com.`) to reach your apps. If your external endpoint is an IP +address, use an A record. If your external endpoint is a hostname, use a CNAME +record. #### Web Application Firewall (ModSecurity) @@ -269,16 +264,15 @@ A Web Application Firewall (WAF) examines traffic being sent or received, and can block malicious traffic before it reaches your application. The benefits of a WAF are: -- Real-time security monitoring for your application -- Logging of all your HTTP traffic to the application -- Access control for your application -- Highly configurable logging and blocking rules +- Real-time security monitoring for your application. +- Logging of all your HTTP traffic to the application. +- Access control for your application. +- Highly configurable logging and blocking rules. -Out of the box, GitLab provides you with a WAF known as [`ModSecurity`](https://www.modsecurity.org/). - -ModSecurity is a toolkit for real-time web application monitoring, logging, -and access control. With GitLab's offering, the [OWASP's Core Rule Set](https://www.modsecurity.org/CRS/Documentation/), -which provides generic attack detection capabilities, is automatically applied. +By default, GitLab provides you with a WAF known as [`ModSecurity`](https://www.modsecurity.org/), +which is a toolkit for real-time web application monitoring, logging, and access +control. GitLab's offering applies the [OWASP's Core Rule Set](https://www.modsecurity.org/CRS/Documentation/), +which provides generic attack detection capabilities. This feature: @@ -299,57 +293,61 @@ There is a small performance overhead by enabling ModSecurity. If this is considered significant for your application, you can disable ModSecurity's rule engine for your deployed application in any of the following ways: -1. Setting [the deployment variable](../../topics/autodevops/index.md) -`AUTO_DEVOPS_MODSECURITY_SEC_RULE_ENGINE` to `Off`. This will prevent ModSecurity -from processing any requests for the given application or environment. - -1. Switching its respective toggle to the disabled position and applying changes through the **Save changes** button. This will reinstall -Ingress with the recent changes. +1. Set the [deployment variable](../../topics/autodevops/index.md) + `AUTO_DEVOPS_MODSECURITY_SEC_RULE_ENGINE` to `Off` to prevent ModSecurity + from processing any requests for the given application or environment. +1. Switch its respective toggle to the disabled position, and then apply changes + by selecting **Save changes** to reinstall Ingress with the recent changes.  ##### Logging and blocking modes To help you tune your WAF rules, you can globally set your WAF to either -**Logging** or **Blocking** mode: +*Logging* or *Blocking* mode: -- **Logging mode** - Allows traffic matching the rule to pass, and logs the event. -- **Blocking mode** - Prevents traffic matching the rule from passing, and logs the event. +- *Logging mode*: Allows traffic matching the rule to pass, and logs the event. +- *Blocking mode*: Prevents traffic matching the rule from passing, and logs the event. To change your WAF's mode: -1. [Install ModSecurity](../../topics/web_application_firewall/quick_start_guide.md) if you have not already done so. +1. If you haven't already done so, [install ModSecurity](../../topics/web_application_firewall/quick_start_guide.md). 1. Navigate to **Operations > Kubernetes**. 1. In **Applications**, scroll to **Ingress**. 1. Under **Global default**, select your desired mode. -1. Click **Save changes**. +1. Select **Save changes**. ##### WAF version updates -Enabling, disabling, or changing the logging mode for **ModSecurity** is only allowed within same version of [Ingress](#ingress) due to limitations in [Helm](https://helm.sh/) which might be overcome in future releases. +Enabling, disabling, or changing the logging mode for **ModSecurity** is only +allowed within same version of [Ingress](#ingress) due to limitations in +[Helm](https://helm.sh/) which might be overcome in future releases. -**ModSecurity** UI controls are disabled if the version deployed differs from the one available in GitLab, while actions at the [Ingress](#ingress) level, such as uninstalling, can still be performed: +**ModSecurity** user interface controls are disabled if the version deployed +differs from the one available in GitLab, while actions at the [Ingress](#ingress) +level, such as uninstalling, can still be performed:  -Updating [Ingress](#ingress) to the most recent version enables you to take advantage of bug fixes, security fixes, and performance improvements. To update [Ingress application](#ingress), you must first uninstall it, and then re-install it as described in [Install ModSecurity](../../topics/web_application_firewall/quick_start_guide.md). +Update [Ingress](#ingress) to the most recent version to take advantage of bug +fixes, security fixes, and performance improvements. To update the +[Ingress application](#ingress), you must first uninstall it, and then re-install +it as described in [Install ModSecurity](../../topics/web_application_firewall/quick_start_guide.md). ##### Viewing Web Application Firewall traffic > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/14707) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.9. You can view Web Application Firewall traffic by navigating to your project's -**Security & Compliance > Threat Monitoring** page. - -From there, you can see tracked over time: +**Security & Compliance > Threat Monitoring** page. From there, you can see +tracked over time: - The total amount of traffic to your application. -- The proportion of traffic that is considered anomalous by the Web Application +- The proportion of traffic that's considered anomalous by the Web Application Firewall's default [OWASP ruleset](https://www.modsecurity.org/CRS/Documentation/). -If a significant percentage of traffic is anomalous, it should be investigated -for potential threats, which can be done by -[examining the Web Application Firewall logs](#web-application-firewall-modsecurity). +If a significant percentage of traffic is anomalous, investigate it for potential threats +by [examining the Web Application Firewall logs](#web-application-firewall-modsecurity).  @@ -358,55 +356,52 @@ for potential threats, which can be done by > - Introduced in GitLab 11.0 for project-level clusters. > - Introduced in GitLab 12.3 for group and instance-level clusters. -[JupyterHub](https://jupyterhub.readthedocs.io/en/stable/) is a -multi-user service for managing notebooks across a team. [Jupyter -Notebooks](https://jupyter-notebook.readthedocs.io/en/latest/) provide a -web-based interactive programming environment used for data analysis, +[JupyterHub](https://jupyterhub.readthedocs.io/en/stable/) is a multi-user service +for managing notebooks across a team. [Jupyter Notebooks](https://jupyter-notebook.readthedocs.io/en/latest/) +provide a web-based interactive programming environment used for data analysis, visualization, and machine learning. -Authentication will be enabled only for [project -members](../project/members/index.md) for project-level clusters and group -members for group-level clusters with [Developer or -higher](../permissions.md) access to the associated project or group. - -We use a [custom Jupyter -image](https://gitlab.com/gitlab-org/jupyterhub-user-image/blob/master/Dockerfile) -that installs additional useful packages on top of the base Jupyter. You -will also see ready-to-use DevOps Runbooks built with Nurtch's [Rubix library](https://github.com/Nurtch/rubix). - -More information on -creating executable runbooks can be found in [our Runbooks -documentation](../project/clusters/runbooks/index.md#configure-an-executable-runbook-with-gitlab). Note that -Ingress must be installed and have an IP address assigned before -JupyterHub can be installed. - -NOTE: **Note:** The [`jupyter/jupyterhub`](https://jupyterhub.github.io/helm-chart/) chart is used to install this application with a [`values.yaml`](https://gitlab.com/gitlab-org/gitlab/blob/master/vendor/jupyter/values.yaml) file. +Authentication is enabled only for [project members](../project/members/index.md) +for project-level clusters and group members for group-level clusters with +[Developer or higher](../permissions.md) access to the associated project or group. + +GitLab uses a [custom Jupyter image](https://gitlab.com/gitlab-org/jupyterhub-user-image/blob/master/Dockerfile) +that installs additional useful packages on top of the base Jupyter. Ready-to-use +DevOps Runbooks built with Nurtch's [Rubix library](https://github.com/Nurtch/rubix) +are also available. + +More information on creating executable runbooks can be found in +[our Runbooks documentation](../project/clusters/runbooks/index.md#configure-an-executable-runbook-with-gitlab). +Ingress must be installed and have an IP address assigned before +JupyterHub can be installed. + #### Jupyter Git Integration > - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/28783) in GitLab 12.0 for project-level clusters. > - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/32512) in GitLab 12.3 for group and instance-level clusters. -When installing JupyterHub onto your Kubernetes cluster, [JupyterLab's Git extension](https://github.com/jupyterlab/jupyterlab-git) -is automatically provisioned and configured using the authenticated user's: +When installing JupyterHub onto your Kubernetes cluster, +[JupyterLab's Git extension](https://github.com/jupyterlab/jupyterlab-git) +is provisioned and configured using the authenticated user's: - Name. - Email. - Newly created access token. -JupyterLab's Git extension enables full version control of your notebooks as well as issuance of Git commands within Jupyter. -Git commands can be issued via the **Git** tab on the left panel or via Jupyter's command line prompt. +JupyterLab's Git extension enables full version control of your notebooks, and +issuance of Git commands within Jupyter. You can issue Git commands through the +**Git** tab on the left panel, or through Jupyter's command-line prompt. -NOTE: **Note:** -JupyterLab's Git extension stores the user token in the JupyterHub DB in encrypted format -and in the single user Jupyter instance as plain text. This is because [Git requires storing -credentials as plain text](https://git-scm.com/docs/git-credential-store). Potentially, if -a nefarious user finds a way to read from the file system in the single user Jupyter instance -they could retrieve the token. +JupyterLab's Git extension stores the user token in the JupyterHub DB in encrypted +format, and in the single user Jupyter instance as plain text, because +[Git requires storing credentials as plain text](https://git-scm.com/docs/git-credential-store) +Potentially, if a nefarious user finds a way to read from the file system in the +single-user Jupyter instance, they could retrieve the token.  @@ -421,22 +416,20 @@ You can clone repositories from the files tab in Jupyter: [Knative](https://cloud.google.com/knative/) provides a platform to create, deploy, and manage serverless workloads from a Kubernetes -cluster. It is used in conjunction with, and includes +cluster. It's used in conjunction with, and includes [Istio](https://istio.io) to provide an external IP address for all programs hosted by Knative. -You will be prompted to enter a wildcard -domain where your applications will be exposed. Configure your DNS -server to use the external IP address for that domain. For any -application created and installed, they will be accessible as -`<program_name>.<kubernetes_namespace>.<domain_name>`. This will require -your Kubernetes cluster to have [RBAC -enabled](../project/clusters/add_remove_clusters.md#rbac-cluster-resources). - -NOTE: **Note:** The [`knative/knative`](https://storage.googleapis.com/triggermesh-charts) chart is used to install this application. +During installation, you must enter a wildcard domain where your applications +will be exposed. Configure your DNS server to use the external IP address for that +domain. Applications created and installed are accessible as +`<program_name>.<kubernetes_namespace>.<domain_name>`, which requires +your Kubernetes cluster to have +[RBAC enabled](../project/clusters/add_remove_clusters.md#rbac-cluster-resources). + ### Prometheus > - Introduced in GitLab 10.4 for project-level clusters. @@ -446,20 +439,19 @@ chart is used to install this application. open-source monitoring and alerting system useful to supervise your deployed applications. -GitLab is able to monitor applications automatically, using the +GitLab is able to monitor applications by using the [Prometheus integration](../project/integrations/prometheus.md). Kubernetes container CPU and -memory metrics are automatically collected, and response metrics are retrieved -from NGINX Ingress as well. +memory metrics are collected, and response metrics are also retrieved +from NGINX Ingress. -To enable monitoring, simply install Prometheus into the cluster with the -**Install** button. - -NOTE: **Note:** The [`stable/prometheus`](https://github.com/helm/charts/tree/master/stable/prometheus) chart is used to install this application with a [`values.yaml`](https://gitlab.com/gitlab-org/gitlab/blob/master/vendor/prometheus/values.yaml) file. +To enable monitoring, install Prometheus into the cluster with the **Install** +button. + ### Crossplane > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/34702) in GitLab 12.5 for project-level clusters. @@ -481,17 +473,16 @@ The Crossplane GitLab-managed application: project repository. - Can then be used to provision infrastructure or managed applications such as PostgreSQL (for example, CloudSQL from GCP or RDS from AWS) and other services - required by the application via the Auto DevOps pipeline. + required by the application with the Auto DevOps pipeline. -For information on configuring Crossplane installed on the cluster, see -[Crossplane configuration](crossplane.md). - -NOTE: **Note:** [`alpha/crossplane`](https://github.com/crossplane/crossplane/tree/v0.4.1/cluster/charts/crossplane) chart v0.4.1 is used to install Crossplane using the [`values.yaml`](https://github.com/crossplane/crossplane/blob/master/cluster/charts/crossplane/values.yaml.tmpl) file. +For information about configuring Crossplane installed on the cluster, see +[Crossplane configuration](crossplane.md). + ### Elastic Stack > Introduced in GitLab 12.7 for project- and group-level clusters. @@ -500,37 +491,32 @@ file. log analysis solution which helps in deep searching, analyzing and visualizing the logs generated from different machines. -GitLab is able to gather logs from pods in your cluster automatically. -Filebeat will run as a DaemonSet on each node in your cluster, and it will ship container logs to Elasticsearch for querying. -GitLab will then connect to Elasticsearch for logs instead of the Kubernetes API, -and you will have access to more advanced querying capabilities. +GitLab can gather logs from pods in your cluster. Filebeat runs as a DaemonSet +on each node in your cluster, and ships container logs to Elasticsearch for +querying. GitLab then connects to Elasticsearch for logs, instead of the +Kubernetes API, giving you access to more advanced querying capabilities. Log +data is deleted after 30 days, using [Curator](https://www.elastic.co/guide/en/elasticsearch/client/curator/5.5/about.html). -Log data is automatically deleted after 30 days using [Curator](https://www.elastic.co/guide/en/elasticsearch/client/curator/5.5/about.html). +The Elastic Stack cluster application is intended as a log aggregation solution +and is not related to our [Advanced Search](../search/advanced_global_search.md) +functionality, which uses a separate Elasticsearch cluster. To enable log shipping: -1. Ensure your cluster contains at least 3 nodes of instance types larger than - `f1-micro`, `g1-small`, or `n1-standard-1`. +1. Ensure your cluster contains at least three nodes of instance types larger + than `f1-micro`, `g1-small`, or `n1-standard-1`. 1. Navigate to **Operations > Kubernetes**. 1. In **Kubernetes Cluster**, select a cluster. -1. In the **Applications** section, find **Elastic Stack** and click **Install**. +1. In the **Applications** section, find **Elastic Stack**, and then select + **Install**. -NOTE: **Note:** The [`gitlab/elastic-stack`](https://gitlab.com/gitlab-org/charts/elastic-stack) chart is used to install this application with a [`values.yaml`](https://gitlab.com/gitlab-org/gitlab/blob/master/vendor/elastic_stack/values.yaml) -file. - -NOTE: **Note:** -The chart deploys 3 identical Elasticsearch pods which can't be colocated, and each -requires 1 CPU and 2 GB of RAM, making them incompatible with clusters containing -fewer than 3 nodes or consisting of `f1-micro`, `g1-small`, `n1-standard-1`, or -`*-highcpu-2` instance types. - -NOTE: **Note:** -The Elastic Stack cluster application is intended as a log aggregation solution and is not related to our -[Advanced Search](../search/advanced_global_search.md) functionality, which uses a separate -Elasticsearch cluster. +file. The chart deploys three identical Elasticsearch pods which can't be +colocated, and each requires one CPU and 2 GB of RAM, making them +incompatible with clusters containing fewer than three nodes, or consisting of +`f1-micro`, `g1-small`, `n1-standard-1`, or `*-highcpu-2` instance types. #### Optional: deploy Kibana to perform advanced queries @@ -578,8 +564,8 @@ your data. Fluentd sends logs in syslog format. To enable Fluentd: 1. Navigate to **Operations > Kubernetes** and click - **Applications**. You will be prompted to enter a host, port and protocol - where the WAF logs will be sent to via syslog. + **Applications**. Enter a host, port, and protocol + for sending the WAF logs with syslog. 1. Provide the host domain name or URL in **SIEM Hostname**. 1. Provide the host port number in **SIEM Port**. 1. Select a **SIEM Protocol**. @@ -599,7 +585,7 @@ to get started. > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/20822) in GitLab 12.6. CAUTION: **Warning:** -This is an _alpha_ feature, and it is subject to change at any time without +This is an _alpha_ feature, and is subject to change at any time without prior notice. This alternative method allows users to install GitLab-managed @@ -639,7 +625,6 @@ To install applications using GitLab CI/CD: - template: Managed-Cluster-Applications.gitlab-ci.yml ``` - NOTE: **Note:** The job provided by this template connects to the cluster using tools provided in a custom Docker image. It requires that you have a runner registered with the Docker, Kubernetes, or Docker Machine executor. @@ -657,11 +642,10 @@ To install applications using GitLab CI/CD: 1. Optionally, define `.gitlab/managed-apps/<application>/values.yaml` file to customize values for the installed application. -A GitLab CI/CD pipeline will then run on the `master` branch to install the +A GitLab CI/CD pipeline runs on the `master` branch to install the applications you have configured. In case of pipeline failure, the -output of the [Helm -Tiller](https://v2.helm.sh/docs/install/#running-tiller-locally) binary -will be saved as a [CI job artifact](../../ci/pipelines/job_artifacts.md). +output of the [Helm Tiller](https://v2.helm.sh/docs/install/#running-tiller-locally) binary +is saved as a [CI job artifact](../../ci/pipelines/job_artifacts.md). ### Important notes @@ -669,10 +653,10 @@ Note the following: - We recommend using the cluster management project exclusively for managing deployments to a cluster. Do not add your application's source code to such projects. -- When you set the value for `installed` key back to `false`, the application will be +- When you set the value for `installed` key back to `false`, the application is unprovisioned from the cluster. - If you update `.gitlab/managed-apps/<application>/values.yaml` with new values, the - application will be redeployed. + application is redeployed. ### Install Ingress using GitLab CI/CD @@ -684,7 +668,7 @@ ingress: installed: true ``` -Ingress will then be installed into the `gitlab-managed-apps` namespace +Ingress is installed into the `gitlab-managed-apps` namespace of your cluster. You can customize the installation of Ingress by defining a @@ -693,9 +677,10 @@ management project. Refer to the [chart](https://github.com/helm/charts/tree/master/stable/nginx-ingress) for the available configuration options. -NOTE: **Note:** Support for installing the Ingress managed application is provided by the GitLab Configure group. -If you run into unknown issues, please [open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new) and ping at least 2 people from the [Configure group](https://about.gitlab.com/handbook/product/product-categories/#configure-group). +If you run into unknown issues, [open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new), +and ping at least 2 people from the +[Configure group](https://about.gitlab.com/handbook/product/product-categories/#configure-group). ### Install cert-manager using GitLab CI/CD @@ -705,7 +690,8 @@ cert-manager is installed using GitLab CI/CD by defining configuration in cert-manager: - Is installed into the `gitlab-managed-apps` namespace of your cluster. -- Can be installed with or without a default [Let's Encrypt `ClusterIssuer`](https://cert-manager.io/docs/configuration/acme/), which requires an +- Can be installed with or without a default + [Let's Encrypt `ClusterIssuer`](https://cert-manager.io/docs/configuration/acme/), which requires an email address to be specified. The email address is used by Let's Encrypt to contact you about expiring certificates and issues related to your account. @@ -734,14 +720,16 @@ management project. Refer to the [chart](https://hub.helm.sh/charts/jetstack/cert-manager) for the available configuration options. -NOTE: **Note:** -Support for installing the Cert Manager managed application is provided by the GitLab Configure group. -If you run into unknown issues, please [open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new) and ping at least 2 people from the [Configure group](https://about.gitlab.com/handbook/product/product-categories/#configure-group). +Support for installing the Cert Manager managed application is provided by the +GitLab Configure group. If you run into unknown issues, +[open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new), and ping at +least 2 people from the +[Configure group](https://about.gitlab.com/handbook/product/product-categories/#configure-group). ### Install Sentry using GitLab CI/CD -NOTE: **Note:** -The Sentry Helm chart [recommends](https://github.com/helm/charts/blob/f6e5784f265dd459c5a77430185d0302ed372665/stable/sentry/values.yaml#L284-L285) at least 3GB of available RAM for database migrations. +The Sentry Helm chart [recommends](https://github.com/helm/charts/blob/f6e5784f265dd459c5a77430185d0302ed372665/stable/sentry/values.yaml#L284-L285) +at least 3 GB of available RAM for database migrations. To install Sentry, define the `.gitlab/managed-apps/config.yaml` file with: @@ -751,7 +739,7 @@ sentry: installed: true ``` -Sentry will then be installed into the `gitlab-managed-apps` namespace +Sentry is installed into the `gitlab-managed-apps` namespace of your cluster. You can customize the installation of Sentry by defining @@ -766,8 +754,10 @@ We recommend you pay close attention to the following configuration options: - `user`. Where you can set the login credentials for the default admin user. - `postgresql`. For a PostgreSQL password that can be used when running future updates. -NOTE: **Note:** -When upgrading it is important to provide the existing PostgreSQL password (given using the `postgresql.postgresqlPassword` key) or you will receive authentication errors. See the [PostgreSQL chart documentation](https://github.com/helm/charts/tree/master/stable/postgresql#upgrade) for more information. +When upgrading, it's important to provide the existing PostgreSQL password (given +using the `postgresql.postgresqlPassword` key) to avoid authentication errors. +Read the [PostgreSQL chart documentation](https://github.com/helm/charts/tree/master/stable/postgresql#upgrade) +for more information. Here is an example configuration for Sentry: @@ -799,9 +789,11 @@ postgresql: postgresqlPassword: example-postgresql-password ``` -NOTE: **Note:** -Support for installing the Sentry managed application is provided by the GitLab Health group. -If you run into unknown issues, please [open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new) and ping at least 2 people from the [Health group](https://about.gitlab.com/handbook/product/product-categories/#health-group). +Support for installing the Sentry managed application is provided by the +GitLab Health group. If you run into unknown issues, +[open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new), and ping at +least 2 people from the +[Health group](https://about.gitlab.com/handbook/product/product-categories/#health-group). ### Install PostHog using GitLab CI/CD @@ -816,13 +808,14 @@ posthog: ``` You can customize the installation of PostHog by defining `.gitlab/managed-apps/posthog/values.yaml` -in your cluster management project. Refer to the [Configuration section of the PostHog chart's README](https://github.com/PostHog/charts/tree/master/charts/posthog) +in your cluster management project. Refer to the +[Configuration section of the PostHog chart's README](https://github.com/PostHog/charts/tree/master/charts/posthog) for the available configuration options. -NOTE: **Note:** You must provide a PostgreSQL password in `postgresql.postgresqlPassword` -or you will receive authentication errors. -See the [PostgreSQL chart documentation](https://github.com/helm/charts/tree/master/stable/postgresql#upgrade) for more information. +to avoid authentication errors. Read the +[PostgreSQL chart documentation](https://github.com/helm/charts/tree/master/stable/postgresql#upgrade) +for more information. Redis pods are restarted between upgrades. To prevent downtime, provide a Redis password using the `redis.password` key. This prevents a new password from @@ -848,9 +841,9 @@ redis: password: example-redis-password ``` -NOTE: **Note:** Support for the PostHog managed application is provided by the PostHog team. -If you run into issues, please [open a support ticket](https://github.com/PostHog/posthog/issues/new/choose) directly. +If you run into issues, +[open a support ticket](https://github.com/PostHog/posthog/issues/new/choose) directly. ### Install Prometheus using GitLab CI/CD @@ -874,9 +867,10 @@ project. Refer to the [Configuration section of the Prometheus chart's README](https://github.com/helm/charts/tree/master/stable/prometheus#configuration) for the available configuration options. -NOTE: **Note:** -Support for installing the Prometheus managed application is provided by the GitLab APM group. -If you run into unknown issues, please [open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new) and ping at least 2 people from the [APM group](https://about.gitlab.com/handbook/product/product-categories/#apm-group). +Support for installing the Prometheus managed application is provided by the +GitLab APM group. If you run into unknown issues, +[open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new), and ping at +least 2 people from the [APM group](https://about.gitlab.com/handbook/product/product-categories/#apm-group). ### Install GitLab Runner using GitLab CI/CD @@ -892,16 +886,17 @@ gitlabRunner: GitLab Runner is installed into the `gitlab-managed-apps` namespace of your cluster. -In order for GitLab Runner to function, you **must** specify the following: +For GitLab Runner to function, you _must_ specify the following: -- `gitlabUrl` - the GitLab server full URL (for example, `https://gitlab.example.com`) to register the Runner against. -- `runnerRegistrationToken` - The registration token for adding new runners to GitLab. This must be - [retrieved from your GitLab instance](../../ci/runners/README.md). +- `gitlabUrl`: The GitLab server full URL (for example, `https://gitlab.example.com`) + to register the Runner against. +- `runnerRegistrationToken`: The registration token for adding new runners to GitLab. + This must be [retrieved from your GitLab instance](../../ci/runners/README.md). These values can be specified using [CI variables](../../ci/variables/README.md): -- `GITLAB_RUNNER_GITLAB_URL` will be used for `gitlabUrl`. -- `GITLAB_RUNNER_REGISTRATION_TOKEN` will be used for `runnerRegistrationToken` +- `GITLAB_RUNNER_GITLAB_URL` is used for `gitlabUrl`. +- `GITLAB_RUNNER_REGISTRATION_TOKEN` is used for `runnerRegistrationToken` You can customize the installation of GitLab Runner by defining `.gitlab/managed-apps/gitlab-runner/values.yaml` file in your cluster @@ -909,9 +904,11 @@ management project. Refer to the [chart](https://gitlab.com/gitlab-org/charts/gitlab-runner) for the available configuration options. -NOTE: **Note:** -Support for installing the GitLab Runner managed application is provided by the GitLab Runner group. -If you run into unknown issues, please [open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new) and ping at least 2 people from the [Runner group](https://about.gitlab.com/handbook/product/product-categories/#runner-group). +Support for installing the GitLab Runner managed application is provided by the +GitLab Runner group. If you run into unknown issues, +[open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new), and ping at +least 2 people from the +[Runner group](https://about.gitlab.com/handbook/product/product-categories/#runner-group). ### Install Cilium using GitLab CI/CD @@ -948,8 +945,10 @@ for the available configuration options. You can check Cilium's installation status on the cluster management page: -- [Project-level cluster](../project/clusters/index.md): Navigate to your project's **Operations > Kubernetes** page. -- [Group-level cluster](../group/clusters/index.md): Navigate to your group's **Kubernetes** page. +- [Project-level cluster](../project/clusters/index.md): Navigate to your project's + **Operations > Kubernetes** page. +- [Group-level cluster](../group/clusters/index.md): Navigate to your group's + **Kubernetes** page. CAUTION: **Caution:** Installation and removal of the Cilium requires a **manual** @@ -959,12 +958,11 @@ of all affected pods in all namespaces to ensure that they are by the correct networking plugin. NOTE: **Note:** -Major upgrades might require additional setup steps, please consult -the official [upgrade guide](https://docs.cilium.io/en/stable/install/upgrade/) for more -information. +Major upgrades might require additional setup steps. For more information, see +the official [upgrade guide](https://docs.cilium.io/en/stable/install/upgrade/). -By default, Cilium's [audit -mode](https://docs.cilium.io/en/v1.8/gettingstarted/policy-creation/?highlight=policy-audit#enable-policy-audit-mode) +By default, Cilium's +[audit mode](https://docs.cilium.io/en/v1.8/gettingstarted/policy-creation/?highlight=policy-audit#enable-policy-audit-mode) is enabled. In audit mode, Cilium doesn't drop disallowed packets. You can use `policy-verdict` log to observe policy-related decisions. You can disable audit mode by adding the following to @@ -1006,7 +1004,7 @@ global: enabled: false ``` -You can also adjust Helm values for Hubble via +You can also adjust Helm values for Hubble by using `.gitlab/managed-apps/cilium/values.yaml`: ```yaml @@ -1018,9 +1016,11 @@ global: - 'flow:sourceContext=namespace;destinationContext=namespace' ``` -NOTE: **Note:** -Support for installing the Cilium managed application is provided by the GitLab Container Security group. -If you run into unknown issues, please [open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new) and ping at least 2 people from the [Container Security group](https://about.gitlab.com/handbook/product/product-categories/#container-security-group). +Support for installing the Cilium managed application is provided by the +GitLab Container Security group. If you run into unknown issues, +[open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new), and ping at +least 2 people from the +[Container Security group](https://about.gitlab.com/handbook/product/product-categories/#container-security-group). ### Install Falco using GitLab CI/CD @@ -1046,17 +1046,20 @@ management project. Refer to the for the available configuration options. CAUTION: **Caution:** -By default eBPF support is enabled and Falco will use an [eBPF probe](https://falco.org/docs/event-sources/drivers/#using-the-ebpf-probe) to pass system calls to userspace. -If your cluster doesn't support this, you can configure it to use Falco kernel module instead by adding the following to `.gitlab/managed-apps/falco/values.yaml`: +By default eBPF support is enabled and Falco uses an +[eBPF probe](https://falco.org/docs/event-sources/drivers/#using-the-ebpf-probe) +to pass system calls to user space. If your cluster doesn't support this, you can +configure it to use Falco kernel module instead by adding the following to +`.gitlab/managed-apps/falco/values.yaml`: ```yaml ebpf: enabled: false ``` -In rare cases where automatic probe installation on your cluster isn't possible and the kernel/probe -isn't precompiled, you may need to manually prepare the kernel module or eBPF probe with -[driverkit](https://github.com/falcosecurity/driverkit#against-a-kubernetes-cluster) +In rare cases where probe installation on your cluster isn't possible and the kernel/probe +isn't pre-compiled, you may need to manually prepare the kernel module or eBPF probe with +[`driverkit`](https://github.com/falcosecurity/driverkit#against-a-kubernetes-cluster) and install it on each cluster node. By default, Falco is deployed with a limited set of rules. To add more rules, add the following to @@ -1109,22 +1112,27 @@ You can check these logs with the following command: kubectl -n gitlab-managed-apps logs -l app=falco ``` -NOTE: **Note:** -Support for installing the Falco managed application is provided by the GitLab Container Security group. -If you run into unknown issues, please [open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new) and ping at least 2 people from the [Container Security group](https://about.gitlab.com/handbook/product/product-categories/#container-security-group). +Support for installing the Falco managed application is provided by the +GitLab Container Security group. If you run into unknown issues, +[open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new), and ping at +least 2 people from the +[Container Security group](https://about.gitlab.com/handbook/product/product-categories/#container-security-group). ### Install Vault using GitLab CI/CD > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9982) in GitLab 12.9. -[Hashicorp Vault](https://www.vaultproject.io/) is a secrets management solution which -can be used to safely manage and store passwords, credentials, certificates and more. A Vault +[HashiCorp Vault](https://www.vaultproject.io/) is a secrets management solution which +can be used to safely manage and store passwords, credentials, certificates, and more. A Vault installation could be leveraged to provide a single secure data store for credentials used in your applications, GitLab CI/CD jobs, and more. It could also serve as a way of providing SSL/TLS certificates to systems and deployments in your infrastructure. Leveraging Vault as a single source for all these credentials allows greater security by having a single source of access, control, and auditability around all your sensitive -credentials and certificates. +credentials and certificates. This feature requires giving GitLab the highest level of access and +control. Therefore, if GitLab is compromised, the security of this Vault instance is as well. To +avoid this security risk, GitLab recommends using your own HashiCorp Vault to leverage +[external secrets with CI](../../ci/secrets/index.md). To install Vault, enable it in the `.gitlab/managed-apps/config.yaml` file: @@ -1133,24 +1141,25 @@ vault: installed: true ``` -By default you will get a basic Vault setup with no scalable -storage backend. This is enough for simple testing and small-scale deployments, though has limits -to how much it can scale, and as it is a single instance deployment, you will experience downtime -when upgrading the Vault application. +By default you receive a basic Vault setup with no scalable storage backend. This +is enough for simple testing and small-scale deployments, though has limits +to how much it can scale, and as it's a single instance deployment, upgrading the +Vault application causes downtime. To optimally use Vault in a production environment, it's ideal to have a good understanding -of the internals of Vault and how to configure it. This can be done by reading the -[the Vault documentation](https://www.vaultproject.io/docs/internals) as well as +of the internals of Vault and how to configure it. This can be done by reading +the [Vault Configuration guide](../../ci/secrets/#configure-your-vault-server), +the [Vault documentation](https://www.vaultproject.io/docs/internals) and the Vault Helm chart [`values.yaml` file](https://github.com/hashicorp/vault-helm/blob/v0.3.3/values.yaml). -At a minimum you will likely set up: +At a minimum, most users set up: - A [seal](https://www.vaultproject.io/docs/configuration/seal) for extra encryption - of the master key. -- A [storage backend](https://www.vaultproject.io/docs/configuration/storage) that is + of the main key. +- A [storage backend](https://www.vaultproject.io/docs/configuration/storage) that's suitable for environment and storage security requirements. - [HA Mode](https://www.vaultproject.io/docs/concepts/ha). -- [The Vault UI](https://www.vaultproject.io/docs/configuration/ui). +- The [Vault UI](https://www.vaultproject.io/docs/configuration/ui). The following is an example values file (`.gitlab/managed-apps/vault/values.yaml`) that configures Google Key Management Service for auto-unseal, using a Google Cloud Storage backend, enabling @@ -1180,7 +1189,7 @@ server: path = "gcs://my-vault-storage/vault-bucket" ha_enabled = "true" } - # Configure Vault to automatically unseal storage using a GKMS key + # Configure Vault to unseal storage using a GKMS key seal "gcpckms" { project = "vault-helm-dev-246514" region = "global" @@ -1189,10 +1198,13 @@ server: } ``` -Once you have successfully installed Vault, you will need to [initialize the Vault](https://learn.hashicorp.com/tutorials/vault/getting-started-deploy#initializing-the-vault) -and obtain the initial root token. You will need access to your Kubernetes cluster that Vault has been deployed into in order to do this. -To initialize the Vault, get a shell to one of the Vault pods running inside Kubernetes (typically this is done by using the `kubectl` command line tool). -Once you have a shell into the pod, run the `vault operator init` command: +Once you have successfully installed Vault, you must +[initialize the Vault](https://learn.hashicorp.com/tutorials/vault/getting-started-deploy#initializing-the-vault) +and obtain the initial root token. You need access to your Kubernetes cluster that +Vault has been deployed into in order to do this. To initialize the Vault, get a +shell to one of the Vault pods running inside Kubernetes (typically this is done +by using the `kubectl` command line tool). Once you have a shell into the pod, +run the `vault operator init` command: ```shell kubectl -n gitlab-managed-apps exec -it vault-0 sh @@ -1200,11 +1212,13 @@ kubectl -n gitlab-managed-apps exec -it vault-0 sh ``` This should give you your unseal keys and initial root token. Make sure to note these down -and keep these safe as you will need them to unseal the Vault throughout its lifecycle. +and keep these safe, as they're required to unseal the Vault throughout its lifecycle. -NOTE: **Note:** -Support for installing the Vault managed application is provided by the GitLab Release Management group. -If you run into unknown issues, please [open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new) and ping at least 2 people from the [Release Management group](https://about.gitlab.com/handbook/product/product-categories/#release-management-group). +Support for installing the Vault managed application is provided by the +GitLab Release Management group. If you run into unknown issues, +[open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new), and ping at +least 2 people from the +[Release Management group](https://about.gitlab.com/handbook/product/product-categories/#release-management-group). ### Install JupyterHub using GitLab CI/CD @@ -1224,7 +1238,7 @@ In the configuration: - `gitlabProjectIdWhitelist` restricts GitLab authentication to only members of the specified projects. - `gitlabGroupWhitelist` restricts GitLab authentication to only members of the specified groups. -- Specifying an empty array for both will allow any user on the GitLab instance to sign in. +- Specifying an empty array for both allows any user on the GitLab instance to sign in. JupyterHub is installed into the `gitlab-managed-apps` namespace of your cluster. @@ -1236,17 +1250,19 @@ Set: In addition, the following variables must be specified using [CI variables](../../ci/variables/README.md): -| CI Variable | Description | -|:---------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `JUPYTERHUB_PROXY_SECRET_TOKEN` | Secure string used for signing communications from the hub. See[`proxy.secretToken`](https://zero-to-jupyterhub.readthedocs.io/en/stable/reference/reference.html#proxy-secrettoken). | -| `JUPYTERHUB_COOKIE_SECRET` | Secure string used for signing secure cookies. See [`hub.cookieSecret`](https://zero-to-jupyterhub.readthedocs.io/en/stable/reference/reference.html#hub-cookiesecret). | -| `JUPYTERHUB_HOST` | Hostname used for the installation. For example, `jupyter.gitlab.example.com`. | -| `JUPYTERHUB_GITLAB_HOST` | Hostname of the GitLab instance used for authentication. For example, `gitlab.example.com`. | -| `JUPYTERHUB_AUTH_CRYPTO_KEY` | A 32-byte encryption key used to set [`auth.state.cryptoKey`](https://zero-to-jupyterhub.readthedocs.io/en/stable/reference/reference.html#auth-state-cryptokey). | -| `JUPYTERHUB_AUTH_GITLAB_CLIENT_ID` | "Application ID" for the OAuth Application. | -| `JUPYTERHUB_AUTH_GITLAB_CLIENT_SECRET` | "Secret" for the OAuth Application. | - -By default, JupyterHub will be installed using a +- `JUPYTERHUB_PROXY_SECRET_TOKEN` - Secure string used for signing communications + from the hub. Read [`proxy.secretToken`](https://zero-to-jupyterhub.readthedocs.io/en/stable/reference/reference.html#proxy-secrettoken). +- `JUPYTERHUB_COOKIE_SECRET` - Secure string used for signing secure cookies. Read + [`hub.cookieSecret`](https://zero-to-jupyterhub.readthedocs.io/en/stable/reference/reference.html#hub-cookiesecret). +- `JUPYTERHUB_HOST` - Hostname used for the installation. For example, `jupyter.gitlab.example.com`. +- `JUPYTERHUB_GITLAB_HOST` - Hostname of the GitLab instance used for authentication. + For example, `gitlab.example.com`. +- `JUPYTERHUB_AUTH_CRYPTO_KEY` - A 32-byte encryption key used to set + [`auth.state.cryptoKey`](https://zero-to-jupyterhub.readthedocs.io/en/stable/reference/reference.html#auth-state-cryptokey). +- `JUPYTERHUB_AUTH_GITLAB_CLIENT_ID` - "Application ID" for the OAuth Application. +- `JUPYTERHUB_AUTH_GITLAB_CLIENT_SECRET` - "Secret" for the OAuth Application. + +By default, JupyterHub is installed using a [default values file](https://gitlab.com/gitlab-org/cluster-integration/cluster-applications/-/blob/master/src/default-data/jupyterhub/values.yaml.gotmpl). You can customize the installation of JupyterHub by defining a `.gitlab/managed-apps/jupyterhub/values.yaml` file in your cluster management project. @@ -1255,9 +1271,11 @@ Refer to the [chart reference](https://zero-to-jupyterhub.readthedocs.io/en/stable/reference/reference.html) for the available configuration options. -NOTE: **Note:** Support for installing the JupyterHub managed application is provided by the GitLab Configure group. -If you run into unknown issues, please [open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new) and ping at least 2 people from the [Configure group](https://about.gitlab.com/handbook/product/product-categories/#configure-group). +If you run into unknown issues, +[open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new), and ping at +least 2 people from the +[Configure group](https://about.gitlab.com/handbook/product/product-categories/#configure-group). ### Install Elastic Stack using GitLab CI/CD @@ -1275,20 +1293,25 @@ elasticStack: Elastic Stack is installed into the `gitlab-managed-apps` namespace of your cluster. -You can check the default [`values.yaml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/vendor/elastic_stack/values.yaml) we set for this chart. +You can check the default +[`values.yaml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/vendor/elastic_stack/values.yaml) +we set for this chart. You can customize the installation of Elastic Stack by defining `.gitlab/managed-apps/elastic-stack/values.yaml` file in your cluster management project. Refer to the -[chart](https://gitlab.com/gitlab-org/charts/elastic-stack) for the +[chart](https://gitlab.com/gitlab-org/charts/elastic-stack) for all available configuration options. NOTE: **Note:** -In this alpha implementation of installing Elastic Stack through CI, reading the environment logs through Elasticsearch is unsupported. This is supported if [installed via the UI](#elastic-stack). +In this alpha implementation of installing Elastic Stack through CI, reading the +environment logs through Elasticsearch is unsupported. This is supported if +[installed with the UI](#elastic-stack). -NOTE: **Note:** -Support for installing the Elastic Stack managed application is provided by the GitLab APM group. -If you run into unknown issues, please [open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new) and ping at least 2 people from the [APM group](https://about.gitlab.com/handbook/product/product-categories/#apm-group). +Support for installing the Elastic Stack managed application is provided by the +GitLab APM group. If you run into unknown issues, +[open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new), and ping at +least 2 people from the [APM group](https://about.gitlab.com/handbook/product/product-categories/#apm-group). ### Install Crossplane using GitLab CI/CD @@ -1316,37 +1339,40 @@ management project. Refer to the [chart](https://github.com/crossplane/crossplane/tree/master/cluster/charts/crossplane#configuration) for the available configuration options. Note that this link points to the documentation for the current development release, which may differ from the version you have installed. -NOTE: **Note:** Support for the Crossplane managed application is provided by the Crossplane team. -If you run into issues, please [open a support ticket](https://github.com/crossplane/crossplane/issues/new/choose) directly. +If you run into issues, +[open a support ticket](https://github.com/crossplane/crossplane/issues/new/choose) directly. ### Install Fluentd using GitLab CI/CD > [Introduced](https://gitlab.com/gitlab-org/cluster-integration/cluster-applications/-/merge_requests/76) in GitLab 12.10. -To install Fluentd into the `gitlab-managed-apps` namespace of your cluster using GitLab CI/CD, define the following configuration in `.gitlab/managed-apps/config.yaml`: +To install Fluentd into the `gitlab-managed-apps` namespace of your cluster using +GitLab CI/CD, define the following configuration in `.gitlab/managed-apps/config.yaml`: ```yaml Fluentd: installed: true ``` -You can also review the default values set for this chart in the [`values.yaml`](https://github.com/helm/charts/blob/master/stable/fluentd/values.yaml) file. +You can also review the default values set for this chart in the +[`values.yaml`](https://github.com/helm/charts/blob/master/stable/fluentd/values.yaml) file. You can customize the installation of Fluentd by defining `.gitlab/managed-apps/fluentd/values.yaml` file in your cluster management project. Refer to the [configuration chart for the current development release of Fluentd](https://github.com/helm/charts/tree/master/stable/fluentd#configuration) -for the available configuration options. +for all available configuration options. -NOTE: **Note:** The configuration chart link points to the current development release, which may differ from the version you have installed. To ensure compatibility, switch to the specific branch or tag you are using. -NOTE: **Note:** -Support for installing the Fluentd managed application is provided by the GitLab Container Security group. -If you run into unknown issues, please [open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new) and ping at least 2 people from the [Container Security group](https://about.gitlab.com/handbook/product/product-categories/#container-security-group). +Support for installing the Fluentd managed application is provided by the +GitLab Container Security group. If you run into unknown issues, +[open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new), and ping at +least 2 people from the +[Container Security group](https://about.gitlab.com/handbook/product/product-categories/#container-security-group). ### Install Knative using GitLab CI/CD @@ -1360,7 +1386,7 @@ knative: You can customize the installation of Knative by defining `.gitlab/managed-apps/knative/values.yaml` file in your cluster management project. Refer to the [chart](https://gitlab.com/gitlab-org/charts/knative) -for the available configuration options. +for all available configuration options. Here is an example configuration for Knative: @@ -1368,11 +1394,14 @@ Here is an example configuration for Knative: domain: 'my.wildcard.A.record.dns' ``` -If you plan to use GitLab Serverless capabilities, be sure to set an A record wildcard domain on your custom configuration. +If you plan to use GitLab Serverless capabilities, be sure to set an `A record` +wildcard domain on your custom configuration. -NOTE: **Note:** -Support for installing the Knative managed application is provided by the GitLab Configure group. -If you run into unknown issues, please [open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new) and ping at least 2 people from the [Configure group](https://about.gitlab.com/handbook/product/product-categories/#configure-group). +Support for installing the Knative managed application is provided by the +GitLab Configure group. If you run into unknown issues, +[open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new), and ping at +least 2 people from the +[Configure group](https://about.gitlab.com/handbook/product/product-categories/#configure-group). #### Knative Metrics @@ -1398,14 +1427,16 @@ kubectl delete -f https://gitlab.com/gitlab-org/cluster-integration/cluster-appl > [Introduced](https://gitlab.com/gitlab-org/cluster-integration/cluster-applications/-/merge_requests/100) in GitLab 13.1. -To install AppArmor into the `gitlab-managed-apps` namespace of your cluster using GitLab CI/CD, define the following configuration in `.gitlab/managed-apps/config.yaml`: +To install AppArmor into the `gitlab-managed-apps` namespace of your cluster using +GitLab CI/CD, define the following configuration in `.gitlab/managed-apps/config.yaml`: ```yaml apparmor: installed: true ``` -You can define one or more AppArmor profiles by adding them into `.gitlab/managed-apps/apparmor/values.yaml` as the following: +You can define one or more AppArmor profiles by adding them into +`.gitlab/managed-apps/apparmor/values.yaml` as the following: ```yaml profiles: @@ -1419,25 +1450,27 @@ Refer to the [AppArmor chart](https://gitlab.com/gitlab-org/charts/apparmor) for #### Using AppArmor profiles in your deployments -After installing AppAmor, you can use profiles by adding Pod Annotations. If you're using Auto -DevOps, you can [customize `auto-deploy-values.yaml`](../../topics/autodevops/customize.md#customize-values-for-helm-chart) -to annotate your pods. Although it's helpful to be aware of the [list of custom attributes](https://gitlab.com/gitlab-org/cluster-integration/auto-deploy-image/-/tree/master/assets/auto-deploy-app#gitlabs-auto-deploy-helm-chart), you're only required to set -`podAnnotations` as follows: +After installing AppAmor, you can use profiles by adding Pod Annotations. If you're using +Auto DevOps, you can [customize `auto-deploy-values.yaml`](../../topics/autodevops/customize.md#customize-values-for-helm-chart) +to annotate your pods. Although it's helpful to be aware of the +[list of custom attributes](https://gitlab.com/gitlab-org/cluster-integration/auto-deploy-image/-/tree/master/assets/auto-deploy-app#gitlabs-auto-deploy-helm-chart), +you're only required to set `podAnnotations` as follows: ```yaml podAnnotations: container.apparmor.security.beta.kubernetes.io/auto-deploy-app: localhost/profile-one ``` -The only information to be changed here is the profile name which is `profile-one` in this example. Refer to the [AppArmor tutorial](https://kubernetes.io/docs/tutorials/clusters/apparmor/#securing-a-pod) for more information on how AppArmor is integrated in Kubernetes. +The only information to be changed here is the profile name which is `profile-one` +in this example. Refer to the [AppArmor tutorial](https://kubernetes.io/docs/tutorials/clusters/apparmor/#securing-a-pod) +for more information on how AppArmor is integrated in Kubernetes. #### Using PodSecurityPolicy in your deployments -NOTE: **Note:** To enable AppArmor annotations on a Pod Security Policy you must first -load the correspondingAppArmor profile. +load the corresponding AppArmor profile. -[Pod Security Policies](https://kubernetes.io/docs/concepts/policy/pod-security-policy/)are +[Pod Security Policies](https://kubernetes.io/docs/concepts/policy/pod-security-policy/) are resources at the cluster level that control security-related properties of deployed pods. You can use such a policy to enable loaded AppArmor profiles and apply necessary pod restrictions across a @@ -1465,14 +1498,14 @@ securityPolicies: - '*' ``` -This example creates a single policy named `example` with the provided -specification, and enables [AppArmor -annotations](https://kubernetes.io/docs/tutorials/clusters/apparmor/#podsecuritypolicy-annotations)on -it. +This example creates a single policy named `example` with the provided specification, +and enables [AppArmor annotations](https://kubernetes.io/docs/tutorials/clusters/apparmor/#podsecuritypolicy-annotations) on it. -NOTE: **Note:** -Support for installing the AppArmor managed application is provided by the GitLab Container Security group. -If you run into unknown issues, please [open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new) and ping at least 2 people from the [Container Security group](https://about.gitlab.com/handbook/product/product-categories/#container-security-group). +Support for installing the AppArmor managed application is provided by the +GitLab Container Security group. If you run into unknown issues, +[open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new), and ping +at least 2 people from the +[Container Security group](https://about.gitlab.com/handbook/product/product-categories/#container-security-group). ## Browse applications logs @@ -1500,9 +1533,7 @@ To upgrade an application: 1. Select your cluster. 1. If an upgrade is available, the **Upgrade** button is displayed. Click the button to upgrade. -NOTE: **Note:** -Upgrades will reset values back to the values built into the `runner` -chart plus the values set by +Upgrades reset values back to the values built into the `runner` chart, plus the values set by [`values.yaml`](https://gitlab.com/gitlab-org/gitlab/blob/master/vendor/runner/values.yaml) ## Uninstalling applications @@ -1513,16 +1544,16 @@ The applications below can be uninstalled. | Application | GitLab version | Notes | | ----------- | -------------- | ----- | -| cert-manager | 12.2+ | The associated private key will be deleted and cannot be restored. Deployed applications will continue to use HTTPS, but certificates will not be renewed. Before uninstalling, you may wish to [back up your configuration](https://cert-manager.io/docs/tutorials/backup/) or [revoke your certificates](https://letsencrypt.org/docs/revoking/). | -| GitLab Runner | 12.2+ | Any running pipelines will be canceled. | -| Helm | 12.2+ | The associated Tiller pod, the `gitlab-managed-apps` namespace, and all of its resources will be deleted and cannot be restored. | -| Ingress | 12.1+ | The associated load balancer and IP will be deleted and cannot be restored. Furthermore, it can only be uninstalled if JupyterHub is not installed. | -| JupyterHub | 12.1+ | All data not committed to GitLab will be deleted and cannot be restored. | -| Knative | 12.1+ | The associated IP will be deleted and cannot be restored. | -| Prometheus | 11.11+ | All data will be deleted and cannot be restored. | -| Crossplane | 12.5+ | All data will be deleted and cannot be restored. | -| Elastic Stack | 12.7+ | All data will be deleted and cannot be restored. | -| Sentry | 12.6+ | The PostgreSQL persistent volume will remain and should be manually removed for complete uninstall. | +| cert-manager | 12.2+ | The associated private key is deleted and cannot be restored. Deployed applications continue to use HTTPS, but certificates aren't renewed. Before uninstalling, you may want to [back up your configuration](https://cert-manager.io/docs/tutorials/backup/) or [revoke your certificates](https://letsencrypt.org/docs/revoking/). | +| GitLab Runner | 12.2+ | Any running pipelines are canceled. | +| Helm | 12.2+ | The associated Tiller pod, the `gitlab-managed-apps` namespace, and all of its resources are deleted and cannot be restored. | +| Ingress | 12.1+ | The associated load balancer and IP are deleted and cannot be restored. Furthermore, it can only be uninstalled if JupyterHub is not installed. | +| JupyterHub | 12.1+ | All data not committed to GitLab are deleted and cannot be restored. | +| Knative | 12.1+ | The associated IP are deleted and cannot be restored. | +| Prometheus | 11.11+ | All data are deleted and cannot be restored. | +| Crossplane | 12.5+ | All data are deleted and cannot be restored. | +| Elastic Stack | 12.7+ | All data are deleted and cannot be restored. | +| Sentry | 12.6+ | The PostgreSQL persistent volume remains and should be manually removed for complete uninstall. | To uninstall an application: @@ -1535,8 +1566,7 @@ To uninstall an application: 1. Click the **Uninstall** button for the application. Support for uninstalling all applications is planned for progressive rollout. -To follow progress, see [the relevant -epic](https://gitlab.com/groups/gitlab-org/-/epics/1201). +To follow progress, see the [relevant epic](https://gitlab.com/groups/gitlab-org/-/epics/1201). ## Troubleshooting applications @@ -1550,9 +1580,10 @@ To avoid installation errors: - Before starting the installation of applications, make sure that time is synchronized between your GitLab server and your Kubernetes cluster. -- Ensure certificates are not out of sync. When installing applications, GitLab expects a new cluster with no previous installation of Helm. +- Ensure certificates are not out of sync. When installing applications, GitLab + expects a new cluster with no previous installation of Helm. - You can confirm that the certificates match via `kubectl`: + You can confirm that the certificates match by using `kubectl`: ```shell kubectl get configmaps/values-content-configuration-ingress -n gitlab-managed-apps -o \ @@ -1590,5 +1621,5 @@ Installing Prometheus is failing with the following error: Error: Could not get apiVersions from Kubernetes: unable to retrieve the complete list of server APIs: admission.certmanager.k8s.io/v1beta1: the server is currently unable to handle the request ``` -This is a bug that was introduced in Helm `2.15` and fixed in `3.0.2`. As a workaround, you'll need -to make sure that [`cert-manager`](#cert-manager) is installed successfully prior to installing Prometheus. +This is a bug that was introduced in Helm `2.15` and fixed in `3.0.2`. As a workaround, +ensure [`cert-manager`](#cert-manager) is installed successfully prior to installing Prometheus. diff --git a/doc/user/clusters/cost_management.md b/doc/user/clusters/cost_management.md new file mode 100644 index 00000000000..f13be15c6bc --- /dev/null +++ b/doc/user/clusters/cost_management.md @@ -0,0 +1,74 @@ +--- +stage: Configure +group: Configure +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Cluster cost management **(ULTIMATE)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/216737) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.5. + +Cluster cost management provides insights into cluster resource usage. GitLab provides an example +[`kubecost-cost-model`](https://gitlab.com/gitlab-examples/kubecost-cost-model/) +project that uses GitLab's Prometheus integration and +[Kubecost's `cost-model`](https://github.com/kubecost/cost-model) to provide cluster cost +insights within GitLab: + + + +## Configure cluster cost management + +To get started with cluster cost management, you need [Maintainer](../permissions.md) +permissions in a project or group. + +1. Clone the [`kubecost-cost-model`](https://gitlab.com/gitlab-examples/kubecost-cost-model/) + example repository, which contains minor modifications to the upstream Kubecost + `cost-model` project: + - Configures your Prometheus endpoint to the GitLab-managed Prometheus. You may + need to change this value if you use a non-managed Prometheus. + - Adds the necessary annotations to the deployment manifest to be scraped by + GitLab-managed Prometheus. + - Changes the Google Pricing API access key to GitLab's access key. + - Contains definitions for a custom GitLab Metrics dashboard to show the cost insights. +1. Connect GitLab with Prometheus, depending on your configuration: + - *If Prometheus is already configured,* navigate to **Settings > Integrations > Prometheus** + to provide the API endpoint of your Prometheus server. + - *For GitLab-managed Prometheus,* navigate to your cluster's **Details** page, + select the **Applications** tab, and install Prometheus. The integration is + auto-configured for you. +1. Set up the Prometheus integration on the cloned example project. +1. Add the Kubecost `cost-model` to your cluster: + - *For non-managed clusters*, deploy it with GitLab CI/CD. + - *To deploy it manually*, use the following commands: + + ```shell + kubectl create namespace cost-model + kubectl apply -f kubernetes/ --namespace cost-model + ``` + +To access the cost insights, navigate to **Operations > Metrics** and select +the `default_costs.yml` dashboard. You can [customize](#customize-the-cost-dashboard) +this dashboard. + +### Customize the cost dashboard + +You can customize the cost dashboard by editing the `.gitlab/dashboards/default_costs.yml` +file or creating similar dashboard configuration files. To learn more, read about +[customizing dashboards in our documentation](/ee/operations/metrics/dashboards/). + +#### Available metrics + +Metrics contain both instance and node labels. The instance label will be deprecated in a future version. + +- `node_cpu_hourly_cost` - Hourly cost per vCPU on this node. +- `node_gpu_hourly_cost` - Hourly cost per GPU on this node. +- `node_ram_hourly_cost` - Hourly cost per gigabyte of memory on this node. +- `node_total_hourly_cost` - Total node cost per hour. +- `container_cpu_allocation` - Average number of CPUs requested/used over the previous minute. +- `container_gpu_allocation` - Average number of GPUs requested over the previous minute. +- `container_memory_allocation_bytes` - Average bytes of RAM requested/used over the previous minute. +- `pod_pvc_allocation` - Bytes provisioned for a PVC attached to a pod. +- `pv_hourly_cost` - Hourly cost per GB on a persistent volume. + +Some examples are provided in the +[`kubecost-cost-model` repository](https://gitlab.com/gitlab-examples/kubecost-cost-model/-/blob/master/PROMETHEUS.md#example-queries). diff --git a/doc/user/clusters/crossplane.md b/doc/user/clusters/crossplane.md index b30ebc57338..8463917f2f3 100644 --- a/doc/user/clusters/crossplane.md +++ b/doc/user/clusters/crossplane.md @@ -78,7 +78,6 @@ provided can manage resources in the `database.crossplane.io` API group: See [Configure Your Cloud Provider Account](https://crossplane.github.io/docs/v0.4/cloud-providers.html) to configure the installed cloud provider stack with a user account. -NOTE: **Note:** The Secret, and the Provider resource referencing the Secret, must be applied to the `gitlab-managed-apps` namespace in the guide. Make sure you change that while following the process. diff --git a/doc/user/clusters/environments.md b/doc/user/clusters/environments.md index 2b342ceb06d..3ab20c5466e 100644 --- a/doc/user/clusters/environments.md +++ b/doc/user/clusters/environments.md @@ -43,6 +43,5 @@ Once you have successful deployments to your group-level or instance-level clust 1. Navigate to your group's **Kubernetes** page. 1. Click on the **Environments** tab. -NOTE: **Note:** -Only successful deployments to the cluster is included in this page. -Non-cluster environments will not be included. +Only successful deployments to the cluster are included in this page. +Non-cluster environments aren't included. diff --git a/doc/user/clusters/img/kubecost_v13_5.png b/doc/user/clusters/img/kubecost_v13_5.png Binary files differnew file mode 100644 index 00000000000..a93e2531b16 --- /dev/null +++ b/doc/user/clusters/img/kubecost_v13_5.png diff --git a/doc/user/compliance/compliance_dashboard/img/compliance_dashboard_v13_3_1.png b/doc/user/compliance/compliance_dashboard/img/compliance_dashboard_v13_3_1.png Binary files differindex a06f8812b41..89f4e917567 100644 --- a/doc/user/compliance/compliance_dashboard/img/compliance_dashboard_v13_3_1.png +++ b/doc/user/compliance/compliance_dashboard/img/compliance_dashboard_v13_3_1.png diff --git a/doc/user/compliance/license_compliance/img/license-check_v13_4.png b/doc/user/compliance/license_compliance/img/license-check_v13_4.png Binary files differindex d3658cbaa18..bc80f938395 100644 --- a/doc/user/compliance/license_compliance/img/license-check_v13_4.png +++ b/doc/user/compliance/license_compliance/img/license-check_v13_4.png diff --git a/doc/user/compliance/license_compliance/index.md b/doc/user/compliance/license_compliance/index.md index 79c2d97b972..894c0e14862 100644 --- a/doc/user/compliance/license_compliance/index.md +++ b/doc/user/compliance/license_compliance/index.md @@ -121,12 +121,17 @@ The results will be saved as a [License Compliance report artifact](../../../ci/pipelines/job_artifacts.md#artifactsreportslicense_scanning) that you can later download and analyze. Due to implementation limitations, we always take the latest License Compliance artifact available. Behind the scenes, the -[GitLab License Compliance Docker image](https://gitlab.com/gitlab-org/security-products/license-management) +[GitLab License Compliance Docker image](https://gitlab.com/gitlab-org/security-products/analyzers/license-finder) is used to detect the languages/frameworks and in turn analyzes the licenses. The License Compliance settings can be changed through [environment variables](#available-variables) by using the [`variables`](../../../ci/yaml/README.md#variables) parameter in `.gitlab-ci.yml`. +### When License Compliance runs + +When using the GitLab `License-Scanning.gitlab-ci.yml` template, the License Compliance job doesn't +wait for other stages to complete. + ### Available variables License Compliance can be configured using environment variables. @@ -446,7 +451,7 @@ package manager. For a comprehensive list, consult [the Conan documentation](htt The default [Conan](https://conan.io/) configuration sets [`CONAN_LOGIN_USERNAME`](https://docs.conan.io/en/latest/reference/env_vars.html#conan-login-username-conan-login-username-remote-name) to `ci_user`, and binds [`CONAN_PASSWORD`](https://docs.conan.io/en/latest/reference/env_vars.html#conan-password-conan-password-remote-name) to the [`CI_JOB_TOKEN`](../../../ci/variables/predefined_variables.md) -for the running job. This allows Conan projects to fetch packages from a [GitLab Conan Repository](../../packages/conan_repository/#fetching-conan-package-information-from-the-gitlab-package-registry) +for the running job. This allows Conan projects to fetch packages from a [GitLab Conan Repository](../../packages/conan_repository/#fetch-conan-package-information-from-the-package-registry) if a GitLab remote is specified in the `.conan/remotes.json` file. To override the default credentials specify a [`CONAN_LOGIN_USERNAME_{REMOTE_NAME}`](https://docs.conan.io/en/latest/reference/env_vars.html#conan-login-username-conan-login-username-remote-name) @@ -629,7 +634,7 @@ import the following default License Compliance analyzer images from `registry.g offline [local Docker container registry](../../packages/container_registry/index.md): ```plaintext -registry.gitlab.com/gitlab-org/security-products/license-management:latest +registry.gitlab.com/gitlab-org/security-products/analyzers/license-finder:latest ``` The process for importing Docker images into a local offline Docker registry depends on diff --git a/doc/user/gitlab_com/index.md b/doc/user/gitlab_com/index.md index 67dd31efe2c..ec0c207e190 100644 --- a/doc/user/gitlab_com/index.md +++ b/doc/user/gitlab_com/index.md @@ -38,6 +38,14 @@ The IP address for `mg.gitlab.com` is subject to change at any time. [See our backup strategy](https://about.gitlab.com/handbook/engineering/infrastructure/production/#backups). +There are several ways to perform backups of your content on GitLab.com. + +Projects can be backed up in their entirety by exporting them either [through the UI](../project/settings/import_export.md) or [API](../../api/project_import_export.md#schedule-an-export), the latter of which can be used to programmatically upload exports to a storage platform such as AWS S3. + +With exports, be sure to take note of [what is and is not](../project/settings/import_export.md#exported-contents), included in a project export. + +Since GitLab is built on Git, you can back up **just** the repository of a project by [cloning](../../gitlab-basics/start-using-git.md#clone-a-repository) it to another machine. Similarly, if you need to back up just the wiki of a repository it can also be cloned and all files uploaded to that wiki will come with it [if they were uploaded after 2020-08-22](../project/wiki/index.md#creating-a-new-wiki-page). + ## Alternative SSH port GitLab.com can be reached via a [different SSH port](https://about.gitlab.com/blog/2016/02/18/gitlab-dot-com-now-supports-an-alternate-git-plus-ssh-port/) for `git+ssh`. @@ -88,6 +96,7 @@ Below are the current settings regarding [GitLab CI/CD](../../ci/README.md). | [Max pipeline schedules in projects](../../administration/instance_limits.md#number-of-pipeline-schedules) | `10` for Free tier, `50` for all paid tiers | Unlimited | | [Max number of instance level variables](../../administration/instance_limits.md#number-of-instance-level-variables) | `25` | `25` | | [Scheduled Job Archival](../../user/admin_area/settings/continuous_integration.md#archive-jobs) | 3 months | Never | +| Max test cases per [unit test report](../../ci/unit_test_reports.md) | `500_000` | Unlimited | ## Account and limit settings @@ -474,6 +483,10 @@ More information on this particular change can be found at of proposed changes can be found at <https://gitlab.com/gitlab-com/infrastructure/-/issues?scope=all&utf8=%E2%9C%93&state=opened&label_name[]=database&label_name[]=change>. +## Puma + +GitLab.com uses the default of 60 seconds for [Puma request timeouts](https://docs.gitlab.com/omnibus/settings/puma.html#worker-timeout). + ## Unicorn GitLab.com adjusts the memory limits for the [unicorn-worker-killer](https://rubygems.org/gems/unicorn-worker-killer) gem. @@ -529,9 +542,9 @@ Source: For performance reasons, if a query returns more than 10,000 records, GitLab doesn't return the following headers: -- `X-Total`. -- `X-Total-Pages`. -- `rel="last"` `Link`. +- `x-total`. +- `x-total-pages`. +- `rel="last"` `link`. ### Rack Attack initializer @@ -596,6 +609,11 @@ dropped and users get To help avoid abuse, project and group imports, exports, and export downloads are rate limited. See [Project import/export rate limits](../../user/project/settings/import_export.md#rate-limits) and [Group import/export rate limits](../../user/group/settings/import_export.md#rate-limits) for details. +### Non-configurable limits + +See [non-configurable limits](../../security/rate_limits.md#non-configurable-limits) for information on +rate limits that are not configurable, and therefore also used on GitLab.com. + ## GitLab.com Logging We use [Fluentd](https://gitlab.com/gitlab-com/runbooks/tree/master/logging/doc#fluentd) to parse our logs. Fluentd sends our logs to diff --git a/doc/user/group/clusters/index.md b/doc/user/group/clusters/index.md index ebf38aef4a6..1a62d67e468 100644 --- a/doc/user/group/clusters/index.md +++ b/doc/user/group/clusters/index.md @@ -14,6 +14,9 @@ Similar to [project-level](../../project/clusters/index.md) and group-level Kubernetes clusters allow you to connect a Kubernetes cluster to your group, enabling you to use the same cluster across multiple projects. +To view your group level Kubernetes clusters, navigate to your project and select +**Kubernetes** from the left-hand menu. + ## Installing applications GitLab can install and manage some applications in your group-level @@ -69,9 +72,8 @@ for deployments with a cluster not managed by GitLab, you must ensure: (this is [not automatic](https://gitlab.com/gitlab-org/gitlab/-/issues/31519)). Editing `KUBE_NAMESPACE` directly is discouraged. -NOTE: **Note:** If you [install applications](#installing-applications) on your cluster, GitLab creates -the resources required to run them even if you choose to manage your own cluster. +the resources required to run them, even if you choose to manage your own cluster. ### Clearing the cluster cache diff --git a/doc/user/group/contribution_analytics/index.md b/doc/user/group/contribution_analytics/index.md index bcc6d958427..a689b7c380b 100644 --- a/doc/user/group/contribution_analytics/index.md +++ b/doc/user/group/contribution_analytics/index.md @@ -9,8 +9,6 @@ info: To determine the technical writer assigned to the Stage/Group associated w > - Introduced in [GitLab Starter](https://about.gitlab.com/pricing/) 8.3. > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/3090) for subgroups in GitLab 12.2. -## Overview - With Contribution Analytics you can get an overview of the following activity in your group: diff --git a/doc/user/group/epics/index.md b/doc/user/group/epics/index.md index e8bcb7219fc..f380b36cc00 100644 --- a/doc/user/group/epics/index.md +++ b/doc/user/group/epics/index.md @@ -10,9 +10,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w > - Introduced in [GitLab Ultimate](https://about.gitlab.com/pricing/) 10.2. > - Single-level Epics [were moved](https://gitlab.com/gitlab-org/gitlab/-/issues/37081) to [GitLab Premium](https://about.gitlab.com/pricing/) in 12.8. -Epics let you manage your portfolio of projects more efficiently and with less -effort by tracking groups of issues that share a theme, across projects and -milestones. +Epics let you manage your portfolio of projects more efficiently by tracking groups of issues that +share a theme across projects and milestones. An epic's page contains the following tabs: @@ -22,7 +21,7 @@ An epic's page contains the following tabs: - Hover over the total counts to see a breakdown of open and closed items. NOTE: **Note:** - The number provided here includes all epics associated with this project. The number includes epics for which users may not currently have permission. + The number provided here includes all epics associated with this project. The number includes epics for which users may not yet have permission. - **Roadmap**: a roadmap view of child epics which have start and due dates. @@ -65,16 +64,19 @@ graph TD ``` See [Manage issues assigned to an epic](manage_epics.md#manage-issues-assigned-to-an-epic) for steps -to add an issue to an epic, reorder issues, move issues between epics, or promote an issue to an epic. +to: + +- Add an issue to an epic +- Reorder issues +- Move an issue between epics +- Promote an issue to an epic ## Issue health status in Epic tree **(ULTIMATE)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/199184) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.10. -> - The health status of a closed issue [will be hidden](https://gitlab.com/gitlab-org/gitlab/-/issues/220867) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.3 or later. +> - The health status of a closed issue [is hidden](https://gitlab.com/gitlab-org/gitlab/-/issues/220867) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.3 or later. -You can report on and quickly respond to the health of individual issues and epics by setting a -red, amber, or green [health status on an issue](../../project/issues/index.md#health-status), -which will appear on your Epic tree. +Report or respond to the health of issues and epics by setting a red, amber, or green [health status](../../project/issues/index.md#health-status), which then appears on your Epic tree. ### Disable Issue health status in Epic tree @@ -100,7 +102,7 @@ steps to create, move, reorder, or delete child epics. To set a **Start date** and **Due date** for an epic, select one of the following: - **Fixed**: Enter a fixed value. -- **From milestones**: Inherit a dynamic value from the milestones currently assigned to the epic's issues. +- **From milestones**: Inherit a dynamic value from the milestones that are assigned to the epic's issues. Note that GitLab 12.5 replaced this option with **Inherited**. - **Inherited**: Inherit a dynamic value from the epic's issues, child epics, and milestones ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7332) in GitLab 12.5 to replace **From milestones**). @@ -108,10 +110,10 @@ To set a **Start date** and **Due date** for an epic, select one of the followin > [Replaced](https://gitlab.com/gitlab-org/gitlab/-/issues/7332) in GitLab 12.5 by **Inherited**. -If you select **From milestones** for the start date, GitLab will automatically set the date to be earliest -start date across all milestones that are currently assigned to the issues that are added to the epic. -Similarly, if you select **From milestones** for the due date, GitLab will set it to be the latest due date across -all milestones that are currently assigned to those issues. +If you select **From milestones** for the start date, GitLab automatically sets the date to be earliest +start date across all milestones that are assigned to the issues that belong to the epic. +If you select **From milestones** for the due date, GitLab sets the date to be the latest due date across +all milestones that are assigned to those issues. These are dynamic dates which are recalculated if any of the following occur: @@ -138,8 +140,8 @@ These are dynamic dates and recalculated if any of the following occur: - Issues are added to, or removed from, the epic. Because the epic's dates can inherit dates from its children, the start date and due date propagate from the bottom to the top. -If the start date of a child epic on the lowest level changes, that becomes the earliest possible start date for its parent epic, -then the parent epic's start date will reflect the change and this will propagate upwards to the top epic. +If the start date of a child epic on the lowest level changes, that becomes the earliest possible start date for its parent epic. +The parent epic's start date then reflects this change and propagates upwards to the top epic. ## Roadmap in epics @@ -153,11 +155,11 @@ have a [start or due date](#start-date-and-due-date), a ## Permissions -If you have access to view an epic and have access to view an issue already -added to that epic, then you can view the issue in the epic issue list. +If you have access to view an epic and an issue added to that epic, you can view the issue in the +epic's issue list. -If you have access to edit an epic and have access to edit an issue, then you -can add the issue to or remove it from the epic. +If you have access to edit an epic and an issue added to that epic, you can add the issue to or +remove it from the epic. Note that for a given group, the visibility of all projects must be the same as the group, or less restrictive. That means if you have access to a group's epic, @@ -175,8 +177,8 @@ You can also consult the [group permissions table](../../permissions.md#group-me Once you write your comment, you can either: -- Click **Comment**, and your comment will be published. -- Click **Start thread**, and you will start a thread within that epic's discussion. +- Click **Comment** to publish your comment. +- Click **Start thread** to start a thread within that epic's discussion. ### Activity sort order diff --git a/doc/user/group/index.md b/doc/user/group/index.md index 32b76cf9280..2c838724cb3 100644 --- a/doc/user/group/index.md +++ b/doc/user/group/index.md @@ -371,7 +371,7 @@ To create group links via filter: ### Overriding user permissions **(STARTER ONLY)** -Since GitLab [v8.15](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/822) LDAP user permissions can now be manually overridden by an admin user. To override a user's permissions: +In GitLab [8.15](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/822) and later, LDAP user permissions can now be manually overridden by an admin user. To override a user's permissions: 1. Go to your group's **Members** page. 1. Select the pencil icon in the row for the user you are editing. @@ -391,11 +391,56 @@ milestones. [Learn more about Epics.](epics/index.md) +## Group wikis **(PREMIUM)** + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/13195) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.5. +> - It's [deployed behind a feature flag](../feature_flags.md), enabled by default. +> - It's enabled on GitLab.com. +> - It's recommended for production use. +> - For GitLab self-managed instances, GitLab administrators can opt to [disable it](#enable-or-disable-group-wikis). + +CAUTION: **Warning:** +This feature might not be available to you. Check the **version history** note above for details. + +Group wikis work the same way as [project wikis](../project/wiki/index.md), please refer to those docs for details on usage. + +Group wikis can be edited by members with [Developer permissions](../../user/permissions.md#group-members-permissions) +and above. + +### Group wikis limitations + +There are a few limitations compared to project wikis: + +- Local Git access is not supported yet. +- Group wikis are not included in global search, group exports, backups, and Geo replication. +- Changes to group wikis don't show up in the group's activity feed. + +You can follow [this epic](https://gitlab.com/groups/gitlab-org/-/epics/2782) for updates. + +### Enable or disable group wikis **(CORE ONLY)** + +Group wikis are under development but ready for production use. +It is deployed behind a feature flag that is **enabled by default**. +[GitLab administrators with access to the GitLab Rails console](../../administration/feature_flags.md) +can opt to disable it for your instance. + +To enable it: + +```ruby +Feature.enable(:group_wikis) +``` + +To disable it: + +```ruby +Feature.disable(:group_wikis) +``` + ## Group Security Dashboard **(ULTIMATE)** Get an overview of the vulnerabilities of all the projects in a group and its subgroups. -[Learn more about the Group Security Dashboard.](security_dashboard/index.md) +[Learn more about the Group Security Dashboard.](../application_security/security_dashboard/index.md) ## Insights **(ULTIMATE)** diff --git a/doc/user/group/iterations/index.md b/doc/user/group/iterations/index.md index 20cbc043d83..2eb50f07de3 100644 --- a/doc/user/group/iterations/index.md +++ b/doc/user/group/iterations/index.md @@ -64,6 +64,15 @@ To edit an iteration, click the three-dot menu (**{ellipsis_v}**) > **Edit itera To learn how to add an issue to an iteration, see the steps in [Managing issues](../../project/issues/managing_issues.md#add-an-issue-to-an-iteration). +## View an iteration report + +> Viewing iteration reports in projects [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/222763) in [GitLab Starter](https://about.gitlab.com/pricing/) 13.5. + +You can track the progress of an iteration by reviewing iteration reports. +An iteration report displays a list of all the issues assigned to an iteration and their status. + +To view an iteration report, go to the iterations list page and click an iteration's title. + ## Disable Iterations **(CORE ONLY)** GitLab Iterations feature is deployed with a feature flag that is **enabled by default**. diff --git a/doc/user/group/repositories_analytics/index.md b/doc/user/group/repositories_analytics/index.md index b013e371ed2..c9a10146440 100644 --- a/doc/user/group/repositories_analytics/index.md +++ b/doc/user/group/repositories_analytics/index.md @@ -1,17 +1,13 @@ --- type: reference stage: Verify -group: Analytics +group: Testing info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # Repositories Analytics **(PREMIUM)** -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/215104) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.4. -> - It's [deployed behind a feature flag](../../feature_flags.md), enabled by default. -> - It's enabled on GitLab.com. -> - It's recommended for production use. -> - For GitLab self-managed instances, GitLab administrators can opt to [disable it](#enable-or-disable-repositories-analytics). **(CORE ONLY)** +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/215104) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.4. CAUTION: **Warning:** This feature might not be available to you. Check the **version history** note above for details. @@ -35,25 +31,6 @@ For each day that a coverage report was generated by a job in a project's pipeli If the project's code coverage was calculated more than once in a day, we will take the last value from that day. -## Enable or disable Repositories Analytics **(CORE ONLY)** - -Repositories Analytics is under development but ready for production use. -It is deployed behind a feature flag that is **enabled by default**. -[GitLab administrators with access to the GitLab Rails console](../../../administration/feature_flags.md) -can opt to disable it. - -To enable it: - -```ruby -Feature.enable(:group_coverage_reports) -``` - -To disable it: - -```ruby -Feature.disable(:group_coverage_reports) -``` - <!-- ## Troubleshooting Include any troubleshooting steps that you can foresee. If you know beforehand what issues diff --git a/doc/user/group/saml_sso/index.md b/doc/user/group/saml_sso/index.md index 57b9cc92c51..3cb566c7f77 100644 --- a/doc/user/group/saml_sso/index.md +++ b/doc/user/group/saml_sso/index.md @@ -5,7 +5,7 @@ group: Access info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# SAML SSO for GitLab.com groups **(PREMIUM)** +# SAML SSO for GitLab.com groups **(SILVER ONLY)** > Introduced in GitLab 11.0. @@ -256,53 +256,6 @@ For example, to unlink the `MyOrg` account, the following **Disconnect** button | Issuer | How GitLab identifies itself to the identity provider. Also known as a "Relying party trust identifier". | | Certificate fingerprint | Used to confirm that communications over SAML are secure by checking that the server is signing communications with the correct certificate. Also known as a certificate thumbprint. | -## Configuring on a self-managed GitLab instance **(PREMIUM ONLY)** - -For self-managed GitLab instances we strongly recommend using the -[instance-wide SAML OmniAuth Provider](../../../integration/saml.md) instead. - -Group SAML SSO helps if you need to allow access via multiple SAML identity providers, but as a multi-tenant solution is less suited to cases where you administer your own GitLab instance. - -To proceed with configuring Group SAML SSO instead, you'll need to enable the `group_saml` OmniAuth provider. This can be done from: - -- `gitlab.rb` for [Omnibus GitLab installations](#omnibus-installations). -- `gitlab/config/gitlab.yml` for [source installations](#source-installations). - -### Limitations - -Group SAML on a self-managed instance is limited when compared to the recommended -[instance-wide SAML](../../../integration/saml.md). The recommended solution allows you to take advantage of: - -- [LDAP compatibility](../../../administration/auth/ldap/index.md). -- [LDAP Group Sync](../index.md#manage-group-memberships-via-ldap). -- [Required groups](../../../integration/saml.md#required-groups). -- [Admin groups](../../../integration/saml.md#admin-groups). -- [Auditor groups](../../../integration/saml.md#auditor-groups). - -### Omnibus installations - -1. Make sure GitLab is - [configured with HTTPS](../../../install/installation.md#using-https). -1. Enable OmniAuth and the `group_saml` provider in `gitlab.rb`: - - ```ruby - gitlab_rails['omniauth_enabled'] = true - gitlab_rails['omniauth_providers'] = [{ name: 'group_saml' }] - ``` - -### Source installations - -1. Make sure GitLab is - [configured with HTTPS](../../../install/installation.md#using-https). -1. Enable OmniAuth and the `group_saml` provider in `gitlab/config/gitlab.yml`: - - ```yaml - omniauth: - enabled: true - providers: - - { name: 'group_saml' } - ``` - ## Passwords for users created via SAML SSO for Groups The [Generated passwords for users created through integrated authentication](../../../security/passwords_for_integrated_authentication_methods.md) guide provides an overview of how GitLab generates and sets passwords for users created via SAML SSO for Groups. @@ -336,6 +289,11 @@ Similarly, group members of a role with the appropriate permissions can make use This can then be compared to the [NameID](#nameid) being sent by the Identity Provider by decoding the message with a [SAML debugging tool](#saml-debugging-tools). We require that these match in order to identify users. +### Users receive a 404 + +If a user is trying to sign in for the first time and the GitLab single sign-on URL has not [been configured](#configuring-your-identity-provider), they may see a 404. +As outlined in the [user access section](#linking-saml-to-your-existing-gitlabcom-account), a group Owner will need to provide the URL to users. + ### Message: "SAML authentication failed: Extern uid has already been taken" This error suggests you are signed in as a GitLab user but have already linked your SAML identity to a different GitLab user. Sign out and then try to sign in again using the SSO SAML link, which should log you into GitLab with the linked user account. diff --git a/doc/user/group/saml_sso/scim_setup.md b/doc/user/group/saml_sso/scim_setup.md index 4f74e672392..c6444ada165 100644 --- a/doc/user/group/saml_sso/scim_setup.md +++ b/doc/user/group/saml_sso/scim_setup.md @@ -17,7 +17,7 @@ GitLab's [SCIM API](../../../api/scim.md) implements part of [the RFC7644 protoc ## Features -Currently, the following actions are available: +The following actions are available: - Create users - Update users (Azure only) @@ -51,7 +51,7 @@ Once [Group Single Sign-On](index.md) has been configured, we can: The SAML application that was created during [Single sign-on](index.md) setup for [Azure](https://docs.microsoft.com/en-us/azure/active-directory/manage-apps/view-applications-portal) now needs to be set up for SCIM. -1. Check the configuration for your GitLab SAML app and ensure that **Name identifier value** (NameID) points to `user.objectid` or another unique identifier. This will match the `extern_uid` used on GitLab. +1. Check the configuration for your GitLab SAML app and ensure that **Name identifier value** (NameID) points to `user.objectid` or another unique identifier. This matches the `extern_uid` used on GitLab.  @@ -63,7 +63,7 @@ During this configuration, note the following: - The `Tenant URL` and `secret token` are the ones retrieved in the [previous step](#gitlab-configuration). - Should there be any problems with the availability of GitLab or similar - errors, the notification email set will get those. + errors, the notification email set gets those. - It is recommended to set a notification email and check the **Send an email notification when a failure occurs** checkbox. - For mappings, we will only leave `Synchronize Azure Active Directory Users to AppName` enabled. @@ -75,10 +75,10 @@ You can then test the connection by clicking on **Test Connection**. If the conn 1. Click **Delete** next to the `mail` mapping. 1. Map `userPrincipalName` to `emails[type eq "work"].value` and change its **Matching precedence** to `2`. 1. Map `mailNickname` to `userName`. -1. Determine how GitLab will uniquely identify users. +1. Determine how GitLab uniquely identifies users. - Use `objectId` unless users already have SAML linked for your group. - - If you already have users with SAML linked then use the `Name ID` value from the [SAML configuration](#azure). Using a different value will likely cause duplicate users and prevent users from accessing the GitLab group. + - If you already have users with SAML linked then use the `Name ID` value from the [SAML configuration](#azure). Using a different value may cause duplicate users and prevent users from accessing the GitLab group. 1. Create a new mapping: 1. Click **Add New Mapping**. @@ -110,14 +110,14 @@ You can then test the connection by clicking on **Test Connection**. If the conn NOTE: **Note:** You can control what is actually synced by selecting the `Scope`. For example, - `Sync only assigned users and groups` will only sync the users assigned to - the application (`Users and groups`), otherwise, it will sync the whole Active Directory. + `Sync only assigned users and groups` only syncs the users assigned to + the application (`Users and groups`), otherwise, it syncs the whole Active Directory. -Once enabled, the synchronization details and any errors will appear on the +Once enabled, the synchronization details and any errors appears on the bottom of the **Provisioning** screen, together with a link to the audit logs. CAUTION: **Warning:** -Once synchronized, changing the field mapped to `id` and `externalId` will likely cause provisioning errors, duplicate users, and prevent existing users from accessing the GitLab group. +Once synchronized, changing the field mapped to `id` and `externalId` may cause a number of errors. These include provisioning errors, duplicate users, and may prevent existing users from accessing the GitLab group. ### Okta configuration steps @@ -260,15 +260,9 @@ It is important that this SCIM `id` and SCIM `externalId` are configured to the Group owners can see the list of users and the `externalId` stored for each user in the group SAML SSO Settings page. -Alternatively, the [SCIM API](../../../api/scim.md#get-a-list-of-saml-users) can be used to manually retrieve the `externalId` we have stored for users, also called the `external_uid` or `NameId`. +A possible alternative is to use the [SCIM API](../../../api/scim.md#get-a-list-of-saml-users) to manually retrieve the `externalId` we have stored for users, also called the `external_uid` or `NameId`. -For example: - -```shell -curl 'https://gitlab.example.com/api/scim/v2/groups/GROUP_NAME/Users?startIndex=1"' --header "Authorization: Bearer <your_scim_token>" --header "Content-Type: application/scim+json" -``` - -To see how this compares to the value returned as the SAML NameId, you can have the user use a [SAML Tracer](index.md#saml-debugging-tools). +To see how the `external_uid` compares to the value returned as the SAML NameId, you can have the user use a [SAML Tracer](index.md#saml-debugging-tools). #### Update or fix mismatched SCIM externalId and SAML NameId @@ -285,15 +279,9 @@ you can address the problem in the following ways: - You can have users unlink and relink themselves, based on the ["SAML authentication failed: User has already been taken"](./index.md#message-saml-authentication-failed-user-has-already-been-taken) section. - You can unlink all users simultaneously, by removing all users from the SAML app while provisioning is turned on. -- You can use the [SCIM API](../../../api/scim.md#update-a-single-saml-user) to manually correct the `externalId` stored for users to match the SAML `NameId`. +- It may be possible to use the [SCIM API](../../../api/scim.md#update-a-single-saml-user) to manually correct the `externalId` stored for users to match the SAML `NameId`. To look up a user, you'll need to know the desired value that matches the `NameId` as well as the current `externalId`. -It is then possible to issue a manual SCIM#update request, for example: - -```shell -curl --verbose --request PATCH 'https://gitlab.com/api/scim/v2/groups/YOUR_GROUP/Users/OLD_EXTERNAL_UID' --data '{ "Operations": [{"op":"Replace","path":"externalId","value":"NEW_EXTERNAL_UID"}] }' --header "Authorization: Bearer <your_scim_token>" --header "Content-Type: application/scim+json" -``` - It is important not to update these to incorrect values, since this will cause users to be unable to sign in. It is also important not to assign a value to the wrong user, as this would cause users to get signed into the wrong account. #### I need to change my SCIM app diff --git a/doc/user/img/feature_flags_history_note_info_v13_2.png b/doc/user/img/feature_flags_history_note_info_v13_2.png Binary files differindex 403a6002603..07d096b6dde 100644 --- a/doc/user/img/feature_flags_history_note_info_v13_2.png +++ b/doc/user/img/feature_flags_history_note_info_v13_2.png diff --git a/doc/user/img/gitlab_snippet_v13_5.png b/doc/user/img/gitlab_snippet_v13_5.png Binary files differnew file mode 100644 index 00000000000..3fce1d25c3d --- /dev/null +++ b/doc/user/img/gitlab_snippet_v13_5.png diff --git a/doc/user/img/version_history_notes_collapsed_v13_2.png b/doc/user/img/version_history_notes_collapsed_v13_2.png Binary files differindex 42ea11ae8ff..b85c9cb36dd 100644 --- a/doc/user/img/version_history_notes_collapsed_v13_2.png +++ b/doc/user/img/version_history_notes_collapsed_v13_2.png diff --git a/doc/user/index.md b/doc/user/index.md index ce8713591ab..32a1c235882 100644 --- a/doc/user/index.md +++ b/doc/user/index.md @@ -46,8 +46,9 @@ GitLab is a Git-based platform that integrates a great number of essential tools - Building, testing, and deploying with built-in [Continuous Integration](../ci/README.md). - Deploying personal and professional static websites with [GitLab Pages](project/pages/index.md). - Integrating with Docker by using [GitLab Container Registry](packages/container_registry/index.md). -- Tracking the development lifecycle by using [GitLab Value Stream Analytics](project/cycle_analytics.md). +- Tracking the development lifecycle by using [GitLab Value Stream Analytics](analytics/value_stream_analytics.md). - Provide support with [Service Desk](project/service_desk.md). +- [Export issues as CSV](project/issues/csv_export.md). With GitLab Enterprise Edition, you can also: @@ -60,8 +61,7 @@ With GitLab Enterprise Edition, you can also: - Leverage [Elasticsearch](../integration/elasticsearch.md) with [Advanced Search](search/advanced_global_search.md) and [Advanced Search Syntax](search/advanced_search_syntax.md) for faster, more advanced code search across your entire GitLab instance. - [Authenticate users with Kerberos](../integration/kerberos.md). - [Mirror a repository](project/repository/repository_mirroring.md) from elsewhere on your local server. -- [Export issues as CSV](project/issues/csv_export.md). -- View your entire CI/CD pipeline involving more than one project with [Multiple-Project Pipelines](../ci/multi_project_pipeline_graphs.md). +- View your entire CI/CD pipeline involving more than one project with [Multiple-Project Pipelines](../ci/multi_project_pipelines.md). - [Lock files](project/file_lock.md) to prevent conflicts. - View the current health and status of each CI environment running on Kubernetes with [Deploy Boards](project/deploy_boards.md). - Leverage continuous delivery method with [Canary Deployments](project/canary_deployments.md). diff --git a/doc/user/infrastructure/index.md b/doc/user/infrastructure/index.md index 227a67f8c8a..cee6e21a5c5 100644 --- a/doc/user/infrastructure/index.md +++ b/doc/user/infrastructure/index.md @@ -13,6 +13,32 @@ workflows to tie into GitLab's authentication and authorization. These features lowering the barrier to entry for teams to adopt Terraform, collaborate effectively within GitLab, and support Terraform best practices. +## Quick Start + +Use the following `.gitlab-ci.yml` to set up a simple Terraform project integration +for GitLab versions 13.5 and greater: + +```yaml +include: + - template: Terraform.latest.gitlab-ci.yml + +variables: + # If not using GitLab's HTTP backend, remove this line and specify TF_HTTP_* variables + TF_STATE_NAME: default + TF_CACHE_KEY: default +``` + +This template uses `.latest.`, instead of stable, and may include breaking changes. +This template also includes some opinionated decisions, which you can override: + +- Including the latest [GitLab Terraform Image](https://gitlab.com/gitlab-org/terraform-images). +- Using the [GitLab managed Terraform State](#gitlab-managed-terraform-state) as + the Terraform state storage backend. +- Creating [four pipeline stages](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml): + `init`, `validate`, `build`, and `deploy`. These stages + [run the Terraform commands](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml) + `init`, `validate`, `plan`, `plan-json`, and `apply`. The `apply` command only runs on `master`. + ## GitLab managed Terraform State > [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/2673) in GitLab 13.0. @@ -67,8 +93,9 @@ local machine, this is a simple way to get started: 1. On your local machine, run `terraform init`, passing in the following options, replacing `<YOUR-STATE-NAME>`, `<YOUR-PROJECT-ID>`, `<YOUR-USERNAME>` and `<YOUR-ACCESS-TOKEN>` with the relevant values. This command initializes your - Terraform state, and stores that state within your GitLab project. This example - uses `gitlab.com`: + Terraform state, and stores that state within your GitLab project. The name of + your state can contain only uppercase and lowercase letters, decimal digits, + hyphens, and underscores. This example uses `gitlab.com`: ```shell terraform init \ @@ -198,7 +225,7 @@ recommends encrypting plan output or modifying the project visibility settings. ## Example project -See [this reference project](https://gitlab.com/nicholasklick/gitlab-terraform-aws) using GitLab and Terraform to deploy a basic AWS EC2 within a custom VPC. +See [this reference project](https://gitlab.com/gitlab-org/configure/examples/gitlab-terraform-aws) using GitLab and Terraform to deploy a basic AWS EC2 within a custom VPC. ## Copy Terraform state between backends @@ -294,15 +321,15 @@ rerun this command to reinitialize your working directory. If you forget, other commands will detect it and remind you to do so if necessary. ``` -If you type `yes`, it will copy your state from the old location to the new +If you type `yes`, it copies your state from the old location to the new location. You can then go back to running it from within GitLab CI. ## Output Terraform Plan information into a merge request Using the [GitLab Terraform Report artifact](../../ci/pipelines/job_artifacts.md#artifactsreportsterraform), you can expose details from `terraform plan` runs directly into a merge request widget, -enabling you to see statistics about the resources that Terraform will create, -modify, or destroy. +enabling you to see statistics about the resources that Terraform creates, +modifies, or destroys. Let's explore how to configure a GitLab Terraform Report artifact. You can either use a pre-built image which includes a `gitlab-terraform` helper as @@ -434,7 +461,8 @@ apply: ### Multiple Terraform Plan reports -Starting with 13.2, you can display multiple reports on the Merge Request page. The reports will also display the `artifacts: name:`. See example below for a suggested setup. +Starting with GitLab version 13.2, you can display multiple reports on the Merge Request +page. The reports also display the `artifacts: name:`. See example below for a suggested setup. ```yaml default: diff --git a/doc/user/instance/clusters/index.md b/doc/user/instance/clusters/index.md index 8059b8ca642..c370f9d8725 100644 --- a/doc/user/instance/clusters/index.md +++ b/doc/user/instance/clusters/index.md @@ -2,14 +2,14 @@ > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/39840) in GitLab 11.11. -## Overview - Similar to [project-level](../../project/clusters/index.md) and [group-level](../../group/clusters/index.md) Kubernetes clusters, instance-level Kubernetes clusters allow you to connect a Kubernetes cluster to the GitLab instance, which enables you to use the same cluster across multiple projects. +The instance level Kubernetes clusters can be found in the top menu by navigating to your instance's **{admin}** **Admin Area > Kubernetes**. + ## Cluster precedence GitLab will try to match clusters in the following order: diff --git a/doc/user/markdown.md b/doc/user/markdown.md index 03a4e4cb244..8b65da4ab94 100644 --- a/doc/user/markdown.md +++ b/doc/user/markdown.md @@ -78,13 +78,11 @@ character of the top list item (`C` in this case): - dark - milk -NOTE: **Note:** -We flag any significant differences between Redcarpet and CommonMark - Markdown in this document. +We flag any significant differences between Redcarpet and CommonMark Markdown in this document. If you have a large volume of Markdown files, it can be tedious to determine if they display correctly or not. You can use the -[diff_redcarpet_cmark](https://gitlab.com/digitalmoksha/diff_redcarpet_cmark) +[`diff_redcarpet_cmark`](https://gitlab.com/digitalmoksha/diff_redcarpet_cmark) tool (not an officially supported product) to generate a list of files and the differences between how RedCarpet and CommonMark render the files. It gives an indication if anything needs to be changed - often nothing needs @@ -126,7 +124,7 @@ changing how standard Markdown is used: ### Colors -> If this is not rendered correctly, [view it in GitLab itself](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/user/markdown.md#colors). +If this section is not rendered correctly, [view it in GitLab](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/user/markdown.md#colors). It's possible to have color written in HEX, RGB, or HSL format rendered with a color indicator. @@ -235,7 +233,7 @@ To make PlantUML available in GitLab, a GitLab administrator needs to enable it ### Emoji -> If this is not rendered correctly, [view it in GitLab itself](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/user/markdown.md#emoji). +If this section is not rendered correctly, [view it in GitLab](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/user/markdown.md#emoji). ```markdown Sometimes you want to :monkey: around a bit and add some :star2: to your :speech_balloon:. Well we have a gift for you: @@ -259,13 +257,14 @@ If you're new to this, don't be <img src="https://gitlab.com/gitlab-org/gitlab-f Consult the [Emoji Cheat Sheet](https://www.webfx.com/tools/emoji-cheat-sheet/) for a list of all supported emoji codes. <img src="https://gitlab.com/gitlab-org/gitlab-foss/raw/master/app/assets/images/emoji/thumbsup.png" width="20px" height="20px" style="display:inline;margin:0"> -NOTE: **Note:** +#### Emoji and your OS + The emoji example above uses hard-coded images for this documentation. The emoji, when rendered within GitLab, may appear different depending on the OS and browser used. -Most emoji are natively supported on macOS, Windows, iOS, Android, and fall back on image-based emoji where there is no support. +Most emoji are natively supported on macOS, Windows, iOS, Android, and fall back on image-based +emoji where there is no support. -NOTE: **Note:** On Linux, you can download [Noto Color Emoji](https://www.google.com/get/noto/help/emoji/) to get full native emoji support. Ubuntu 18.04 (like many modern Linux distributions) has this font installed by default. @@ -335,7 +334,7 @@ $example = array( ### Inline diff -> If this is not rendered correctly, [view it in GitLab itself](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/user/markdown.md#inline-diff). +If this section is not rendered correctly, [view it in GitLab](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/user/markdown.md#inline-diff). With inline diff tags you can display `{+ additions +}` or `[- deletions -]`. @@ -374,7 +373,7 @@ backslash `\`, otherwise the diff highlight don't render correctly: ### Math -> If this is not rendered correctly, [view it in GitLab itself](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/user/markdown.md#math). +If this section is not rendered correctly, [view it in GitLab](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/user/markdown.md#math). It's possible to have math written with LaTeX syntax rendered using [KaTeX](https://github.com/KaTeX/KaTeX). @@ -402,8 +401,7 @@ a^2+b^2=c^2 _Be advised that KaTeX only supports a [subset](https://katex.org/docs/supported.html) of LaTeX._ -NOTE: **Note:** -This also works for the Asciidoctor `:stem: latexmath`. For details see +This also works for the Asciidoctor `:stem: latexmath`. For details, see the [Asciidoctor user manual](https://asciidoctor.org/docs/user-manual/#activating-stem-support). ### Special GitLab references @@ -446,7 +444,7 @@ recognized and formatted with text `#123`. In addition to this, links to some objects are also recognized and formatted. Some examples of these are: -- Comments on issues: `"https://gitlab.com/gitlab-org/gitlab/-/issues/1234#note_101075757"`, which are rendered as `#1234 (note1)` +- Comments on issues: `"https://gitlab.com/gitlab-org/gitlab/-/issues/1234#note_101075757"`, which are rendered as `#1234 (comment 101075757)` - The issues designs tab: `"https://gitlab.com/gitlab-org/gitlab/-/issues/1234/designs"`, which are rendered as `#1234 (designs)`. - Links to individual designs: `"https://gitlab.com/gitlab-org/gitlab/-/issues/1234/designs/layout.png"`, which are rendered as `#1234[layout.png]`. @@ -607,9 +605,9 @@ Quote break. #### Multiline blockquote -> If this is not rendered correctly, [view it in GitLab itself](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/user/markdown.md#multiline-blockquote). +If this section is not rendered correctly, [view it in GitLab](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/user/markdown.md#multiline-blockquote). -GFM extends the standard Markdown standard by also supporting multi-line blockquotes +GFM extends the standard Markdown by also supporting multi-line blockquotes fenced by `>>>`: ```markdown @@ -688,7 +686,7 @@ Tildes are OK too. #### Colored code and syntax highlighting -> If this is not rendered correctly, [view it in GitLab itself](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/user/markdown.md#colored-code-and-syntax-highlighting). +If this section is not rendered correctly, [view it in GitLab](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/user/markdown.md#colored-code-and-syntax-highlighting). GitLab uses the [Rouge Ruby library](http://rouge.jneen.net/) for more colorful syntax highlighting in code blocks. For a list of supported languages visit the @@ -781,7 +779,7 @@ Strikethrough is not part of the core Markdown standard, but is part of GFM. #### Multiple underscores in words and mid-word emphasis -> If this is not rendered correctly, [view it in GitLab itself](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/user/markdown.md#multiple-underscores-in-words). +If this section is not rendered correctly, [view it in GitLab](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/user/markdown.md#multiple-underscores-in-words). It's not usually useful to italicize just _part_ of a word, especially when you're dealing with code and names that often appear with multiple underscores. As a result, @@ -973,7 +971,7 @@ Do not change to a reference style link. #### Videos -> If this is not rendered correctly, [view it in GitLab itself](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/user/markdown.md#videos). +If this section is not rendered correctly, [view it in GitLab](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/user/markdown.md#videos). Image tags that link to files with a video extension are automatically converted to a video player. The valid video extensions are `.mp4`, `.m4v`, `.mov`, `.webm`, and `.ogv`: @@ -990,7 +988,7 @@ Here's a sample video: #### Audio -> If this is not rendered correctly, [view it in GitLab itself](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/user/markdown.md#audio). +If this section is not rendered correctly, [view it in GitLab](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/user/markdown.md#audio). Similar to videos, link tags for files with an audio extension are automatically converted to an audio player. The valid audio extensions are `.mp3`, `.oga`, `.ogg`, `.spx`, and `.wav`: @@ -1007,7 +1005,7 @@ Here's a sample audio clip: ### Inline HTML -> To see the Markdown rendered within HTML in the second example, [view it in GitLab itself](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/user/markdown.md#inline-html). +To see the Markdown rendered within HTML in the second example, [view it in GitLab](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/user/markdown.md#inline-html). You can also use raw HTML in your Markdown, and it usually works pretty well. @@ -1071,7 +1069,7 @@ Markdown is fine in GitLab. #### Details and summary -> To see the Markdown rendered within HTML in the second example, [view it in GitLab itself](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/user/markdown.md#details-and-summary). +To see the Markdown rendered within HTML in the second example, [view it in GitLab](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/user/markdown.md#details-and-summary). Content can be collapsed using HTML's [`<details>`](https://developer.mozilla.org/en-US/docs/Web/HTML/Element/details) and [`<summary>`](https://developer.mozilla.org/en-US/docs/Web/HTML/Element/summary) @@ -1105,7 +1103,7 @@ These details <em>remain</em> <strong>hidden</strong> until expanded. Markdown inside these tags is supported as well. -NOTE: **Note:** +TIP: **Tip:** If your Markdown isn't rendering correctly, try adding `{::options parse_block_html="true" /}` to the top of the page, and add `markdown="span"` to the opening summary tag like this: `<summary markdown="span">`. @@ -1420,26 +1418,20 @@ Example: ```markdown | header 1 | header 2 | header 3 | -| --- | ------ |---------:| +| --- | ------ |----------| | cell 1 | cell 2 | cell 3 | | cell 4 | cell 5 is longer | cell 6 is much longer than the others, but that's ok. It eventually wraps the text when the cell is too large for the display size. | -| cell 7 | | cell <br> 9 | -| cell 10 | <ul><li> - [ ] Task One </li></ul> | <ul><li> - [ ] Task Two </li><li> - [ ] Task Three </li></ul> | +| cell 7 | | cell 9 | ``` | header 1 | header 2 | header 3 | -| --- | ------ |---------:| +| --- | ------ |----------| | cell 1 | cell 2 | cell 3 | | cell 4 | cell 5 is longer | cell 6 is much longer than the others, but that's ok. It eventually wraps the text when the cell is too large for the display size. | -| cell 7 | | cell <br> 9 | -| cell 10 | <ul><li> - [ ] Task One </li></ul> | <ul><li> - [ ] Task Two </li><li> - [ ] Task Three </li></ul> | +| cell 7 | | cell 9 | Additionally, you can choose the alignment of text within columns by adding colons (`:`) -to the sides of the "dash" lines in the second row. This affects every cell in the column. - -NOTE: **Note:** -[Within GitLab itself](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/user/markdown.md#tables), -the headers are always left-aligned in Chrome and Firefox, and centered in Safari. +to the sides of the "dash" lines in the second row. This affects every cell in the column: ```markdown | Left Aligned | Centered | Right Aligned | Left Aligned | Centered | Right Aligned | @@ -1453,6 +1445,34 @@ the headers are always left-aligned in Chrome and Firefox, and centered in Safar | Cell 1 | Cell 2 | Cell 3 | Cell 4 | Cell 5 | Cell 6 | | Cell 7 | Cell 8 | Cell 9 | Cell 10 | Cell 11 | Cell 12 | +[Within GitLab itself](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/user/markdown.md#tables), +the headers are always left-aligned in Chrome and Firefox, and centered in Safari. + +You can use HTML formatting to adjust the rendering of tables. For example, you can +use `<br>` tags to force a cell to have multiple lines: + +```markdown +| Name | Details | +|------|---------| +| Item1 | This is on one line | +| Item2 | This item has:<br>- Multiple items<br>- That we want listed separately | +``` + +| Name | Details | +|------|---------| +| Item1 | This is on one line | +| Item2 | This item has:<br>- Multiple items<br>- That we want listed separately | + +You can use HTML formatting within GitLab itself to add [task lists](#task-lists) with checkboxes, +but they do not render properly on `docs.gitlab.com`: + +```markdown +| header 1 | header 2 | +|----------|----------| +| cell 1 | cell 2 | +| cell 3 | <ul><li> - [ ] Task one </li><li> - [ ] Task two </li></ul> | +``` + #### Copy from spreadsheet and paste in Markdown [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/27205) in GitLab 12.7. @@ -1474,5 +1494,5 @@ entry and paste the spreadsheet: - This document leveraged heavily from the [Markdown-Cheatsheet](https://github.com/adam-p/markdown-here/wiki/Markdown-Cheatsheet). - The original [Markdown Syntax Guide](https://daringfireball.net/projects/markdown/syntax) at Daring Fireball is an excellent resource for a detailed explanation of standard Markdown. -- The detailed specification for CommonMark can be found in the [CommonMark Spec](https://spec.commonmark.org/current/) -- The [CommonMark Dingus](http://try.commonmark.org) is a handy tool for testing CommonMark syntax. +- You can find the detailed specification for CommonMark in the [CommonMark Spec](https://spec.commonmark.org/current/). +- The [CommonMark Dingus](https://spec.commonmark.org/dingus/) is a handy tool for testing CommonMark syntax. diff --git a/doc/user/packages/composer_repository/img/project_id_v13_5.png b/doc/user/packages/composer_repository/img/project_id_v13_5.png Binary files differnew file mode 100644 index 00000000000..45861b6ff1b --- /dev/null +++ b/doc/user/packages/composer_repository/img/project_id_v13_5.png diff --git a/doc/user/packages/composer_repository/index.md b/doc/user/packages/composer_repository/index.md index 89e02b4847c..d4fb662c3a6 100644 --- a/doc/user/packages/composer_repository/index.md +++ b/doc/user/packages/composer_repository/index.md @@ -4,172 +4,259 @@ group: Package info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# GitLab Composer Repository +# Composer packages in the Package Registry > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/15886) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.2. > - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/221259) to GitLab Core in 13.3. -With the GitLab Composer Repository, every project can have its own space to store [Composer](https://getcomposer.org/) packages. +Publish [Composer](https://getcomposer.org/) packages in your project's Package Registry. +Then, install the packages whenever you need to use them as a dependency. -## Enabling the Composer Repository +Only Composer 1.x is supported. Consider contributing or even adding support for +[Composer 2.0 in the Package Registry](https://gitlab.com/gitlab-org/gitlab/-/issues/259840). -NOTE: **Note:** -This option is available only if your GitLab administrator has -[enabled support for the Package Registry](../../../administration/packages/index.md). +## Create a Composer package -When the Composer Repository is enabled, it is available for all new projects -by default. To enable it for existing projects, or if you want to disable it: +If you do not have a Composer package, create one and check it in to +a repository. This example shows a GitLab repository, but the repository +can be any public or private repository. -1. Navigate to your project's **Settings > General > Visibility, project features, permissions**. -1. Find the Packages feature and enable or disable it. -1. Click on **Save changes** for the changes to take effect. +1. Create a directory called `my-composer-package` and change to that directory: -You should then be able to see the **Packages & Registries** section on the left sidebar. + ```shell + mkdir my-composer-package && cd my-composer-package + ``` -## Getting started +1. Run [`composer init`](https://getcomposer.org/doc/03-cli.md#init) and answer the prompts. -This section covers creating a new example Composer package to publish. This is a -quickstart to test out the **GitLab Composer Registry**. + For namespace, enter your unique [namespace](../../../user/group/index.md#namespaces), like your GitLab username or group name. -To complete this section, you need a recent version of [Composer](https://getcomposer.org/). + A file called `composer.json` is created: -### Creating a package project + ```json + { + "name": "<namespace>/composer-test", + "description": "Library XY", + "type": "library", + "license": "GPL-3.0-only", + "authors": [ + { + "name": "John Doe", + "email": "john@example.com" + } + ], + "require": {} + } + ``` -Understanding how to create a full Composer project is outside the scope of this -guide, but you can create a small package to test out the registry. Start by -creating a new directory called `my-composer-package`: +1. Run Git commands to tag the changes and push them to your repository: -```shell -mkdir my-composer-package && cd my-composer-package -``` + ```shell + git init + git add composer.json + git commit -m 'Composer package test' + git tag v1.0.0 + git remote add origin git@gitlab.example.com:<namespace>/<project-name>.git + git push --set-upstream origin master + git push origin v1.0.0 + ``` -Create a new `composer.json` file inside this directory to set up the basic project: +The package is now in your GitLab Package Registry. -```shell -touch composer.json -``` +## Publish a Composer package by using the API -Inside `composer.json`, add the following code: +Publish a Composer package to the Package Registry, +so that anyone who can access the project can use the package as a dependency. -```json -{ - "name": "<namespace>/composer-test", - "type": "library", - "license": "GPL-3.0-only", - "version": "1.0.0" -} -``` +Prerequisites: -Replace `<namespace>` with a unique namespace like your GitLab username or group name. +- A package in a GitLab repository. +- A valid `composer.json` file. +- The Packages feature is enabled in a GitLab repository. +- The project ID, which is on the project's home page. +- A [personal access token](../../../user/profile/personal_access_tokens.md) with the scope set to `api`. -After this basic package structure is created, we need to tag it in Git and push it to the repository. + NOTE: **Note:** + [Deploy tokens](./../../project/deploy_tokens/index.md) are + [not yet supported](https://gitlab.com/gitlab-org/gitlab/-/issues/240897) for use with Composer. -```shell -git init -git add composer.json -git commit -m 'Composer package test' -git tag v1.0.0 -git remote add origin git@gitlab.com:<namespace>/<project-name>.git -git push --set-upstream origin master -git push origin v1.0.0 -``` +To publish the package: -### Publishing the package +- Send a `POST` request to the [Packages API](../../../api/packages.md). -Now that the basics of our project is completed, we can publish the package. -To publish the package, you need: + For example, you can use `curl`: -- A personal access token or `CI_JOB_TOKEN`. + ```shell + curl --data tag=<tag> "https://__token__:<personal-access-token>@gitlab.example.com/api/v4/projects/<project_id>/packages/composer" + ``` - ([Deploy tokens](./../../project/deploy_tokens/index.md) are not yet supported for use with Composer.) + - `<personal-access-token>` is your personal access token. + - `<project_id>` is your project ID. + - `<tag>` is the Git tag name of the version you want to publish. + To publish a branch, use `branch=<branch>` instead of `tag=<tag>`. -- Your project ID which can be found on the home page of your project. +You can view the published package by going to **Packages & Registries > Package Registry** and +selecting the **Composer** tab. -To publish the package hosted on GitLab, make a `POST` request to the GitLab package API. -A tool like `curl` can be used to make this request: +## Publish a Composer package by using CI/CD -You can generate a [personal access token](../../../user/profile/personal_access_tokens.md) with the scope set to `api` for repository authentication. For example: +You can publish a Composer package to the Package Registry as part of your CI/CD process. -```shell -curl --data tag=<tag> 'https://__token__:<personal-access-token>@gitlab.com/api/v4/projects/<project_id>/packages/composer' -``` +1. Specify a `CI_JOB_TOKEN` in your `.gitlab-ci.yml` file: -Where: + ```yaml + stages: + - deploy -- `<personal-access-token>` is your personal access token. -- `<project_id>` is your project ID. -- `<tag>` is the Git tag name of the version you want to publish. In this example it should be `v1.0.0`. Notice that instead of `tag=<tag>` you can also use `branch=<branch>` to publish branches. + deploy: + stage: deploy + script: + - 'curl --header "Job-Token: $CI_JOB_TOKEN" --data tag=<tag> "https://gitlab.example.com/api/v4/projects/$CI_PROJECT_ID/packages/composer"' + ``` -If the above command succeeds, you now should be able to see the package under the **Packages & Registries** section of your project page. +1. Run the pipeline. -### Publishing the package with CI/CD +You can view the published package by going to **Packages & Registries > Package Registry** and selecting the **Composer** tab. -To work with Composer commands within [GitLab CI/CD](./../../../ci/README.md), you can -publish Composer packages by using `CI_JOB_TOKEN` in your `.gitlab-ci.yml` file: +### Use a CI/CD template -```yaml -stages: - - deploy +A more detailed Composer CI/CD file is also available as a `.gitlab-ci.yml` template: -deploy: - stage: deploy - script: - - 'curl --header "Job-Token: $CI_JOB_TOKEN" --data tag=<tag> "https://gitlab.example.com/api/v4/projects/$CI_PROJECT_ID/packages/composer"' -``` +1. On the left sidebar, click **Project overview**. +1. Above the file list, click **Set up CI/CD**. If this button is not available, select **CI/CD Configuration** and then **Edit**. +1. From the **Apply a template** list, select **Composer**. -### Installing a package +CAUTION: **Warning:** +Do not save unless you want to overwrite the existing CI/CD file. -To install your package, you need: +## Install a Composer package -- A personal access token. You can generate a [personal access token](../../../user/profile/personal_access_tokens.md) with the scope set to `api` for repository authentication. -- Your group ID which can be found on the home page of your project's group. +Install a package from the Package Registry so you can use it as a dependency. -Add the GitLab Composer package repository to your existing project's `composer.json` file, along with the package name and version you want to install like so: +Prerequisites: -```json -{ - ... - "repositories": [ - { "type": "composer", "url": "https://gitlab.com/api/v4/group/<group_id>/-/packages/composer/packages.json" } - ], - "require": { - ... - "<package_name>": "<version>" - }, - ... -} -``` +- A package in the Package Registry. +- The group ID, which is on the group's home page. +- A [personal access token](../../../user/profile/personal_access_tokens.md) with the scope set to, at minimum, `read_api`. -Where: + NOTE: **Note:** + [Deploy tokens](./../../project/deploy_tokens/index.md) are + [not yet supported](https://gitlab.com/gitlab-org/gitlab/-/issues/240897) for use with Composer. -- `<group_id>` is the group ID found under your project's group page. -- `<package_name>` is your package name as defined in your package's `composer.json` file. -- `<version>` is your package version (`1.0.0` in this example). +To install a package: -You also need to create a `auth.json` file with your GitLab credentials: +1. Add the Package Registry URL to your project's `composer.json` file, along with the package name and version you want to install: -```json -{ - "gitlab-token": { - "gitlab.com": "<personal_access_token>" - } -} -``` + - Connect to the Package Registry for your group: -Where: + ```shell + composer config repositories.<group_id> composer https://gitlab.example.com/api/v4/group/<group_id>/-/packages/composer/ + ``` -- `<personal_access_token>` is your personal access token. + - Set the required package version: -With the `composer.json` and `auth.json` files configured, you can install the package by running `composer`: + ```shell + composer require <package_name>:<version> + ``` -```shell -composer update -``` + Result in the `composer.json` file: -If successful, you should be able to see the output indicating that the package has been successfully installed. + ```json + { + ... + "repositories": { + "<group_id>": { + "type": "composer", + "url": "https://gitlab.example.com/api/v4/group/<group_id>/-/packages/composer/" + }, + ... + }, + "require": { + ... + "<package_name>": "<version>" + }, + ... + } + ``` + + You can unset this with the command: + + ```shell + composer config --unset repositories.<group_id> + ``` + + - `<group_id>` is the group ID. + - `<package_name>` is the package name defined in your package's `composer.json` file. + - `<version>` is the package version. + +1. Create an `auth.json` file with your GitLab credentials: + + ```shell + composer config gitlab-token.<DOMAIN-NAME> <personal_access_token> + ``` + + Result in the `auth.json` file: + + ```json + { + ... + "gitlab-token": { + "<DOMAIN-NAME>": "<personal_access_token>", + ... + } + } + ``` + + You can unset this with the command: + + ```shell + composer config --unset --auth gitlab-token.<DOMAIN-NAME> + ``` + + - `<DOMAIN-NAME>` is the GitLab instance URL `gitlab.com` or `gitlab.example.com`. + - `<personal_access_token>` with the scope set to `read_api`. + +1. If you are on a GitLab self-managed instance, add `gitlab-domains` to `composer.json`. + + ```shell + composer config gitlab-domains gitlab01.example.com gitlab02.example.com + ``` + + Result in the `composer.json` file: + + ```json + { + ... + "repositories": [ + { "type": "composer", "url": "https://gitlab.example.com/api/v4/group/<group_id>/-/packages/composer/" } + ], + "config": { + ... + "gitlab-domains": ["gitlab01.example.com", "gitlab02.example.com"] + }, + "require": { + ... + "<package_name>": "<version>" + }, + ... + } + ``` + + You can unset this with the command: + + ```shell + composer config --unset gitlab-domains + ``` + + NOTE: **Note:** + On GitLab.com, Composer uses the GitLab token from `auth.json` as a private token by default. + Without the `gitlab-domains` definition in `composer.json`, Composer uses the GitLab token + as basic-auth, with the token as a username and a blank password. This results in a 401 error. + +Output indicates that the package has been successfully installed. CAUTION: **Important:** -Make sure to never commit the `auth.json` file to your repository. To install packages from a CI job, +Never commit the `auth.json` file to your repository. To install packages from a CI/CD job, consider using the [`composer config`](https://getcomposer.org/doc/articles/handling-private-packages-with-satis.md#authentication) tool with your personal access token stored in a [GitLab CI/CD environment variable](../../../ci/variables/README.md) or in -[Hashicorp Vault](../../../ci/secrets/index.md). +[HashiCorp Vault](../../../ci/secrets/index.md). diff --git a/doc/user/packages/conan_repository/img/conan_package_view.png b/doc/user/packages/conan_repository/img/conan_package_view.png Binary files differdeleted file mode 100644 index 742fb4195da..00000000000 --- a/doc/user/packages/conan_repository/img/conan_package_view.png +++ /dev/null diff --git a/doc/user/packages/conan_repository/index.md b/doc/user/packages/conan_repository/index.md index 7c3082e0f83..10c820a86be 100644 --- a/doc/user/packages/conan_repository/index.md +++ b/doc/user/packages/conan_repository/index.md @@ -4,153 +4,150 @@ group: Package info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# GitLab Conan Repository +# Conan packages in the Package Registry > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/8248) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.6. > - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/221259) to GitLab Core in 13.3. -With the GitLab Conan Repository, every -project can have its own space to store Conan packages. +Publish Conan packages in your project’s Package Registry. Then install the +packages whenever you need to use them as a dependency. - +To publish Conan packages to the Package Registry, add the +Package Registry as a remote and authenticate with it. -## Enabling the Conan Repository +Then you can run `conan` commands and publish your package to the Package Registry. -NOTE: **Note:** -This option is available only if your GitLab administrator has -[enabled support for the Conan Repository](../../../administration/packages/index.md). - -After the Conan Repository is enabled, it's available for all new projects -by default. To enable it for existing projects, or if you want to disable it: - -1. Navigate to your project's **Settings > General > Visibility, project features, permissions**. -1. Find the Packages feature and enable or disable it. -1. Click on **Save changes** for the changes to take effect. - -You should then be able to see the **Packages & Registries** section on the left sidebar. +## Build a Conan package -## Getting started +This section explains how to install Conan and build a package for your C/C++ project. -This section covers installing Conan and building a package for your C/C++ project. This is a quickstart if you're new -to Conan. If you already are using Conan and understand how to build your own packages, move on to the [next section](#adding-the-gitlab-package-registry-as-a-conan-remote). +If you already use Conan and know how to build your own packages, go to the [next section](#add-the-package-registry-as-a-conan-remote). -### Installing Conan +### Install Conan -Follow the instructions at [conan.io](https://conan.io/downloads.html) to download the Conan package manager to your local development environment. +Download the Conan package manager to your local development environment by following +the instructions at [conan.io](https://conan.io/downloads.html). -Once installation is complete, verify you can use Conan in your terminal by running +When installation is complete, verify you can use Conan in your terminal by running: ```shell conan --version ``` -You should see the Conan version printed in the output: +The Conan version is printed in the output: ```plaintext Conan version 1.20.5 ``` -### Installing CMake +### Install CMake + +When you develop with C++ and Conan, you have a range of compilers to choose from. +This example uses the CMake compiler. + +To install CMake: + +- For Mac, use [homebrew](https://brew.sh/) and run `brew install cmake`. +- For other operating systems, follow the instructions at [cmake.org](https://cmake.org/install/). -When developing with C++ and Conan, you have a wide range of options for compilers. This tutorial walks through using the cmake -compiler. In your terminal, run the command +When installation is complete, verify you can use CMake in your terminal by running: ```shell cmake --version ``` -You should see the cmake version printed in the output. If you see something else, you may have to install cmake. +The CMake version is printed in the output. -On a Mac, you can use [homebrew](https://brew.sh/) to install cmake by running `brew install cmake`. Otherwise, follow -instructions at [cmake.org](https://cmake.org/install/) for your operating system. +### Create a project -### Creating a project +To test the Package Registry, you need a C++ project. If you don't already have one, you can clone the +Conan [hello world starter project](https://github.com/conan-io/hello). -Understanding what is needed to create a valid and compilable C++ project is out of the scope of this guide, but if you're new to C++ and want to try out the GitLab -package registry, Conan.io has a great [hello world starter project](https://github.com/conan-io/hello) that you can clone to get started. +### Build a package -Clone the repository and it can be used for the rest of the tutorial if you don't have your own C++ project. +To build a package: -### Building a package +1. Open a terminal and navigate to your project's root folder. +1. Generate a new recipe by running `conan new` with a package name and version: -In your terminal, navigate to the root folder of your project. Generate a new recipe by running `conan new` and providing it with a -package name and version: + ```shell + conan new Hello/0.1 -t + ``` -```shell -conan new Hello/0.1 -t -``` - -Next, create a package for that recipe by running `conan create` providing the Conan user and channel: +1. Create a package for the recipe by running `conan create` with the Conan user and channel: -```shell -conan create . mycompany/beta -``` + ```shell + conan create . mycompany/beta + ``` -NOTE: **Note:** -If you are using the [instance level remote](#instance-level-remote), a specific [naming convention](#package-recipe-naming-convention-for-instance-level-remote) must be followed. + NOTE: **Note:** + If you use an [instance remote](#add-a-remote-for-your-instance), you must follow a specific [naming convention](#package-recipe-naming-convention-for-instance-remotes). -These two example commands generate a final package with the recipe `Hello/0.1@mycompany/beta`. +A package with the recipe `Hello/0.1@mycompany/beta` is created. -For more advanced details on creating and managing your packages, refer to the [Conan docs](https://docs.conan.io/en/latest/creating_packages.html). +For more details on creating and managing Conan packages, see the [Conan docs](https://docs.conan.io/en/latest/creating_packages.html). -You are now ready to upload your package to the GitLab registry. To get started, first you need to set GitLab as a remote. Then you need to add a Conan user for that remote to authenticate your requests. +## Add the Package Registry as a Conan remote -## Adding the GitLab Package Registry as a Conan remote +To run `conan` commands, you must add the Package Registry as a Conan remote for your project or instance. -You can add the GitLab Package Registry as a Conan remote at the project or instance level. - -### Project level remote +### Add a remote for your project > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/11679) in GitLab 13.4. -The project level remote allows you to work with packages within a given project. -The advantage of using the project level remote is there are no restrictions to your -package name, however all GitLab Conan packages require a full recipe -with the user and channel (`package_name/version@user/channel`). +Set a remote so you can work with packages in a project without +having to specify the remote name in every command. + +When you set a remote for a project, there are no restrictions to your package names. +However, your commands must include the full recipe, including the user and channel, +for example, `package_name/version@user/channel`. To add the remote: -```shell -conan remote add gitlab https://gitlab.example.com/api/v4/projects/<project_id>/packages/conan -``` +1. In your terminal, run this command: -Once the remote is set, you can use the remote when running Conan commands by adding `--remote=gitlab` to the end of your commands. + ```shell + conan remote add gitlab https://gitlab.example.com/api/v4/projects/<project_id>/packages/conan + ``` -For example: +1. Use the remote by adding `--remote=gitlab` to the end of your Conan command. -```shell -conan search Hello* --all --remote=gitlab -``` + For example: -### Instance level remote + ```shell + conan search Hello* --all --remote=gitlab + ``` -The instance level remote allows you to use a single remote to access packages accross your entire -GitLab instance. However, when using this remote, there are certain -[package naming restrictions](#package-recipe-naming-convention-for-instance-level-remote) -that must be followed. +### Add a remote for your instance -Add a new remote to your Conan configuration: +Use a single remote to access packages across your entire GitLab instance. -```shell -conan remote add gitlab https://gitlab.example.com/api/v4/packages/conan -``` +However, when using this remote, you must follow these +[package naming restrictions](#package-recipe-naming-convention-for-instance-remotes). + +To add the remote: -Once the remote is set, you can use the remote when running Conan commands by adding `--remote=gitlab` to the end of your commands. +1. In your terminal, run this command: -For example: + ```shell + conan remote add gitlab https://gitlab.example.com/api/v4/packages/conan + ``` -```shell -conan search 'Hello*' --remote=gitlab -``` +1. Use the remote by adding `--remote=gitlab` to the end of your Conan command. -#### Package recipe naming convention for instance level remote + For example: -The standard Conan recipe convention looks like `package_name/version@user/channel`, -but if you're using the [instance level remote](#instance-level-remote), the recipe + ```shell + conan search 'Hello*' --remote=gitlab + ``` + +#### Package recipe naming convention for instance remotes + +The standard Conan recipe convention is `package_name/version@user/channel`, +but if you're using an [instance remote](#add-a-remote-for-your-instance), the recipe `user` must be the plus sign (`+`) separated project path. -The following table shows some example recipes you can give your package based on -the project name and path. +Example recipe names: | Project | Package | Supported | | ---------------------------------- | ----------------------------------------------- | --------- | @@ -159,179 +156,202 @@ the project name and path. | `gitlab-org/gitlab-ce` | `my-package/1.0.0@gitlab-org+gitlab-ce/stable` | Yes | | `gitlab-org/gitlab-ce` | `my-package/1.0.0@foo/stable` | No | -NOTE: **Note:** -[Project level remotes](#project-level-remote) allow for more flexible package names. +[Project remotes](#add-a-remote-for-your-project) have a more flexible naming convention. -## Authenticating to the GitLab Conan Repository +## Authenticate to the Package Registry -You need a personal access token or deploy token. +To authenticate to the Package Registry, you need either a personal access token or deploy token. -For repository authentication: +- If you use a [personal access token](../../../user/profile/personal_access_tokens.md), set the scope to `api`. +- If you use a [deploy token](./../../project/deploy_tokens/index.md), set the scope to `read_package_registry`, `write_package_registry`, or both. -- You can generate a [personal access token](../../../user/profile/personal_access_tokens.md) with the scope set to `api`. -- You can generate a [deploy token](./../../project/deploy_tokens/index.md) with the scope set to `read_package_registry`, `write_package_registry`, or both. +### Add your credentials to the GitLab remote -### Adding a Conan user to the GitLab remote +Associate your token with the GitLab remote, so that you don't have to explicitly +add a token to every Conan command. -Once you have a personal access token and have [set your Conan remote](#adding-the-gitlab-package-registry-as-a-conan-remote), you can associate the token with the remote so you don't have to explicitly add them to each Conan command you run: +Prerequisites: + +- You must have an authentication token. +- The Conan remote [must be set](#add-the-package-registry-as-a-conan-remote). + +In a terminal, run this command. In this example, the remote name is `gitlab`. Use the name of your remote. ```shell conan user <gitlab_username or deploy_token_username> -r gitlab -p <personal_access_token or deploy_token> ``` -NOTE: **Note:** -If you named your remote something other than `gitlab`, your remote name should be used in this command instead of `gitlab`. - -From now on, when you run commands using `--remote=gitlab`, your username and password is automatically included in the requests. - -NOTE: **Note:** -The personal access token is not stored locally at any moment. Conan uses JSON Web Tokens (JWT), so when you run this command, Conan requests an expirable token from GitLab using your token. The JWT does expire on a regular basis, so you need to re-enter your personal access token when that happens. +Now when you run commands with `--remote=gitlab`, your username and password are automatically included in the requests. -Alternatively, you could explicitly include your credentials in any given command. -For example: +Alternately, you can explicitly include your credentials in any given command. For example: ```shell CONAN_LOGIN_USERNAME=<gitlab_username or deploy_token_username> CONAN_PASSWORD=<personal_access_token or deploy_token> conan upload Hello/0.1@mycompany/beta --all --remote=gitlab ``` -### Setting a default remote to your project (optional) +NOTE: **Note:** +Your authentication with GitLab expires on a regular basis, +so occasionally you may need to re-enter your personal access token. + +### Set a default remote for your project (optional) + +If you want to interact with the GitLab Package Registry without having to specify a remote, +you can tell Conan to always use the Package Registry for your packages. -If you'd like Conan to always use GitLab as the registry for your package, you can tell Conan to always reference the GitLab remote for a given package recipe: +In a terminal, run this command. ```shell conan remote add_ref Hello/0.1@mycompany/beta gitlab ``` NOTE: **Note:** -The package recipe does include the version, so setting the default remote for `Hello/0.1@user/channel` will not work for `Hello/0.2@user/channel`. -This functionality is best suited for when you want to consume or install packages from the GitLab registry without having to specify a remote. +The package recipe includes the version, so the default remote for `Hello/0.1@user/channel` does not work for `Hello/0.2@user/channel`. -The rest of the example commands in this documentation assume that you've added a Conan user with your credentials to the `gitlab` remote and will not include the explicit credentials or remote option. With that said, be aware that any of the commands could be run without having added a user or default remote: +If you do not set a default user or remote, you can still include the user and remote in your commands: ```shell `CONAN_LOGIN_USERNAME=<gitlab_username or deploy_token_username> CONAN_PASSWORD=<personal_access_token or deploy_token> <conan command> --remote=gitlab ``` -## Uploading a package +## Publish a Conan package -First you need to [create your Conan package locally](https://docs.conan.io/en/latest/creating_packages/getting_started.html). +Publish a Conan package to the Package Registry, so that anyone who can access the project can use the package as a dependency. -NOTE: **Note:** -If you are using the [instance level remote](#instance-level-remote), a specific [naming convention](#package-recipe-naming-convention-for-instance-level-remote) must be followed. +Prerequisites: + +To publish a Conan package, you need: -Ensure you have a project created on GitLab and that the personal access token you're using has the correct permissions for write access to the container registry by selecting the `api` [scope](../../../user/profile/personal_access_tokens.md#limiting-scopes-of-a-personal-access-token). +- The Package Registry [set as a remote](#add-the-package-registry-as-a-conan-remote). +- [Authentication](#authenticate-to-the-package-registry) set up with the Package Registry. +- A local [Conan package](https://docs.conan.io/en/latest/creating_packages/getting_started.html). + - For an instance remote, the package must meet the [naming convention](#package-recipe-naming-convention-for-instance-remotes). +- A project ID, which is on the project's homepage. -You can upload your package to the GitLab Package Registry using the `conan upload` command: +To publish the package, use the `conan upload` command: ```shell conan upload Hello/0.1@mycompany/beta --all ``` -## Installing a package +## Publish a Conan package by using CI/CD -Conan packages are commonly installed as dependencies using the `conanfile.txt` file. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/11678) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.7. -In your project where you would like to install the Conan package as a dependency, open `conanfile.txt` or create -an empty file named `conanfile.txt` in the root of your project. +To work with Conan commands in [GitLab CI/CD](./../../../ci/README.md), you can use +`CI_JOB_TOKEN` in place of the personal access token in your commands. -Add the Conan recipe to the `[requires]` section of the file: +You can provide the `CONAN_LOGIN_USERNAME` and `CONAN_PASSWORD` with each +Conan command in your `.gitlab-ci.yml` file. For example: -```ini - [requires] - Hello/0.1@mycompany/beta +```yaml +image: conanio/gcc7 - [generators] - cmake +create_package: + stage: deploy + script: + - conan remote add gitlab https://gitlab.example.com/api/v4/packages/conan + - conan new <package-name>/0.1 -t + - conan create . <group-name>+<project-name>/stable + - CONAN_LOGIN_USERNAME=ci_user CONAN_PASSWORD=${CI_JOB_TOKEN} conan upload <package-name>/0.1@<group-name>+<project-name>/stable --all --remote=gitlab ``` -Next, create a build directory from the root of your project and navigate to it: +Additional Conan images to use as the basis of your CI file are available +in the [Conan docs](https://docs.conan.io/en/latest/howtos/run_conan_in_docker.html#available-docker-images). -```shell -mkdir build && cd build -``` +## Install a Conan package -Now you can install the dependencies listed in `conanfile.txt`: +Install a Conan package from the Package Registry so you can use it as a dependency. -```shell -conan install .. -``` +Conan packages are often installed as dependencies by using the `conanfile.txt` file. + +Prerequisites: + +To install a Conan package, you need: + +- The Package Registry [set as a remote](#add-the-package-registry-as-a-conan-remote). +- [Authentication](#authenticate-to-the-package-registry) set up with the Package Registry. + +1. In the project where you want to install the package as a dependency, open `conanfile.txt`. + Or, in the root of your project, create a file called `conanfile.txt`. + +1. Add the Conan recipe to the `[requires]` section of the file: + + ```plaintext + [requires] + Hello/0.1@mycompany/beta + + [generators] + cmake + ``` + +1. At the root of your project, create a `build` directory and change to that directory: + + ```shell + mkdir build && cd build + ``` + +1. Install the dependencies listed in `conanfile.txt`: + + ```shell + conan install <options> + ``` NOTE: **Note:** -If you're trying to install the package you just created in this tutorial, not much will happen since that package -already exists on your local machine. +If you try to install the package you just created in this tutorial, the package +already exists on your local machine, so this command has no effect. -## Removing a package +## Remove a Conan package There are two ways to remove a Conan package from the GitLab Package Registry. -- **Using the Conan client in the command line:** +- From the command line, using the Conan client: ```shell conan remove Hello/0.2@user/channel --remote=gitlab ``` - You need to explicitly include the remote in this command, otherwise the package is only removed from your + You must explicitly include the remote in this command, otherwise the package is only removed from your local system cache. NOTE: **Note:** This command removes all recipe and binary package files from the Package Registry. -- **GitLab project interface**: in the packages view of your project page, you can delete packages by clicking the red trash icons. +- From the GitLab user interface: -## Searching the GitLab Package Registry for Conan packages + Go to your project's **Packages & Registries > Package Registry**. Remove the package by clicking the red trash icon. -The `conan search` command can be run searching by full or partial package name, or by exact recipe. +## Search for Conan packages in the Package Registry -To search using a partial name, use the wildcard symbol `*`, which should be placed at the end of your search (for example, `my-packa*`): +To search by full or partial package name, or by exact recipe, run the `conan search` command. -```shell -conan search Hello --all --remote=gitlab -conan search He* --all --remote=gitlab -conan search Hello/0.1@mycompany/beta --all --remote=gitlab -``` +- To search for all packages with a specific package name: + + ```shell + conan search Hello --remote=gitlab + ``` -The scope of your search includes all projects you have permission to access, this includes your private projects as well as all public projects. +- To search for a partial name, like all packages starting with `He`: -## Fetching Conan package information from the GitLab Package Registry + ```shell + conan search He* --remote=gitlab + ``` -The `conan info` command returns information about a given package: +The scope of your search includes all projects you have permission to access. This includes your private projects as well as all public projects. + +## Fetch Conan package information from the Package Registry + +The `conan info` command returns information about a package: ```shell conan info Hello/0.1@mycompany/beta ``` -## List of supported CLI commands +## Supported CLI commands The GitLab Conan repository supports the following Conan CLI commands: -- `conan upload`: Upload your recipe and package files to the GitLab Package Registry. -- `conan install`: Install a conan package from the GitLab Package Registry, this includes using the `conanfile.txt` file. -- `conan search`: Search the GitLab Package Registry for public packages, and private packages you have permission to view. -- `conan info`: View the information on a given package from the GitLab Package Registry. -- `conan remove`: Delete the package from the GitLab Package Registry. - -## Using GitLab CI with Conan packages - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/11678) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.7. - -To work with Conan commands within [GitLab CI/CD](./../../../ci/README.md), you can use -`CI_JOB_TOKEN` in place of the personal access token in your commands. - -It's easiest to provide the `CONAN_LOGIN_USERNAME` and `CONAN_PASSWORD` with each -Conan command in your `.gitlab-ci.yml` file. For example: - -```yaml -image: conanio/gcc7 - -create_package: - stage: deploy - script: - - conan remote add gitlab https://gitlab.example.com/api/v4/packages/conan - - conan new <package-name>/0.1 -t - - conan create . <group-name>+<project-name>/stable - - CONAN_LOGIN_USERNAME=ci_user CONAN_PASSWORD=${CI_JOB_TOKEN} conan upload <package-name>/0.1@<group-name>+<project-name>/stable --all --remote=gitlab - -``` - -You can find additional Conan images to use as the base of your CI file -in the [Conan docs](https://docs.conan.io/en/latest/howtos/run_conan_in_docker.html#available-docker-images). +- `conan upload`: Upload your recipe and package files to the Package Registry. +- `conan install`: Install a Conan package from the Package Registry, this includes using the `conanfile.txt` file. +- `conan search`: Search the Package Registry for public packages, and private packages you have permission to view. +- `conan info`: View the information on a given package from the Package Registry. +- `conan remove`: Delete the package from the Package Registry. diff --git a/doc/user/packages/container_registry/img/container_registry_group_repositories_v13_1.png b/doc/user/packages/container_registry/img/container_registry_group_repositories_v13_1.png Binary files differdeleted file mode 100644 index bbbba44eb9b..00000000000 --- a/doc/user/packages/container_registry/img/container_registry_group_repositories_v13_1.png +++ /dev/null diff --git a/doc/user/packages/container_registry/img/container_registry_hover_path_13_4.png b/doc/user/packages/container_registry/img/container_registry_hover_path_13_4.png Binary files differnew file mode 100644 index 00000000000..2d16c11fe61 --- /dev/null +++ b/doc/user/packages/container_registry/img/container_registry_hover_path_13_4.png diff --git a/doc/user/packages/container_registry/img/container_registry_repositories_v13_1.png b/doc/user/packages/container_registry/img/container_registry_repositories_v13_1.png Binary files differdeleted file mode 100644 index 13a6d1a4470..00000000000 --- a/doc/user/packages/container_registry/img/container_registry_repositories_v13_1.png +++ /dev/null diff --git a/doc/user/packages/container_registry/img/container_registry_repositories_with_quickstart_v13_1.png b/doc/user/packages/container_registry/img/container_registry_repositories_with_quickstart_v13_1.png Binary files differdeleted file mode 100644 index 35a02182a77..00000000000 --- a/doc/user/packages/container_registry/img/container_registry_repositories_with_quickstart_v13_1.png +++ /dev/null diff --git a/doc/user/packages/container_registry/img/container_registry_repository_details_v13.0.png b/doc/user/packages/container_registry/img/container_registry_repository_details_v13.0.png Binary files differdeleted file mode 100644 index 088e80221de..00000000000 --- a/doc/user/packages/container_registry/img/container_registry_repository_details_v13.0.png +++ /dev/null diff --git a/doc/user/packages/container_registry/index.md b/doc/user/packages/container_registry/index.md index 077666bc036..baadd3c91a7 100644 --- a/doc/user/packages/container_registry/index.md +++ b/doc/user/packages/container_registry/index.md @@ -9,235 +9,186 @@ info: To determine the technical writer assigned to the Stage/Group associated w > - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/4040) in GitLab 8.8. > - Docker Registry manifest `v1` support was added in GitLab 8.9 to support Docker > versions earlier than 1.10. -> - Starting from GitLab 8.12, if you have 2FA enabled in your account, you need -> to pass a [personal access token](../../profile/personal_access_tokens.md) instead of your password in order to -> login to GitLab's Container Registry. -> - Multiple level image names support was added in GitLab 9.1. -> - The group level Container Registry was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/23315) in GitLab 12.10. +> - Starting in GitLab 8.12, if you have [two-factor authentication](../../profile/account/two_factor_authentication.md) enabled in your account, you need +> to pass a [personal access token](../../profile/personal_access_tokens.md) instead of your password to +> sign in to the Container Registry. +> - Support for multiple level image names was added in GitLab 9.1. +> - The group-level Container Registry was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/23315) in GitLab 12.10. > - Searching by image repository name was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/31322) in GitLab 13.0. -NOTE: **Note:** -This document is the user guide. To learn how to enable GitLab Container -Registry across your GitLab instance, visit the -[administrator documentation](../../../administration/packages/container_registry.md). - -With the Docker Container Registry integrated into GitLab, every project can +With the Docker Container Registry integrated into GitLab, every GitLab project can have its own space to store its Docker images. You can read more about Docker Registry at <https://docs.docker.com/registry/introduction/>. - - -## Enable the Container Registry for your project - -CAUTION: **Warning:** -The Container Registry follows the visibility settings of the project. If the project is public, so is the Container Registry. - -If you cannot find the **Packages & Registries > Container Registry** entry under your -project's sidebar, it is not enabled in your GitLab instance. Ask your -administrator to enable GitLab Container Registry following the -[administration documentation](../../../administration/packages/container_registry.md). +This document is the user guide. To learn how to enable the Container +Registry for your GitLab instance, visit the +[administrator documentation](../../../administration/packages/container_registry.md). -If you are using GitLab.com, this is enabled by default so you can start using -the Registry immediately. Currently there is a soft (10GB) size restriction for -Registry on GitLab.com, as part of the [repository size limit](../../project/repository/index.md). +## View the Container Registry -Once enabled for your GitLab instance, to enable Container Registry for your -project: +You can view the Container Registry for a project or group. -1. Go to your project's **Settings > General** page. -1. Expand the **Visibility, project features, permissions** section - and enable the **Container Registry** feature on your project. For new - projects this might be enabled by default. For existing projects - (prior GitLab 8.8), enable it explicitly. -1. Press **Save changes** for the changes to take effect. You should now be able - to see the **Packages & Registries > Container Registry** link in the sidebar. +1. Go to your project or group. +1. Go to **Packages & Registries > Container Registry**. -## Control Container Registry from within GitLab +You can search, sort, filter, and [delete](#delete-images-from-within-gitlab) containers on this page. -GitLab offers a simple Container Registry management panel. This management panel is available -for both projects and groups. +Only members of the project or group can access a private project's Container Registry. -### Control Container Registry for your project +If a project is public, so is the Container Registry. -Navigate to your project's **{package}** **Packages & Registries > Container Registry**. +## Use images from the Container Registry - +To download and run a container image hosted in the GitLab Container Registry: -This view allows you to: +1. Copy the link to your container image: + - Go to your project or group's **Packages & Registries > Container Registry** + and find the image you want. + - Next to the image name, click the **Copy** button. -- Show all the image repositories that belong to the project. -- Filter image repositories by their name. -- [Delete](#delete-images-from-within-gitlab) one or more image repository. -- Navigate to the image repository details page. -- Show a **Quick start** dropdown with the most common commands to log in, build and push. -- Show a banner if the optional [cleanup policy](#cleanup-policy) is enabled for this project. +  -### Control Container Registry for your group +1. Use `docker run` with the image link: -Navigate to your group's **{package}** **Packages & Registries > Container Registry**. + ```shell + docker run [options] registry.example.com/group/project/image [arguments] + ``` - +For more information on running Docker containers, visit the +[Docker documentation](https://docs.docker.com/engine/userguide/intro/). -This view allows you to: +## Image naming convention -- Show all the image repositories of the projects that belong to this group. -- [Delete](#delete-images-from-within-gitlab) one or more image repositories. -- Navigate to a specific image repository details page. +Images follow this naming convention: -### Image Repository details page +```plaintext +<registry URL>/<namespace>/<project>/<image> +``` -Clicking on the name of any image repository navigates to the details. +If your project is `gitlab.example.com/mynamespace/myproject`, for example, +then your image must be named `gitlab.example.com/mynamespace/myproject/my-app` at a minimum. - +You can append additional names to the end of an image name, up to three levels deep. -NOTE: **Note:** -The following page has the same functionalities both in the **Group level container registry** -and in the **Project level container registry**. +For example, these are all valid image names for images within the project named `myproject`: -This view: +```plaintext +registry.example.com/mynamespace/myproject:some-tag +``` -- Shows all the image repository details. -- Shows all the tags of the image repository. -- Allows you to quickly copy the tag path (by clicking on the clipboard button near the tag name). -- Allows you to [delete one or more tags](#delete-images-from-within-gitlab). +```plaintext +registry.example.com/mynamespace/myproject/image:latest +``` -## Use images from GitLab Container Registry +```plaintext +registry.example.com/mynamespace/myproject/my/image:rc1 +``` -To download and run a container from images hosted in GitLab Container Registry, -use `docker run`: +## Build and push images by using Docker commands -```shell -docker run [options] registry.example.com/group/project/image [arguments] -``` +To build and push to the Container Registry, you can use Docker commands. -For more information on running Docker containers, visit the -[Docker documentation](https://docs.docker.com/engine/userguide/intro/). +### Authenticate with the Container Registry -## Authenticating to the GitLab Container Registry +Before you can build and push images, you must authenticate with the Container Registry. -If you visit the **Packages & Registries > Container Registry** link under your project's -menu, you can see the explicit instructions to login to the Container Registry -using your GitLab credentials. +To authenticate, you can use: -For example if the Registry's URL is `registry.example.com`, then you should be -able to login with: +- A [personal access token](../../profile/personal_access_tokens.md). +- A [deploy token](../../project/deploy_tokens/index.md). -```shell -docker login registry.example.com -``` +Both of these require the minimum scope to be: -NOTE: **Note:** -If you have [2 Factor Authentication](../../profile/account/two_factor_authentication.md) -enabled in your account, you need to pass a -[personal access token](../../profile/personal_access_tokens.md) instead -of your password in order to login to GitLab's Container Registry. +- For read (pull) access, `read_registry`. +- For write (push) access, `write_registry`. -Credentials must be provided for authorization to any non-public registry. Only project members can access private, -GitLab-hosted registries. +To authenticate, run the `docker` command. For example: -There are two ways to authenticate: + ```shell + docker login registry.example.com -u <username> -p <token> + ``` -- By using a [personal access token](../../profile/personal_access_tokens.md). -- By using a [deploy token](../../project/deploy_tokens/index.md). +### Build and push images by using Docker commands -The minimum scope needed for both of them is `read_registry`. +To build and push to the Container Registry: -Example of using a token: +1. Authenticate with the Container Registry. -```shell -docker login registry.example.com -u <username> -p <token> -``` +1. Run the command to build or push. For example, to build: -## Build and push images from your local machine + ```shell + docker build -t registry.example.com/group/project/image . + ``` -Building and publishing images should be a straightforward process. Just make -sure that you are using the Registry URL with the namespace and project name -that is hosted on GitLab: + Or to push: -```shell -docker build -t registry.example.com/group/project/image . -docker push registry.example.com/group/project/image -``` + ```shell + docker push registry.example.com/group/project/image + ``` -Your image is named after the following scheme: +You can also view these commands by going to your project's **Packages & Registries > Container Registry**. -```plaintext -<registry URL>/<namespace>/<project>/<image> -``` +## Build and push by using GitLab CI/CD -GitLab supports up to three levels of image repository names. -The following examples of image tags are valid: +Use [GitLab CI/CD](../../../ci/yaml/README.md) to build and push images to the +Container Registry. Use it to test, build, and deploy your project from the Docker +image you created. -```plaintext -registry.example.com/group/project:some-tag -registry.example.com/group/project/image:latest -registry.example.com/group/project/my/image:rc1 -``` +### Authenticate by using GitLab CI/CD -## Build and push images using GitLab CI/CD - -While you can build and push your images from your local machine, take -full advantage of the Container Registry by combining it with GitLab CI/CD. -You can then create workflows and automate any processes that involve testing, -building, and eventually deploying your project from the Docker image you -created. - -Before diving into the details, some things you should be aware of: - -- You must [authenticate to the container registry](#authenticating-to-the-container-registry-with-gitlab-cicd) - before running any commands. You can do this in the `before_script` if multiple - jobs depend on it. -- Using `docker build --pull` fetches any changes to base - images before building in case your cache is stale. It takes slightly - longer, but it means you don’t get stuck without security patches for base images. -- Doing an explicit `docker pull` before each `docker run` fetches - the latest image that was just built. This is especially important if you are - using multiple runners that cache images locally. Using the Git SHA in your - image tag makes this less necessary since each job is unique and you - shouldn't ever have a stale image. However, it's still possible to have a - stale image if you re-build a given commit after a dependency has changed. -- You don't want to build directly to `latest` tag in case there are multiple jobs - happening simultaneously. +Before you can build and push images by using GitLab CI/CD, you must authenticate with the Container Registry. -### Authenticating to the Container Registry with GitLab CI/CD +To use CI/CD to authenticate, you can use: -There are three ways to authenticate to the Container Registry via -[GitLab CI/CD](../../../ci/yaml/README.md): +- The `CI_REGISTRY_USER` variable. -- **Using the special `CI_REGISTRY_USER` variable**: The user specified by this variable is created for you in order to - push to the Registry connected to your project. Its password is automatically - set with the `CI_REGISTRY_PASSWORD` variable. This allows you to automate building and deploying - your Docker images and has read/write access to the Registry. This is ephemeral, - so it's only valid for one job. You can use the following example as-is: + This variable has read-write access to the Container Registry and is valid for + one job only. Its password is also automatically created and assigned to `CI_REGISTRY_PASSWORD`. ```shell docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY ``` -- **Using the GitLab Deploy Token**: You can create and use a - [special deploy token](../../project/deploy_tokens/index.md#gitlab-deploy-token) - with your projects. - Once created, you can use the special environment variables, and GitLab CI/CD - fills them in for you. You can use the following example as-is: +- A [CI job token](../../../ci/triggers/README.md#ci-job-token). ```shell - docker login -u $CI_DEPLOY_USER -p $CI_DEPLOY_PASSWORD $CI_REGISTRY + docker login -u $CI_JOB_USER -p $CI_JOB_TOKEN $CI_REGISTRY ``` -- **Using a personal access token**: You can create and use a - [personal access token](../../profile/personal_access_tokens.md) - in case your project is private: +- A [deploy token](../../project/deploy_tokens/index.md#gitlab-deploy-token) with the minimum scope of: + - For read (pull) access, `read_registry`. + - For write (push) access, `write_registry`. - - For read (pull) access, the scope should be `read_registry`. - - For write (push) access, the scope should be `write_registry`. + ```shell + docker login -u $CI_DEPLOY_USER -p $CI_DEPLOY_PASSWORD $CI_REGISTRY + ``` - Replace the `<username>` and `<access_token>` in the following example: +- A [personal access token](../../profile/personal_access_tokens.md) with the minimum scope of: + - For read (pull) access, `read_registry`. + - For write (push) access, `write_registry`. ```shell docker login -u <username> -p <access_token> $CI_REGISTRY ``` +### Configure your `.gitlab-ci.yml` file + +You can configure your `.gitlab-ci.yml` file to build and push images to the Container Registry. + +- If multiple jobs require authentication, put the authentication command in the `before_script`. +- Before building, use `docker build --pull` to fetch changes to base images. It takes slightly + longer, but it ensures your image is up-to-date. +- Before each `docker run`, do an explicit `docker pull` to fetch + the image that was just built. This is especially important if you are + using multiple runners that cache images locally. + + If you use the Git SHA in your image tag, each job is unique and you + should never have a stale image. However, it's still possible to have a + stale image if you re-build a given commit after a dependency has changed. +- Don't build directly to the `latest` tag because multiple jobs may be + happening simultaneously. + ### Container Registry examples with GitLab CI/CD If you're using Docker-in-Docker on your runners, this is how your `.gitlab-ci.yml` @@ -359,15 +310,15 @@ in addition to the steps in the Below is an example of what your `.gitlab-ci.yml` should look like: ```yaml - build: - image: $CI_REGISTRY/group/project/docker:19.03.12 - services: - - name: $CI_REGISTRY/group/project/docker:19.03.12-dind - alias: docker - stage: build - script: - - docker build -t my-docker-image . - - docker run my-docker-image /script/to/run/tests +build: + image: $CI_REGISTRY/group/project/docker:19.03.12 + services: + - name: $CI_REGISTRY/group/project/docker:19.03.12-dind + alias: docker + stage: build + script: + - docker build -t my-docker-image . + - docker run my-docker-image /script/to/run/tests ``` If you forget to set the service alias, the `docker:19.03.12` image is unable to find the @@ -394,7 +345,7 @@ the deleted images. To delete images from within GitLab: -1. Navigate to your project's or group's **{package}** **Packages & Registries > Container Registry**. +1. Navigate to your project's or group's **Packages & Registries > Container Registry**. 1. From the **Container Registry** page, you can select what you want to delete, by either: @@ -406,8 +357,6 @@ To delete images from within GitLab: 1. In the dialog box, click **Remove tag**. -  - ### Delete images using the API If you want to automate the process of deleting images, GitLab provides an API. For more @@ -512,9 +461,28 @@ Cleanup policies can be run on all projects, with these exceptions: for all projects (even those created before 12.8) in [GitLab application settings](../../../api/settings.md#change-application-settings) by setting `container_expiration_policies_enable_historic_entries` to true. + Alternatively, you can execute the following command in the [Rails console](../../../administration/operations/rails_console.md#starting-a-rails-console-session): + + ```ruby + ApplicationSetting.last.update(container_expiration_policies_enable_historic_entries: true) + ``` There are performance risks with enabling it for all projects, especially if you are using an [external registry](./index.md#use-with-external-container-registries). +- For self-managed GitLab instances, you can enable or disable the cleanup policy for a specific + project. + + To enable it: + + ```ruby + Feature.enable(:container_expiration_policies_historic_entry, Project.find(<project id>)) + ``` + + To disable it: + + ```ruby + Feature.disable(:container_expiration_policies_historic_entry, Project.find(<project id>)) + ``` ### How the cleanup policy works @@ -636,7 +604,7 @@ you can use the Container Registry to store Helm Charts. However, due to the way and stored by Docker, it is not possible for GitLab to parse this data and meet performance standards. [This epic](https://gitlab.com/groups/gitlab-org/-/epics/2313) updates the architecture of the Container Registry to support Helm Charts. -You can read more about the above challenges [here](https://gitlab.com/gitlab-org/gitlab/-/issues/38047#note_298842890). +[Read more about the above challenges](https://gitlab.com/gitlab-org/gitlab/-/issues/38047#note_298842890). ## Limitations @@ -647,6 +615,19 @@ Container Registry, you must delete all existing images. - Prior to GitLab 12.10, any tags that use the same image ID as the `latest` tag are not deleted by the cleanup policy. +## Disable the Container Registry for a project + +The Container Registry is enabled by default. + +You can, however, remove the Container Registry for a project: + +1. Go to your project's **Settings > General** page. +1. Expand the **Visibility, project features, permissions** section + and disable **Container Registry**. +1. Click **Save changes**. + +The **Packages & Registries > Container Registry** entry is removed from the project's sidebar. + ## Troubleshooting the GitLab Container Registry ### Docker connection error @@ -661,6 +642,21 @@ To get around this, you can [change the group path](../../group/index.md#changin [change the project path](../../project/settings/index.md#renaming-a-repository) or change the branch name. +You may also get a `404 Not Found` or `Unknown Manifest` message if you are using +a Docker Engine version earlier than 17.12. Later versions of Docker Engine use +[the v2 API](https://docs.docker.com/registry/spec/manifest-v2-2/). + +The images in your GitLab Container Registry must also use the Docker v2 API. +For information on how to update your images, see the [Docker help](https://docs.docker.com/registry/spec/deprecated-schema-v1). + +### `Blob unknown to registry` error when pushing a manifest list + +When [pushing a Docker manifest list](https://docs.docker.com/engine/reference/commandline/manifest/#create-and-push-a-manifest-list) to the GitLab Container Registry, you may receive the error `manifest blob unknown: blob unknown to registry`. This issue occurs when the individual child manifests referenced in the manifest list were not pushed to the same repository. + +For example, you may have two individual images, one for `amd64` and another for `arm64v8`, and you want to build a multi-arch image with them. The `amd64` and `arm64v8` images must be pushed to the same repository where you want to push the multi-arch image. + +As a workaround, you should include the architecture in the tag name of individual images. For example, use `mygroup/myapp:1.0.0-amd64` instead of using sub repositories, like `mygroup/myapp/amd64:1.0.0`. You can then tag the manifest list with `mygroup/myapp:1.0.0`. + ### Troubleshoot as a GitLab server admin Troubleshooting the GitLab Container Registry, most of the times, requires diff --git a/doc/user/packages/dependency_proxy/img/group_dependency_proxy.png b/doc/user/packages/dependency_proxy/img/group_dependency_proxy.png Binary files differdeleted file mode 100644 index e550d296d5a..00000000000 --- a/doc/user/packages/dependency_proxy/img/group_dependency_proxy.png +++ /dev/null diff --git a/doc/user/packages/dependency_proxy/index.md b/doc/user/packages/dependency_proxy/index.md index e3ee8909165..3fa21ef486b 100644 --- a/doc/user/packages/dependency_proxy/index.md +++ b/doc/user/packages/dependency_proxy/index.md @@ -8,80 +8,80 @@ info: To determine the technical writer assigned to the Stage/Group associated w > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7934) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.11. -NOTE: **Note:** -This is the user guide. In order to use the dependency proxy, an administrator -must first [configure it](../../../administration/packages/dependency_proxy.md). +The GitLab Dependency Proxy is a local proxy you can use for your frequently-accessed +upstream images. -For many organizations, it is desirable to have a local proxy for frequently used -upstream images/packages. In the case of CI/CD, the proxy is responsible for -receiving a request and returning the upstream image from a registry, acting -as a pull-through cache. +In the case of CI/CD, the Dependency Proxy receives a request and returns the +upstream image from a registry, acting as a pull-through cache. -The dependency proxy is available in the group level. To access it, navigate to -a group's **Packages & Registries > Dependency Proxy**. +## Prerequisites - +To use the Dependency Proxy: -## Supported dependency proxies +- Your group must be public. Authentication for private groups is [not supported yet](https://gitlab.com/gitlab-org/gitlab/-/issues/11582). -NOTE: **Note:** -For a list of the upcoming additions to the proxies, visit the -[direction page](https://about.gitlab.com/direction/package/dependency_proxy/#top-vision-items). +### Supported images and packages -The following dependency proxies are supported. +The following images and packages are supported. -| Dependency proxy | GitLab version | +| Image/Package | GitLab version | | ---------------- | -------------- | | Docker | 11.11+ | -## Using the Docker dependency proxy +For a list of planned additions, view the +[direction page](https://about.gitlab.com/direction/package/dependency_proxy/#top-vision-items). + +## View the Dependency Proxy + +To view the Dependency Proxy: + +- Go to your group's **Packages & Registries > Dependency Proxy**. + +The Dependency Proxy is not available for projects. + +## Use the Dependency Proxy for Docker images + +You can use GitLab as a source for your Docker images. -With the Docker dependency proxy, you can use GitLab as a source for a Docker image. -To get a Docker image into the dependency proxy: +Prerequisites: -1. Find the proxy URL on your group's page under **Packages & Registries > Dependency Proxy**, - for example `gitlab.com/groupname/dependency_proxy/containers`. -1. Trigger GitLab to pull the Docker image you want (e.g., `alpine:latest` or - `linuxserver/nextcloud:latest`) and store it in the proxy storage by using - one of the following ways: +- Your images must be stored on [Docker Hub](https://hub.docker.com/). +- Docker Hub must be available. Follow [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/241639) + for progress on accessing images when Docker Hub is down. - - Manually pulling the Docker image: +To store a Docker image in Dependency Proxy storage: + +1. Go to your group's **Packages & Registries > Dependency Proxy**. +1. Copy the **Dependency Proxy URL**. +1. Use one of these commands. In these examples, the image is `alpine:latest`. + + - Add the URL to your [`.gitlab-ci.yml`](../../../ci/yaml/README.md#image) file: ```shell - docker pull gitlab.com/groupname/dependency_proxy/containers/alpine:latest + image: gitlab.example.com/groupname/dependency_proxy/containers/alpine:latest ``` - - From a `Dockerfile`: + - Manually pull the Docker image: ```shell - FROM gitlab.com/groupname/dependency_proxy/containers/alpine:latest + docker pull gitlab.example.com/groupname/dependency_proxy/containers/alpine:latest ``` - - In [`.gitlab-ci.yml`](../../../ci/yaml/README.md#image): + - Add the URL to a `Dockerfile`: ```shell - image: gitlab.com/groupname/dependency_proxy/containers/alpine:latest + FROM gitlab.example.com/groupname/dependency_proxy/containers/alpine:latest ``` GitLab pulls the Docker image from Docker Hub and caches the blobs on the GitLab server. The next time you pull the same image, GitLab gets the latest -information about the image from Docker Hub but serves the existing blobs +information about the image from Docker Hub, but serves the existing blobs from the GitLab server. -The blobs are kept forever, and there is no hard limit on how much data can be -stored. - -## Clearing the cache +## Clear the Dependency Proxy cache -It is possible to use the GitLab API to purge the dependency proxy cache for a -given group to gain back disk space that may be taken up by image blobs that -are no longer needed. See the [dependency proxy API documentation](../../../api/dependency_proxy.md) -for more details. - -## Limitations - -The following limitations apply: +Blobs are kept forever on the GitLab server, and there is no hard limit on how much data can be +stored. -- Only [public groups are supported](https://gitlab.com/gitlab-org/gitlab/-/issues/11582) (authentication is not supported yet). -- Only Docker Hub is supported. -- This feature requires Docker Hub being available. +To reclaim disk space used by image blobs that are no longer needed, use +the [Dependency Proxy API](../../../api/dependency_proxy.md). diff --git a/doc/user/packages/generic_packages/index.md b/doc/user/packages/generic_packages/index.md new file mode 100644 index 00000000000..0c961b5c86f --- /dev/null +++ b/doc/user/packages/generic_packages/index.md @@ -0,0 +1,139 @@ +--- +stage: Package +group: Package +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# GitLab Generic Packages Repository **(CORE)** + +> - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/4209) in GitLab 13.5. +> - It's [deployed behind a feature flag](../../../user/feature_flags.md), enabled by default. +> - It's enabled on GitLab.com. +> - It's able to be enabled or disabled per-project. +> - It's recommended for production use. +> - For GitLab self-managed instances, GitLab administrators can opt to [disable it](#enable-or-disable-generic-packages-in-the-package-registry). + +CAUTION: **Warning:** +This feature might not be available to you. Check the **version history** note above for details. + +Publish generic files, like release binaries, in your project’s Package Registry. Then, install the packages whenever you need to use them as a dependency. + +## Authenticate to the Package Registry + +To authenticate to the Package Registry, you need either a [personal access token](../../../api/README.md#personalproject-access-tokens) +or [CI job token](../../../api/README.md#gitlab-ci-job-token). + +## Publish a package file + +When you publish a package file, if the package does not exist, it is created. + +Prerequisites: + +- You need to [authenticate with the API](../../../api/README.md#authentication). + +```plaintext +PUT /projects/:id/packages/generic/:package_name/:package_version/:file_name +``` + +| Attribute | Type | Required | Description | +| -------------------| --------------- | ---------| -------------------------------------------------------------------------------------------------------------------------------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](../../../api/README.md#namespaced-path-encoding). | +| `package_name` | string | yes | The package name. It can contain only lowercase letters (`a-z`), uppercase letter (`A-Z`), numbers (`0-9`), dots (`.`), hyphens (`-`), or underscores (`_`). +| `package_version` | string | yes | The package version. It can contain only numbers (`0-9`), and dots (`.`). Must be in the format of `X.Y.Z`, i.e. should match `/\A\d+\.\d+\.\d+\z/` regular expresion. +| `file_name` | string | yes | The file name. It can contain only lowercase letters (`a-z`), uppercase letter (`A-Z`), numbers (`0-9`), dots (`.`), hyphens (`-`), or underscores (`_`). + +Provide the file context in the request body. + +Example request: + +```shell +curl --header "PRIVATE-TOKEN: <your_access_token>" \ + --upload-file path/to/file.txt \ + https://gitlab.example.com/api/v4/projects/24/generic/my_package/0.0.1/file.txt +``` + +Example response: + +```json +{ + "message":"201 Created" +} +``` + +## Download package file + +Install a package file. + +Prerequisites: + +- You need to [authenticate with the API](../../../api/README.md#authentication). + +```plaintext +GET /projects/:id/packages/generic/:package_name/:package_version/:file_name +``` + +| Attribute | Type | Required | Description | +| -------------------| --------------- | ---------| ------------------------------------------------------------------------------------| +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](../../../api/README.md#namespaced-path-encoding). | +| `package_name` | string | yes | The package name. | +| `package_version` | string | yes | The package version. | +| `file_name` | string | yes | The file name. | + +The file context is served in the response body. The response content type will be `application/octet-stream`. + +Example request that uses a personal access token: + +```shell +curl --header "PRIVATE-TOKEN: <your_access_token>" \ + https://gitlab.example.com/api/v4/projects/24/generic/my_package/0.0.1/file.txt +``` + +## Publish a generic package by using CI/CD + +To work with generic packages in [GitLab CI/CD](./../../../ci/README.md), you can use +`CI_JOB_TOKEN` in place of the personal access token in your commands. + +For example: + +```yaml +image: curlimages/curl:latest + +stages: + - upload + - download + +upload: + stage: upload + script: + - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file path/to/file.txt ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/my_package/0.0.1/file.txt' + +download: + stage: download + script: + - 'wget --header="JOB-TOKEN: $CI_JOB_TOKEN" ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/my_package/0.0.1/file.txt' +``` + +### Enable or disable generic packages in the Package Registry + +Support for generic packages is under development but ready for production use. +It is deployed behind a feature flag that is **enabled by default**. +[GitLab administrators with access to the GitLab Rails console](../../../administration/feature_flags.md) +can opt to disable it. + +To enable it: + +```ruby +# For the instance +Feature.enable(:generic_packages) +# For a single project +Feature.enable(:generic_packages, Project.find(<project id>)) +``` + +To disable it: + +```ruby +# For the instance +Feature.disable(:generic_packages) +# For a single project +Feature.disable(:generic_packages, Project.find(<project id>)) +``` diff --git a/doc/user/packages/go_proxy/index.md b/doc/user/packages/go_proxy/index.md index edf1528a751..bd3b5b49ebd 100644 --- a/doc/user/packages/go_proxy/index.md +++ b/doc/user/packages/go_proxy/index.md @@ -50,7 +50,7 @@ Feature.disable(:go_proxy, Project.find(2)) ### Enable the Package Registry The Package Registry is enabled for new projects by default. If you cannot find -the **{package}** **Packages > List** entry under your project's sidebar, verify +the **Packages > List** entry under your project's sidebar, verify the following: 1. Your GitLab administrator has diff --git a/doc/user/packages/index.md b/doc/user/packages/index.md index 92d31c31986..53ba3d01b3e 100644 --- a/doc/user/packages/index.md +++ b/doc/user/packages/index.md @@ -23,6 +23,7 @@ The Package Registry supports the following formats: <tr><td><a href="https://docs.gitlab.com/ee/user/packages/npm_registry/index.html">NPM</a></td><td>11.7+</td></tr> <tr><td><a href="https://docs.gitlab.com/ee/user/packages/nuget_repository/index.html">NuGet</a></td><td>12.8+</td></tr> <tr><td><a href="https://docs.gitlab.com/ee/user/packages/pypi_repository/index.html">PyPI</a></td><td>12.10+</td></tr> +<tr><td><a href="https://docs.gitlab.com/ee/user/packages/generic_packages/index.html">Generic packages</a></td><td>13.5+</td></tr> </table> </div> </div> diff --git a/doc/user/packages/maven_repository/img/maven_package_view_v12_6.png b/doc/user/packages/maven_repository/img/maven_package_view_v12_6.png Binary files differdeleted file mode 100644 index 92cefc26660..00000000000 --- a/doc/user/packages/maven_repository/img/maven_package_view_v12_6.png +++ /dev/null diff --git a/doc/user/packages/maven_repository/index.md b/doc/user/packages/maven_repository/index.md index 7329725a643..d4a8ff0cdb4 100644 --- a/doc/user/packages/maven_repository/index.md +++ b/doc/user/packages/maven_repository/index.md @@ -4,44 +4,24 @@ group: Package info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# GitLab Maven Repository +# Maven packages in the Package Repository > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/5811) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.3. > - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/221259) to GitLab Core in 13.3. -With the GitLab [Maven](https://maven.apache.org) Repository, every -project can have its own space to store its Maven artifacts. +Publish [Maven](https://maven.apache.org) artifacts in your project’s Package Registry. +Then, install the packages whenever you need to use them as a dependency. - +## Build a Maven package -## Enabling the Maven Repository +This section explains how to install Maven and build a package. -NOTE: **Note:** -This option is available only if your GitLab administrator has -[enabled support for the Maven repository](../../../administration/packages/index.md). - -After the Packages feature is enabled, the Maven Repository is available for -all new projects by default. To enable it for existing projects, or if you want -to disable it: - -1. Navigate to your project's **Settings > General > Visibility, project features, permissions**. -1. Find the Packages feature and enable or disable it. -1. Click on **Save changes** for the changes to take effect. - -You should then be able to see the **Packages & Registries** section on the left sidebar. -Next, you must configure your project to authorize with the GitLab Maven -repository. +If you already use Maven and know how to build your own packages, go to the +[next section](#add-the-package-registry-as-a-maven-remote). -## Getting Started with Maven +Maven repositories work well with Gradle, too. To set up a Gradle project, see [get started with Gradle](#use-gradle-to-create-a-java-project). -This section covers installing Maven and building a package. This is a -quickstart to help if you're new to building Maven packages. If you're already -using Maven and understand how to build your own packages, move onto the -[next section](#adding-the-gitlab-package-registry-as-a-maven-remote). - -Maven repositories work well with Gradle, too. Move onto [getting started with Gradle](#getting-started-with-gradle) if you want to setup a Gradle project. - -### Installing Maven +### Install Maven The required minimum versions are: @@ -66,7 +46,7 @@ Default locale: en_GB, platform encoding: UTF-8 OS name: "mac os x", version: "10.15.2", arch: "x86_64", family: "mac" ``` -### Creating a project +### Create a project Understanding how to create a full Java project is outside the scope of this guide but you can follow the steps below to create a new project that can be @@ -105,14 +85,14 @@ your project has been set up successfully: You should see a new directory where you ran this command matching your `DartifactId` parameter (in this case it should be `my-project`). -## Getting started with Gradle +## Use Gradle to create a Java project + +This section explains how to install Gradle and initialize a Java project. -This section covers installing Gradle and initializing a Java project. This is a -quickstart to help if you're new to Gradle. If you're already -using Gradle and understand how to build your own packages, move onto the -[next section](#adding-the-gitlab-package-registry-as-a-maven-remote). +If you already use Gradle and know how to build your own packages, go to the +[next section](#add-the-package-registry-as-a-maven-remote). -### Installing Gradle +### Install Gradle Installation is needed only if you want to create a new Gradle project. Follow instructions at [gradle.org](https://gradle.org/install/) to download and install @@ -145,7 +125,7 @@ JVM: 11.0.5 (Oracle Corporation 11.0.5+10) OS: Windows 10 10.0 amd64 ``` -### Creating a project in Gradle +### Create a Java project Understanding how to create a full Java project in Gradle is outside the scope of this guide, but you can follow the steps below to create a new project that can be @@ -209,23 +189,23 @@ Project name (default: test): Enter a project name or hit enter to use the directory name as project name. -## Adding the GitLab Package Registry as a Maven remote +## Add the Package Registry as a Maven remote The next step is to add the GitLab Package Registry as a Maven remote. If a project is private or you want to upload Maven artifacts to GitLab, credentials must be provided for authorization too. Support is available -for [personal access tokens](#authenticating-with-a-personal-access-token), -[CI job tokens](#authenticating-with-a-ci-job-token), and +for [personal access tokens](#authenticate-with-a-personal-access-token), +[CI job tokens](#authenticate-with-a-ci-job-token), and [deploy tokens](../../project/deploy_tokens/index.md) only. Regular username/password credentials do not work. -### Authenticating with a personal access token +### Authenticate with a personal access token To authenticate with a [personal access token](../../profile/personal_access_tokens.md), set the scope to `api` when creating one, and add it to your Maven or Gradle configuration files. -#### Authenticating with a personal access token in Maven +#### Authenticate with a personal access token in Maven Add a corresponding section to your [`settings.xml`](https://maven.apache.org/settings.html) file: @@ -248,7 +228,7 @@ Add a corresponding section to your </settings> ``` -#### Authenticating with a personal access token in Gradle +#### Authenticate with a personal access token in Gradle Create a file `~/.gradle/gradle.properties` with the following content: @@ -278,12 +258,12 @@ repositories { You should now be able to upload Maven artifacts to your project. -### Authenticating with a CI job token +### Authenticate with a CI job token If you're using GitLab CI/CD, a CI job token can be used instead of a personal access token. -#### Authenticating with a CI job token in Maven +#### Authenticate with a CI job token in Maven To authenticate with a CI job token, add a corresponding section to your [`settings.xml`](https://maven.apache.org/settings.html) file: @@ -309,7 +289,7 @@ To authenticate with a CI job token, add a corresponding section to your You can read more on [how to create Maven packages using GitLab CI/CD](#creating-maven-packages-with-gitlab-cicd). -#### Authenticating with a CI job token in Gradle +#### Authenticate with a CI job token in Gradle To authenticate with a CI job token, add a repositories section to your [`build.gradle`](https://docs.gradle.org/current/userguide/tutorial_using_tasks.html) @@ -331,7 +311,7 @@ repositories { } ``` -### Authenticating with a deploy token +### Authenticate with a deploy token > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/213566) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.0. @@ -339,7 +319,7 @@ To authenticate with a [deploy token](./../../project/deploy_tokens/index.md), set the scope to `api` when creating one, and add it to your Maven or Gradle configuration files. -#### Authenticating with a deploy token in Maven +#### Authenticate with a deploy token in Maven Add a corresponding section to your [`settings.xml`](https://maven.apache.org/settings.html) file: @@ -362,7 +342,7 @@ Add a corresponding section to your </settings> ``` -#### Authenticating with a deploy token in Gradle +#### Authenticate with a deploy token in Gradle To authenticate with a deploy token, add a repositories section to your [`build.gradle`](https://docs.gradle.org/current/userguide/tutorial_using_tasks.html) @@ -441,7 +421,7 @@ repositories { ``` The `id` must be the same with what you -[defined in `settings.xml`](#adding-the-gitlab-package-registry-as-a-maven-remote). +[defined in `settings.xml`](#add-the-package-registry-as-a-maven-remote). Replace `PROJECT_ID` with your project ID which can be found on the home page of your project. @@ -452,7 +432,7 @@ domain name. NOTE: **Note:** For retrieving artifacts, you can use either the [URL encoded](../../../api/README.md#namespaced-path-encoding) path of the project -(e.g., `group%2Fproject`) or the project's ID (e.g., `42`). However, only the +(such as `group%2Fproject`) or the project's ID (such as `42`). However, only the project's ID can be used for uploading. ### Group level Maven endpoint @@ -505,7 +485,7 @@ repositories { ``` The `id` must be the same with what you -[defined in `settings.xml`](#adding-the-gitlab-package-registry-as-a-maven-remote). +[defined in `settings.xml`](#add-the-package-registry-as-a-maven-remote). Replace `my-group` with your group name and `PROJECT_ID` with your project ID which can be found on the home page of your project. @@ -516,7 +496,7 @@ domain name. NOTE: **Note:** For retrieving artifacts, you can use either the [URL encoded](../../../api/README.md#namespaced-path-encoding) path of the group -(e.g., `group%2Fsubgroup`) or the group's ID (e.g., `12`). +(such as `group%2Fsubgroup`) or the group's ID (such as `12`). ### Instance level Maven endpoint @@ -571,7 +551,7 @@ repositories { ``` The `id` must be the same with what you -[defined in `settings.xml`](#adding-the-gitlab-package-registry-as-a-maven-remote). +[defined in `settings.xml`](#add-the-package-registry-as-a-maven-remote). Replace `PROJECT_ID` with your project ID which can be found on the home page of your project. @@ -582,12 +562,12 @@ domain name. NOTE: **Note:** For retrieving artifacts, you can use either the [URL encoded](../../../api/README.md#namespaced-path-encoding) path of the project -(e.g., `group%2Fproject`) or the project's ID (e.g., `42`). However, only the +(such as `group%2Fproject`) or the project's ID (such as `42`). However, only the project's ID can be used for uploading. ## Uploading packages -Once you have set up the [remote and authentication](#adding-the-gitlab-package-registry-as-a-maven-remote) +Once you have set up the [remote and authentication](#add-the-package-registry-as-a-maven-remote) and [configured your project](#configuring-your-project-to-use-the-gitlab-maven-repository-url), test to upload a Maven artifact from a project of yours. @@ -661,7 +641,7 @@ artifacts or even delete them. ## Installing a package Installing a package from the GitLab Package Registry requires that you set up -the [remote and authentication](#adding-the-gitlab-package-registry-as-a-maven-remote) +the [remote and authentication](#add-the-package-registry-as-a-maven-remote) as above. Once this is completed, there are two ways to install a package. ### Install using Maven with `mvn install` @@ -708,7 +688,7 @@ Package details page, allowing for quick and easy installation. ### Install using Gradle -Add a [dependency](https://docs.gradle.org/current/userguide/declaring_dependencies.html) to build.gradle in the dependencies section: +Add a [dependency](https://docs.gradle.org/current/userguide/declaring_dependencies.html) to `build.gradle` in the dependencies section: ```groovy dependencies { @@ -802,7 +782,7 @@ Docker container), and Maven uses the configured CI The example below shows how to create a new package each time the `master` branch is updated: -1. Make sure you use the Job-Token authentication as described in ["Authenticating with a CI job token in Gradle"](#authenticating-with-a-ci-job-token-in-gradle). +1. Make sure you use the Job-Token authentication as described in ["Authenticating with a CI job token in Gradle"](#authenticate-with-a-ci-job-token-in-gradle). 1. Add a `deploy` job to your `.gitlab-ci.yml` file: diff --git a/doc/user/packages/npm_registry/index.md b/doc/user/packages/npm_registry/index.md index 2a1c12c2afd..f15b31d8b67 100644 --- a/doc/user/packages/npm_registry/index.md +++ b/doc/user/packages/npm_registry/index.md @@ -286,6 +286,22 @@ By default, when an NPM package is not found in the GitLab NPM Registry, the req Administrators can disable this behavior in the [Continuous Integration settings](../../admin_area/settings/continuous_integration.md). +### Installing packages from other organizations + +You can route package requests to organizations and users outside of GitLab. + +To do this, add lines to your `.npmrc` file, replacing `my-org` with the namespace or group that owns your project's repository. The name is case-sensitive and must match the name of your group or namespace exactly. + +```shell +@foo:registry=https://gitlab.example.com/api/v4/packages/npm/ +//gitlab.com/api/v4/packages/npm/:_authToken= "<your_token>" +//gitlab.com/api/v4/projects/<your_project_id>/packages/npm/:_authToken= "<your_token>" + +@my-other-org:registry=https://gitlab.example.com/api/v4/packages/npm/ +//gitlab.com/api/v4/packages/npm/:_authToken= "<your_token>" +//gitlab.com/api/v4/projects/<your_project_id>/packages/npm/:_authToken= "<your_token>" +``` + ## Removing a package In the packages view of your project page, you can delete packages by clicking @@ -340,6 +356,13 @@ with your personal access token or deploy token): //gitlab.com/api/v4/projects/:_authToken=<your_token> ``` +You can also use `yarn config` instead of `npm config` when setting your auth-token dynamically: + +```shell +yarn config set '//gitlab.com/api/v4/projects/<your_project_id>/packages/npm/:_authToken' "<your_token>" +yarn config set '//gitlab.com/api/v4/packages/npm/:_authToken' "<your_token>" +``` + ### `npm publish` targets default NPM registry (`registry.npmjs.org`) Ensure that your package scope is set consistently in your `package.json` and `.npmrc` files. @@ -418,5 +441,8 @@ npm dist-tag rm @scope/package@version my-tag # Delete a tag from the package npm install @scope/package@my-tag # Install a specific tag ``` +NOTE: **Note:** +You cannot use your `CI_JOB_TOKEN` or deploy token with the `npm dist-tag` commands. View [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/258835) for details. + CAUTION: **Warning:** Due to a bug in NPM 6.9.0, deleting dist tags fails. Make sure your NPM version is greater than 6.9.1. diff --git a/doc/user/packages/nuget_repository/index.md b/doc/user/packages/nuget_repository/index.md index 9fb50ce71fb..22e1a95649d 100644 --- a/doc/user/packages/nuget_repository/index.md +++ b/doc/user/packages/nuget_repository/index.md @@ -19,7 +19,7 @@ The GitLab NuGet Repository works with: ## Setting up your development environment -[NuGet CLI 5.2 or later](https://www.nuget.org/downloads) is required. Earlier versions have not been tested +[NuGet CLI 5.1 or later](https://www.nuget.org/downloads) is required. Earlier versions have not been tested against the GitLab NuGet Repository and might not work. If you have [Visual Studio](https://visualstudio.microsoft.com/vs/), NuGet CLI is probably already installed. @@ -34,7 +34,7 @@ nuget help You should see something similar to: ```plaintext -NuGet Version: 5.2.0.6090 +NuGet Version: 5.1.0.6013 usage: NuGet <command> [args] [options] Type 'NuGet help <command>' for help on a specific command. @@ -44,7 +44,7 @@ Available commands: ``` NOTE: **Note:** -GitLab currently only supports NuGet v3. Earlier versions are not supported. +GitLab currently only supports NuGet's protocol version 3. Earlier versions are not supported. ### macOS support @@ -154,7 +154,7 @@ To add the GitLab NuGet Repository as a source for .NET, create a file named `nu When uploading packages, note that: -- The maximum allowed size is 50 Megabytes. +- The Package Registry on GitLab.com can store up to 500 MB of content. This limit is [configurable for self-managed GitLab instances](../../../administration/instance_limits.md#package-registry-limits). - If you upload the same package with the same version multiple times, each consecutive upload is saved as a separate file. When installing a package, GitLab serves the most recent file. - When uploading packages to GitLab, they are not displayed in the packages UI of your project @@ -250,21 +250,21 @@ is updated: 1. Add a `deploy` job to your `.gitlab-ci.yml` file: ```yaml - image: mcr.microsoft.com/dotnet/core/sdk:3.1 - - stages: - - deploy - - deploy: - stage: deploy - script: - - dotnet restore -p:Configuration=Release - - dotnet build -c Release - - dotnet pack -c Release - - dotnet nuget add source "$CI_SERVER_URL/api/v4/projects/$CI_PROJECT_ID/packages/nuget/index.json" --name gitlab --username gitlab-ci-token --password $CI_JOB_TOKEN --store-password-in-clear-text - - dotnet nuget push "bin/Release/*.nupkg" --source gitlab - only: - - master + image: mcr.microsoft.com/dotnet/core/sdk:3.1 + + stages: + - deploy + + deploy: + stage: deploy + script: + - dotnet restore -p:Configuration=Release + - dotnet build -c Release + - dotnet pack -c Release + - dotnet nuget add source "$CI_SERVER_URL/api/v4/projects/$CI_PROJECT_ID/packages/nuget/index.json" --name gitlab --username gitlab-ci-token --password $CI_JOB_TOKEN --store-password-in-clear-text + - dotnet nuget push "bin/Release/*.nupkg" --source gitlab + only: + - master ``` 1. Commit the changes and push it to your GitLab repository to trigger a new CI build. diff --git a/doc/user/packages/package_registry/index.md b/doc/user/packages/package_registry/index.md index 9f954627b05..ae9ca5b2333 100644 --- a/doc/user/packages/package_registry/index.md +++ b/doc/user/packages/package_registry/index.md @@ -17,7 +17,7 @@ packages, which can be easily consumed as a dependency in downstream projects. You can view packages for your project or group. 1. Go to the project or group. -1. Go to **{package}** **Packages & Registries > Package Registry**. +1. Go to **Packages & Registries > Package Registry**. You can search, sort, and filter packages on this page. @@ -31,7 +31,7 @@ authenticate with GitLab by using the `CI_JOB_TOKEN`. CI/CD templates, which you can use to get started, are in [this repo](https://gitlab.com/gitlab-org/gitlab/-/tree/master/lib/gitlab/ci/templates). -Learn more about [using CI/CD to build Maven packages](../maven_repository/index.md#creating-maven-packages-with-gitlab-cicd), [NPM packages](../npm_registry/index.md#publishing-a-package-with-cicd), [Composer packages](../composer_repository/index.md#publishing-the-package-with-cicd), [NuGet Packages](../nuget_repository/index.md#publishing-a-nuget-package-with-cicd), [Conan Packages](../conan_repository/index.md#using-gitlab-ci-with-conan-packages), and [PyPI packages](../pypi_repository/index.md#using-gitlab-ci-with-pypi-packages). +Learn more about [using CI/CD to build Maven packages](../maven_repository/index.md#creating-maven-packages-with-gitlab-cicd), [NPM packages](../npm_registry/index.md#publishing-a-package-with-cicd), [Composer packages](../composer_repository/index.md#publish-a-composer-package-by-using-cicd), [NuGet Packages](../nuget_repository/index.md#publishing-a-nuget-package-with-cicd), [Conan Packages](../conan_repository/index.md#publish-a-conan-package-by-using-cicd), [PyPI packages](../pypi_repository/index.md#using-gitlab-ci-with-pypi-packages), and [generic packages](../generic_packages/index.md#publish-a-generic-package-by-using-cicd). If you use CI/CD to build a package, extended activity information is displayed when you view the package details: @@ -45,7 +45,7 @@ user who triggered it. To download a package: -1. Go to **{package}** **Packages & Registries > Package Registry**. +1. Go to **Packages & Registries > Package Registry**. 1. Click the name of the package you want to download. 1. In the **Activity** section, click the name of the package you want to download. @@ -60,7 +60,7 @@ You can delete packages by using [the API](../../../api/packages.md#delete-a-pro To delete a package in the UI, from your group or project: -1. Go to **{package}** **Packages & Registries > Package Registry**. +1. Go to **Packages & Registries > Package Registry**. 1. Find the name of the package you want to delete. 1. Click **Delete**. @@ -71,7 +71,7 @@ The package is permanently deleted. The Package Registry is automatically enabled. If you are using a self-managed instance of GitLab, your administrator can remove -the menu item, **{package}** **Packages & Registries**, from the GitLab sidebar. For more information, +the menu item, **Packages & Registries**, from the GitLab sidebar. For more information, see the [administration documentation](../../../administration/packages/index.md). You can also remove the Package Registry for your project specifically: @@ -81,7 +81,7 @@ You can also remove the Package Registry for your project specifically: **Packages** feature. 1. Click **Save changes**. -The **{package}** **Packages & Registries > Package Registry** entry is removed from the sidebar. +The **Packages & Registries > Package Registry** entry is removed from the sidebar. ## Package workflows diff --git a/doc/user/packages/pypi_repository/index.md b/doc/user/packages/pypi_repository/index.md index 97f3f69d676..33c37ee6a6c 100644 --- a/doc/user/packages/pypi_repository/index.md +++ b/doc/user/packages/pypi_repository/index.md @@ -4,14 +4,14 @@ group: Package info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# GitLab PyPi Repository +# GitLab PyPI Repository > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/208747) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.10. > - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/221259) to GitLab Core in 13.3. -With the GitLab PyPi Repository, every project can have its own space to store PyPi packages. +With the GitLab PyPI Repository, every project can have its own space to store PyPI packages. -The GitLab PyPi Repository works with: +The GitLab PyPI Repository works with: - [pip](https://pypi.org/project/pip/) - [twine](https://pypi.org/project/twine/) @@ -20,13 +20,13 @@ The GitLab PyPi Repository works with: You need a recent version of [pip](https://pypi.org/project/pip/) and [twine](https://pypi.org/project/twine/). -## Enabling the PyPi Repository +## Enabling the PyPI Repository NOTE: **Note:** This option is available only if your GitLab administrator has [enabled support for the Package Registry](../../../administration/packages/index.md). -After the PyPi Repository is enabled, it is available for all new projects +After the PyPI Repository is enabled, it is available for all new projects by default. To enable it for existing projects, or if you want to disable it: 1. Navigate to your project's **Settings > General > Visibility, project features, permissions**. @@ -37,8 +37,8 @@ You should then be able to see the **Packages & Registries** section on the left ## Getting started -This section covers creating a new example PyPi package to upload. This is a -quickstart to test out the **GitLab PyPi Registry**. If you already understand how +This section covers creating a new example PyPI package to upload. This is a +quickstart to test out the **GitLab PyPI Registry**. If you already understand how to build and publish your own packages, move on to the [next section](#adding-the-gitlab-pypi-repository-as-a-source). ### Create a project @@ -111,6 +111,11 @@ touch setup.py This file contains all the information about our package. For more information about this file, see [creating setup.py](https://packaging.python.org/tutorials/packaging-projects/#creating-setup-py). +GitLab identifies packages based on +[Python normalized names (PEP-503)](https://www.python.org/dev/peps/pep-0503/#normalized-names), +so ensure your package name meets these requirements. +See the [installation section](#install-packages) for more details. + For this guide, we don't need to extensively fill out this file, simply add the below to your `setup.py`: @@ -152,10 +157,10 @@ And confirm your output matches the below: mypypipackage-0.0.1-py3-none-any.whl mypypipackage-0.0.1.tar.gz ``` -Our package is now all set up and ready to be uploaded to the **GitLab PyPi +Our package is now all set up and ready to be uploaded to the **GitLab PyPI Package Registry**. Before we do so, we next need to set up authentication. -## Adding the GitLab PyPi Repository as a source +## Adding the GitLab PyPI Repository as a source ### Authenticating with a personal access token @@ -256,7 +261,7 @@ TWINE_PASSWORD=<personal_access_token or deploy_token> TWINE_USERNAME=<username ``` If you did not follow the guide above, then you need to ensure your package -has been properly built and you [created a PyPi package with `setuptools`](https://packaging.python.org/tutorials/packaging-projects/). +has been properly built and you [created a PyPI package with `setuptools`](https://packaging.python.org/tutorials/packaging-projects/). You can then upload your package using the following command: @@ -274,7 +279,7 @@ Where: Install the latest version of a package using the following command: ```shell -pip install --index-url https://__token__:<personal_access_token>@gitlab.com/api/v4/projects/<project_id>/packages/pypi/simple --no-deps <package_name> +pip install --extra-index-url https://__token__:<personal_access_token>@gitlab.com/api/v4/projects/<project_id>/packages/pypi/simple --no-deps <package_name> ``` Where: @@ -287,7 +292,7 @@ If you were following the guide above and want to test installing the `MyPyPiPackage` package, you can run the following: ```shell -pip install mypypipackage --no-deps --index-url https://__token__:<personal_access_token>@gitlab.com/api/v4/projects/<your_project_id>/packages/pypi/simple +pip install mypypipackage --no-deps --extra-index-url https://__token__:<personal_access_token>@gitlab.com/api/v4/projects/<your_project_id>/packages/pypi/simple ``` This should result in the following: @@ -300,6 +305,12 @@ Installing collected packages: mypypipackage Successfully installed mypypipackage-0.0.1 ``` +GitLab looks for packages using +[Python normalized names (PEP-503)](https://www.python.org/dev/peps/pep-0503/#normalized-names), +so the characters `-`, `_`, and `.` are all treated the same and repeated characters are removed. +A `pip install` request for `my.package` looks for packages that match any of +the three characters, such as `my-package`, `my_package`, and `my....package`. + ## Using GitLab CI with PyPI packages > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/202012) in GitLab 13.4. diff --git a/doc/user/packages/workflows/monorepo.md b/doc/user/packages/workflows/monorepo.md index c87f181bf82..f20f3427ac5 100644 --- a/doc/user/packages/workflows/monorepo.md +++ b/doc/user/packages/workflows/monorepo.md @@ -7,17 +7,17 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Monorepo package management workflows Oftentimes, one project or Git repository may contain multiple different -subprojects or submodules that all get packaged and published individually. +sub-projects or submodules that all get packaged and published individually. ## Publishing different packages to the parent project The number and name of packages you can publish to one project is not limited. You can accomplish this by setting up different configuration files for each package. See the documentation for the package manager of your choice since -each will have its own specific files and instructions to follow to publish +each has its own specific files and instructions to follow to publish a given package. -Here, we will walk through how to do this with [NPM](../npm_registry/index.md). +Here, we take a walk through how to do this with [NPM](../npm_registry/index.md). Let us say we have a project structure like so: @@ -36,27 +36,27 @@ as well as `Foo`. Following the instructions in the [GitLab NPM registry documentation](../npm_registry/index.md), publishing `MyProject` consists of modifying the `package.json` file with a -`publishConfig` section, as well as either modifying your local NPM config with +`publishConfig` section, as well as either modifying your local NPM configuration with CLI commands like `npm config set`, or saving a `.npmrc` file in the root of the -project specifying these config settings. +project specifying these configuration settings. If you follow the instructions you can publish `MyProject` by running `npm publish` from the root directory. Publishing `Foo` is almost exactly the same, you simply have to follow the steps -while in the `Foo` directory. `Foo` will need its own `package.json` file, -which can be added manually or using `npm init`. And it will need its own +while in the `Foo` directory. `Foo` needs its own `package.json` file, +which can be added manually or using `npm init`. It also needs its own configuration settings. Since you are publishing to the same place, if you used `npm config set` to set the registry for the parent project, then no -additional setup is necessary. If you used a `.npmrc` file, you will need an +additional setup is necessary. If you used a `.npmrc` file, you need an additional `.npmrc` file in the `Foo` directory (be sure to add `.npmrc` files to the `.gitignore` file or use environment variables in place of your access tokens to prevent them from being exposed). It can be identical to the one you used in `MyProject`. You can now run `npm publish` from the `Foo` -directory and you will be able to publish `Foo` separately from `MyProject` +directory and you can publish `Foo` separately from `MyProject` A similar process could be followed for Conan packages, instead of dealing with -`.npmrc` and `package.json`, you will just be dealing with `conanfile.py` in +`.npmrc` and `package.json`, you just deal with `conanfile.py` in multiple locations within the project. ## Publishing to other projects @@ -64,9 +64,9 @@ multiple locations within the project. A package is associated with a project on GitLab, but the package does not need to be associated with the code in that project. Notice when configuring NPM or Maven, you only use the `Project ID` to set the registry URL that the -package will be uploaded to. If you set this to any project that you have -access to and update any other config similarly depending on the package type, -your packages will be published to that project. This means you can publish +package is to be uploaded to. If you set this to any project that you have +access to and update any other configuration similarly depending on the package type, +your packages are published to that project. This means you can publish multiple packages to one project, even if their code does not exist in the same place. See the [project registry workflow documentation](./project_registry.md) for more details. @@ -77,7 +77,7 @@ CI pipelines open an entire world of possibilities for dealing with the patterns described in the previous sections. A common desire would be to publish specific packages only if changes were made to those directories. -Using the example project above, this `gitlab-ci.yml` file will publish +Using the example project above, this `gitlab-ci.yml` file publishes `Foo` anytime changes are made to the `Foo` directory on the `master` branch, and publish `MyPackage` anytime changes are made to anywhere _except_ the `Foo` directory on the `master` branch. diff --git a/doc/user/packages/workflows/project_registry.md b/doc/user/packages/workflows/project_registry.md index 571cda09e69..24437e6dfac 100644 --- a/doc/user/packages/workflows/project_registry.md +++ b/doc/user/packages/workflows/project_registry.md @@ -26,8 +26,8 @@ point them at the same project on GitLab. There are a few reasons you might want to publish all your packages to one project on GitLab: 1. You want to publish your packages on GitLab, but to a project that is different from where your code is stored. -1. You would like to group packages together in ways that make sense for your usage (all NPM packages in one project, - all packages being used by a specific department in one project, all private packages in one project, etc.) +1. You would like to group packages together in ways that make sense for your usage (such as all NPM packages in one project, + all packages being used by a specific department in one project, or all private packages in one project) 1. You want to use one remote for all of your packages when installing them into other projects. 1. You would like to migrate your packages to a single place on GitLab from a third-party package registry and do not want to worry about setting up separate projects for each package. @@ -44,7 +44,7 @@ Let's take a look at how you might create a public place to hold all of your pub ### Create a project First, create a new project on GitLab. It does not have to have any code or content. Make note of the project ID -displayed on the project overview page, as you will need this later. +displayed on the project overview page for use later in this process. ### Create an access token @@ -67,24 +67,24 @@ If you are using NPM, this involves creating an `.npmrc` file and adding the app to your project using your project ID, then adding a section to your `package.json` file with a similar URL. Follow -the instructions in the [GitLab NPM Registry documentation](../npm_registry/index.md#authenticating-to-the-gitlab-npm-registry). Once -you do this, you will be able to push your NPM package to your project using `npm publish`, as described in the +the instructions in the [GitLab NPM Registry documentation](../npm_registry/index.md#authenticating-to-the-gitlab-npm-registry). After +you do this, you can push your NPM package to your project using `npm publish`, as described in the [uploading packages](../npm_registry/index.md#uploading-packages) section of the docs. #### Maven If you are using Maven, this involves updating your `pom.xml` file with distribution sections, including the appropriate URL for your project, as described in the [GitLab Maven Repository documentation](../maven_repository/index.md#project-level-maven-endpoint). -Then, you need to add a `settings.xml` file and [include your access token](../maven_repository/index.md#authenticating-with-a-personal-access-token). +Then, you need to add a `settings.xml` file and [include your access token](../maven_repository/index.md#authenticate-with-a-personal-access-token). Now you can [deploy Maven packages](../maven_repository/index.md#uploading-packages) to your project. #### Conan -For Conan, first you need to add GitLab as a Conan registry remote. Follow the instructions in the [GitLab Conan Repository docs](../conan_repository/index.md#adding-the-gitlab-package-registry-as-a-conan-remote) +For Conan, first you need to add GitLab as a Conan registry remote. Follow the instructions in the [GitLab Conan Repository docs](../conan_repository/index.md#add-the-package-registry-as-a-conan-remote) to do so. Then, create your package using the plus-separated (`+`) project path as your Conan user. For example, if your project is located at `https://gitlab.com/foo/bar/my-proj`, then you can [create your Conan package](../conan_repository/index.md) -using `conan create . foo+bar+my-proj/channel`, where `channel` is your package channel (`stable`, `beta`, etc.). Once your package -is created, you are ready to [upload your package](../conan_repository/index.md#uploading-a-package) depending on your final package recipe. For example: +using `conan create . foo+bar+my-proj/channel`, where `channel` is your package channel (such as `stable` or `beta`). After your package +is created, you are ready to [upload your package](../conan_repository/index.md#publish-a-conan-package) depending on your final package recipe. For example: ```shell CONAN_LOGIN_USERNAME=<gitlab-username> CONAN_PASSWORD=<personal_access_token> conan upload MyPackage/1.0.0@foo+bar+my-proj/channel --all --remote=gitlab diff --git a/doc/user/permissions.md b/doc/user/permissions.md index e2baac1a962..2e9f36360c6 100644 --- a/doc/user/permissions.md +++ b/doc/user/permissions.md @@ -1,5 +1,7 @@ --- -description: 'Understand and explore the user permission levels in GitLab, and what features each of them grants you access to.' +stage: Manage +group: Access +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # Permissions @@ -8,7 +10,7 @@ Users have different abilities depending on the access level they have in a particular group or project. If a user is both in a project's group and the project itself, the highest permission level is used. -On public and internal projects the Guest role is not enforced. All users can: +On public and internal projects, the Guest role is not enforced. All users can: - Create issues. - Leave comments. @@ -42,17 +44,17 @@ or an instance administrator, who receives all permissions. For more information The following table depicts the various user permission levels in a project. -| Action | Guest | Reporter | Developer |Maintainer| Owner* | +| Action | Guest | Reporter | Developer |Maintainer| Owner (*10*) | |---------------------------------------------------|---------|------------|-------------|----------|--------| | Download project | ✓ (*1*) | ✓ | ✓ | ✓ | ✓ | | Leave comments | ✓ | ✓ | ✓ | ✓ | ✓ | -| View allowed and denied licenses **(ULTIMATE)** | ✓ (*1*) | ✓ | ✓ | ✓ | ✓ | +| View allowed and denied licenses **(ULTIMATE)** | ✓ (*1*) | ✓ | ✓ | ✓ | ✓ | | View License Compliance reports **(ULTIMATE)** | ✓ (*1*) | ✓ | ✓ | ✓ | ✓ | | View Security reports **(ULTIMATE)** | ✓ (*3*) | ✓ | ✓ | ✓ | ✓ | | View Dependency list **(ULTIMATE)** | ✓ (*1*) | ✓ | ✓ | ✓ | ✓ | | View License list **(ULTIMATE)** | ✓ (*1*) | ✓ | ✓ | ✓ | ✓ | | View licenses in Dependency list **(ULTIMATE)** | ✓ (*1*) | ✓ | ✓ | ✓ | ✓ | -| View [Design Management](project/issues/design_management.md) pages | ✓ | ✓ | ✓ | ✓ | ✓ | +| View [Design Management](project/issues/design_management.md) pages | ✓ | ✓ | ✓ | ✓ | ✓ | | View project code | ✓ (*1*) | ✓ | ✓ | ✓ | ✓ | | Pull project code | ✓ (*1*) | ✓ | ✓ | ✓ | ✓ | | View GitLab Pages protected by [access control](project/pages/introduction.md#gitlab-pages-access-control) | ✓ | ✓ | ✓ | ✓ | ✓ | @@ -63,30 +65,35 @@ The following table depicts the various user permission levels in a project. | Create new issue | ✓ | ✓ | ✓ | ✓ | ✓ | | See related issues | ✓ | ✓ | ✓ | ✓ | ✓ | | Create confidential issue | ✓ | ✓ | ✓ | ✓ | ✓ | -| View confidential issues | (*2*) | ✓ | ✓ | ✓ | ✓ | | View [Releases](project/releases/index.md) | ✓ (*6*) | ✓ | ✓ | ✓ | ✓ | | View requirements **(ULTIMATE)** | ✓ | ✓ | ✓ | ✓ | ✓ | +| View Insights **(ULTIMATE)** | ✓ | ✓ | ✓ | ✓ | ✓ | +| View Issue analytics **(PREMIUM)** | ✓ | ✓ | ✓ | ✓ | ✓ | +| View Merge Request analytics **(STARTER)** | ✓ | ✓ | ✓ | ✓ | ✓ | +| View Value Stream analytics | ✓ | ✓ | ✓ | ✓ | ✓ | | Manage user-starred metrics dashboards (*7*) | ✓ | ✓ | ✓ | ✓ | ✓ | +| View confidential issues | (*2*) | ✓ | ✓ | ✓ | ✓ | | Assign issues | | ✓ | ✓ | ✓ | ✓ | | Label issues | | ✓ | ✓ | ✓ | ✓ | | Set issue weight | | ✓ | ✓ | ✓ | ✓ | | Lock issue threads | | ✓ | ✓ | ✓ | ✓ | | Manage issue tracker | | ✓ | ✓ | ✓ | ✓ | -| Manage related issues **(STARTER)** | | ✓ | ✓ | ✓ | ✓ | +| Manage related issues | | ✓ | ✓ | ✓ | ✓ | | Manage labels | | ✓ | ✓ | ✓ | ✓ | | Create code snippets | | ✓ | ✓ | ✓ | ✓ | | See a commit status | | ✓ | ✓ | ✓ | ✓ | | See a container registry | | ✓ | ✓ | ✓ | ✓ | | See environments | | ✓ | ✓ | ✓ | ✓ | | See a list of merge requests | | ✓ | ✓ | ✓ | ✓ | -| View project statistics | | | ✓ | ✓ | ✓ | +| View CI/CD analytics | | ✓ | ✓ | ✓ | ✓ | +| View Code Review analytics **(STARTER)** | | ✓ | ✓ | ✓ | ✓ | +| View Repository analytics | | ✓ | ✓ | ✓ | ✓ | | View Error Tracking list | | ✓ | ✓ | ✓ | ✓ | | Create new merge request | | ✓ | ✓ | ✓ | ✓ | | View metrics dashboard annotations | | ✓ | ✓ | ✓ | ✓ | | Create/edit requirements **(ULTIMATE)** | | ✓ | ✓ | ✓ | ✓ | | Pull [packages](packages/index.md) | | ✓ | ✓ | ✓ | ✓ | | Publish [packages](packages/index.md) | | | ✓ | ✓ | ✓ | -| Delete [packages](packages/index.md) | | | | ✓ | ✓ | | Create/edit/delete a Cleanup policy | | | ✓ | ✓ | ✓ | | Upload [Design Management](project/issues/design_management.md) files | | | ✓ | ✓ | ✓ | | Create/edit/delete [Releases](project/releases/index.md)| | | ✓ | ✓ | ✓ | @@ -99,9 +106,12 @@ The following table depicts the various user permission levels in a project. | Lock merge request threads | | | ✓ | ✓ | ✓ | | Approve merge requests (*9*) | | | ✓ | ✓ | ✓ | | Manage/Accept merge requests | | | ✓ | ✓ | ✓ | +| View project statistics | | | ✓ | ✓ | ✓ | | Create new environments | | | ✓ | ✓ | ✓ | | Stop environments | | | ✓ | ✓ | ✓ | | Enable Review Apps | | | ✓ | ✓ | ✓ | +| View Pods logs | | | ✓ | ✓ | ✓ | +| Read Terraform state | | | ✓ | ✓ | ✓ | | Add tags | | | ✓ | ✓ | ✓ | | Cancel and retry jobs | | | ✓ | ✓ | ✓ | | Create or update commit status | | | ✓ (*5*) | ✓ | ✓ | @@ -116,12 +126,14 @@ The following table depicts the various user permission levels in a project. | Create vulnerability from vulnerability finding **(ULTIMATE)** | | | ✓ | ✓ | ✓ | | Resolve vulnerability **(ULTIMATE)** | | | ✓ | ✓ | ✓ | | Dismiss vulnerability **(ULTIMATE)** | | | ✓ | ✓ | ✓ | +| Revert vulnerability to detected state **(ULTIMATE)** | | | ✓ | ✓ | ✓ | | Apply code change suggestions | | | ✓ | ✓ | ✓ | | Create and edit wiki pages | | | ✓ | ✓ | ✓ | | Rewrite/remove Git tags | | | ✓ | ✓ | ✓ | | Manage Feature Flags **(PREMIUM)** | | | ✓ | ✓ | ✓ | | Create/edit/delete metrics dashboard annotations | | | ✓ | ✓ | ✓ | | Run CI/CD pipeline against a protected branch | | | ✓ (*5*) | ✓ | ✓ | +| Delete [packages](packages/index.md) | | | | ✓ | ✓ | | Request a CVE ID **(FREE ONLY)** | | | | ✓ | ✓ | | Use environment terminals | | | | ✓ | ✓ | | Run Web IDE's Interactive Web Terminals **(ULTIMATE ONLY)** | | | | ✓ | ✓ | @@ -132,6 +144,7 @@ The following table depicts the various user permission levels in a project. | Enable/disable tag protections | | | | ✓ | ✓ | | Edit project settings | | | | ✓ | ✓ | | Edit project badges | | | | ✓ | ✓ | +| Export project | | | | ✓ | ✓ | | Share (invite) projects with groups | | | | ✓ (*8*) | ✓ (*8*)| | Add deploy keys to project | | | | ✓ | ✓ | | Configure project hooks | | | | ✓ | ✓ | @@ -143,8 +156,6 @@ The following table depicts the various user permission levels in a project. | Remove GitLab Pages | | | | ✓ | ✓ | | Manage clusters | | | | ✓ | ✓ | | Manage Project Operations | | | | ✓ | ✓ | -| View Pods logs | | | ✓ | ✓ | ✓ | -| Read Terraform state | | | ✓ | ✓ | ✓ | | Manage Terraform state | | | | ✓ | ✓ | | Manage license policy **(ULTIMATE)** | | | | ✓ | ✓ | | Edit comments (posted by any user) | | | | ✓ | ✓ | @@ -165,17 +176,8 @@ The following table depicts the various user permission levels in a project. | Disable notification emails | | | | | ✓ | | Force push to protected branches (*4*) | | | | | | | Remove protected branches (*4*) | | | | | | -| View CI\CD analytics | | ✓ | ✓ | ✓ | ✓ | -| View Code Review analytics **(STARTER)** | | ✓ | ✓ | ✓ | ✓ | -| View Insights **(ULTIMATE)** | ✓ | ✓ | ✓ | ✓ | ✓ | -| View Issue analytics **(PREMIUM)** | ✓ | ✓ | ✓ | ✓ | ✓ | -| View Merge Request analytics **(STARTER)** | ✓ | ✓ | ✓ | ✓ | ✓ | -| View Repository analytics | | ✓ | ✓ | ✓ | ✓ | -| View Value Stream analytics | ✓ | ✓ | ✓ | ✓ | ✓ | -\* Owner permission is only available at the group or personal namespace level (and for instance admins) and is inherited by its projects. - -1. Guest users are able to perform this action on public and internal projects, but not private projects. +1. Guest users are able to perform this action on public and internal projects, but not private projects. This doesn't apply to [external users](#external-users) where explicit access must be given even if the project is internal. 1. Guest users can only view the confidential issues they created themselves. 1. If **Public pipelines** is enabled in **Project Settings > CI/CD**. 1. Not allowed for Guest, Reporter, Developer, Maintainer, or Owner. See [Protected Branches](./project/protected_branches.md). @@ -185,6 +187,7 @@ The following table depicts the various user permission levels in a project. 1. When [Share Group Lock](./group/index.md#share-with-group-lock) is enabled the project can't be shared with other groups. It does not affect group with group sharing. 1. For information on eligible approvers for merge requests, see [Eligible approvers](project/merge_requests/merge_request_approvals.md#eligible-approvers). +1. Owner permission is only available at the group or personal namespace level (and for instance admins) and is inherited by its projects. ## Project features permissions @@ -195,7 +198,7 @@ which visibility level you select on project settings. - Disabled: disabled for everyone - Only team members: only team members can see even if your project is public or internal -- Everyone with access: everyone can see depending on your project visibility level +- Everyone with access: everyone can see depending on your project's visibility level - Everyone: enabled for everyone (only available for GitLab Pages) ### Protected branches @@ -224,7 +227,7 @@ Read through the documentation on [permissions for File Locking](project/file_lo ### Confidential Issues permissions -Confidential issues can be accessed by reporters and higher permission levels, +Confidential issues can be accessed by users with reporter and higher permission levels, as well as by guest users that create a confidential issue. To learn more, read through the documentation on [permissions and access to confidential issues](project/issues/confidential_issues.md#permissions-and-access-to-confidential-issues). @@ -240,6 +243,7 @@ group. | Action | Guest | Reporter | Developer | Maintainer | Owner | |--------------------------------------------------------|-------|----------|-----------|------------|-------| | Browse group | ✓ | ✓ | ✓ | ✓ | ✓ | +| View group wiki pages **(PREMIUM)** | ✓ (6) | ✓ | ✓ | ✓ | ✓ | | View Insights charts **(ULTIMATE)** | ✓ | ✓ | ✓ | ✓ | ✓ | | View group epic **(ULTIMATE)** | ✓ | ✓ | ✓ | ✓ | ✓ | | Create/edit group epic **(ULTIMATE)** | | ✓ | ✓ | ✓ | ✓ | @@ -253,10 +257,12 @@ group. | Create/edit/delete group milestones | | | ✓ | ✓ | ✓ | | Create/edit/delete iterations | | | ✓ | ✓ | ✓ | | Enable/disable a dependency proxy **(PREMIUM)** | | | ✓ | ✓ | ✓ | +| Create and edit group wiki pages **(PREMIUM)** | | | ✓ | ✓ | ✓ | | Use security dashboard **(ULTIMATE)** | | | ✓ | ✓ | ✓ | | Create/edit/delete metrics dashboard annotations | | | ✓ | ✓ | ✓ | | View/manage group-level Kubernetes cluster | | | | ✓ | ✓ | | Create subgroup | | | | ✓ (1) | ✓ | +| Delete group wiki pages **(PREMIUM)** | | | | ✓ | ✓ | | Edit epic comments (posted by any user) **(ULTIMATE)** | | | | ✓ (2) | ✓ (2) | | Edit group settings | | | | | ✓ | | Manage group level CI/CD variables | | | | | ✓ | @@ -270,7 +276,7 @@ group. | Disable notification emails | | | | | ✓ | | View Contribution analytics | ✓ | ✓ | ✓ | ✓ | ✓ | | View Insights **(ULTIMATE)** | ✓ | ✓ | ✓ | ✓ | ✓ | -| View Issue analytics **(PREMIUM)** | ✓ | ✓ | ✓ | ✓ | ✓ | +| View Issue analytics **(PREMIUM)** | ✓ | ✓ | ✓ | ✓ | ✓ | | View Productivity analytics **(PREMIUM)** | | ✓ | ✓ | ✓ | ✓ | | View Value Stream analytics | ✓ | ✓ | ✓ | ✓ | ✓ | | View Billing **(FREE ONLY)** | | | | | ✓ (4) | @@ -283,7 +289,8 @@ group. - The [instance level](admin_area/settings/visibility_and_access_controls.md#default-project-creation-protection). - The [group level](group/index.md#default-project-creation-level). 1. Does not apply to subgroups. -1. Developers can push commits to the default branch of a new project only if the [default branch protection](group/index.md#changing-the-default-branch-protection-of-a-group) is set to "Partially protected" or "Not protected". +1. Developers can push commits to the default branch of a new project only if the [default branch protection](group/index.md#changing-the-default-branch-protection-of-a-group) is set to "Partially protected" or "Not protected". +1. In addition, if your group is public or internal, all users who can see the group can also see group wiki pages. ### Subgroup permissions @@ -311,7 +318,7 @@ External users: Access can be granted by adding the user as member to the project or group. Like usual users, they receive a role in the project or group with all the abilities that are mentioned in the [permissions table above](#project-members-permissions). -For example, if an external user is added as Guest, and your project is +For example, if an external user is added as Guest, and your project is internal or private, they do not have access to the code; you need to grant the external user access at the Reporter level or above if you want them to have access to the code. You should always take into account the @@ -339,7 +346,7 @@ The **Internal users** field allows specifying an email address regex pattern to identify default internal users. New users whose email address matches the regex pattern are set to internal by default rather than an external collaborator. -The regex pattern format is Ruby, but it needs to be convertible to JavaScript, +The regex pattern format is in Ruby, but it needs to be convertible to JavaScript, and the ignore case flag is set (`/regex pattern/i`). Here are some examples: - Use `\.internal@domain\.com$` to mark email addresses ending with @@ -384,6 +391,16 @@ with the permissions described on the documentation on [auditor users permission [Read more about Auditor users.](../administration/auditor_users.md) +## Users with minimal access **(PREMIUM ONLY)** + +>[Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/40942) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.4. + +Administrators can add members with a "minimal access" role to a parent group. Such users don't +automatically have access to projects and subgroups underneath. To support such access, administrators must explicitly add these "minimal access" users to the specific subgroups/projects. + +Users with minimal access can list the group in the UI and through the API. However, they cannot see +details such as projects or subgroups. They do not have access to the group's page or list any of its subgroups or projects. + ## Project features Project features like wiki and issues can be hidden from users depending on @@ -426,7 +443,7 @@ instance and project. In addition, all admins can use the admin interface under 1. Only if the job was: - Triggered by the user - - [Since GitLab 13.0](https://gitlab.com/gitlab-org/gitlab/-/issues/35069), not run for a protected branch + - [In GitLab 13.0](https://gitlab.com/gitlab-org/gitlab/-/issues/35069) and later, not run for a protected branch ### Job permissions @@ -473,7 +490,7 @@ for details about the pipelines security model. ## LDAP users permissions -Since GitLab 8.15, LDAP user permissions can now be manually overridden by an admin user. +In GitLab 8.15 and later, LDAP user permissions can now be manually overridden by an admin user. Read through the documentation on [LDAP users permissions](group/index.md#manage-group-memberships-via-ldap) to learn more. ## Project aliases diff --git a/doc/user/profile/account/create_accounts.md b/doc/user/profile/account/create_accounts.md index 26c2c1bed89..09bfa7afc9e 100644 --- a/doc/user/profile/account/create_accounts.md +++ b/doc/user/profile/account/create_accounts.md @@ -31,9 +31,9 @@ You can also [create users through the API](../../../api/users.md) as an admin.  -## Create users through integrations +## Create users through authentication integrations Users will be: -- Automatically created upon first login with the [LDAP integration](../../../administration/auth/ldap/index.md). -- Created when first logging in via an [OmniAuth provider](../../../integration/omniauth.md) if the `allow_single_sign_on` setting is present. +- Automatically created upon first sign in with the [LDAP integration](../../../administration/auth/ldap/index.md). +- Created when first signing in via an [OmniAuth provider](../../../integration/omniauth.md) if the `allow_single_sign_on` setting is present. diff --git a/doc/user/profile/index.md b/doc/user/profile/index.md index 894494da513..0400d9ca2e5 100644 --- a/doc/user/profile/index.md +++ b/doc/user/profile/index.md @@ -51,7 +51,14 @@ To access your profile settings: From there, you can: -- Update your personal information +- Update your personal information, including: + - Full name + - Primary email, public email, and commit email + - Social media handles + - Website URL + - Location + - Job title + - Bio - Change your [password](#changing-your-password) - Set a [custom status](#current-status) for your profile - Manage your [commit email](#commit-email) for your profile diff --git a/doc/user/profile/personal_access_tokens.md b/doc/user/profile/personal_access_tokens.md index 1b8c16f401c..911be117716 100644 --- a/doc/user/profile/personal_access_tokens.md +++ b/doc/user/profile/personal_access_tokens.md @@ -14,7 +14,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w If you're unable to use [OAuth2](../../api/oauth2.md), you can use a personal access token to authenticate with the [GitLab API](../../api/README.md#personalproject-access-tokens). -You can also use personal access tokens with Git to authenticate over HTTP or SSH. Personal access tokens are required when [Two-Factor Authentication (2FA)](../account/two_factor_authentication.md) is enabled. In both cases, you can authenticate with a token in place of your password. +You can also use personal access tokens with Git to authenticate over HTTP or SSH. Personal access tokens are required when [Two-Factor Authentication (2FA)](account/two_factor_authentication.md) is enabled. In both cases, you can authenticate with a token in place of your password. Personal access tokens expire on the date you define, at midnight UTC. @@ -71,7 +71,7 @@ the following table. You can programmatically create a predetermined personal access token for use in automation or tests. You need sufficient access to run a -[Rails console session](../../administration/troubleshooting/debug.md#starting-a-rails-console-session) +[Rails console session](../../administration/operations/rails_console.md#starting-a-rails-console-session) for your GitLab instance. To create a token belonging to a user with username `automation-bot`, run the @@ -101,7 +101,7 @@ The list of valid scopes and what they do can be found ## Programmatically revoking a personal access token You can programmatically revoke a personal access token. You need -sufficient access to run a [Rails console session](../../administration/troubleshooting/debug.md#starting-a-rails-console-session) +sufficient access to run a [Rails console session](../../administration/operations/rails_console.md#starting-a-rails-console-session) for your GitLab instance. To revoke a known token `token-string-here123`, run the following in the Rails diff --git a/doc/user/profile/preferences.md b/doc/user/profile/preferences.md index f84fc1ae898..61944bb9d0b 100644 --- a/doc/user/profile/preferences.md +++ b/doc/user/profile/preferences.md @@ -84,7 +84,7 @@ The default syntax theme is White, and you can choose among 5 different themes: [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/2389) in 13.0, the theme you choose also applies to the [Web IDE](../project/web_ide/index.md)'s code editor and [Snippets](../snippets.md). The themes are available only in the Web IDE file editor, except for the [dark theme](https://gitlab.com/gitlab-org/gitlab/-/issues/209808) and -the [solarized dark theme](https://gitlab.com/gitlab-org/gitlab/-/issues/219228), +the [Solarized dark theme](https://gitlab.com/gitlab-org/gitlab/-/issues/219228), which apply to the entire Web IDE screen. ## Behavior @@ -131,15 +131,9 @@ You can choose between 2 options: ### Project overview content -The project overview content setting allows you to choose what content you want to +The **Project overview content** setting allows you to choose what content you want to see on a project’s home page. -You can choose between 3 options: - -- Files and Readme (default) -- Readme -- Activity - ### Tab width You can set the displayed width of tab characters across various parts of diff --git a/doc/user/project/autocomplete_characters.md b/doc/user/project/autocomplete_characters.md index 1acdc56de54..ff9cb1c712e 100644 --- a/doc/user/project/autocomplete_characters.md +++ b/doc/user/project/autocomplete_characters.md @@ -35,6 +35,8 @@ Autocomplete characters are useful when combined with [Quick Actions](quick_acti Assume your GitLab instance includes the following users: +<!-- vale gitlab.Spelling = NO --> + | Username | Name | | :-------------- | :--- | | alessandra | Rosy Grant | @@ -43,6 +45,8 @@ Assume your GitLab instance includes the following users: | logan_gutkowski | Lee Wuckert | | shelba | Josefine Haley | +<!-- vale gitlab.Spelling = YES --> + In an Issue comment, entering `@l` results in the following popup list appearing. Note that user `shelba` is not included, because the list includes only the 5 users most relevant to the Issue. diff --git a/doc/user/project/badges.md b/doc/user/project/badges.md index ed6e2460554..fd0287fb5fb 100644 --- a/doc/user/project/badges.md +++ b/doc/user/project/badges.md @@ -10,7 +10,7 @@ type: reference, howto > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/41174) in GitLab 10.7. Badges are a unified way to present condensed pieces of information about your -projects. They consist of a small image and additionally a URL that the image +projects. They consist of a small image and a URL that the image points to. Examples for badges can be the [pipeline status](../../ci/pipelines/settings.md#pipeline-status-badge), [test coverage](../../ci/pipelines/settings.md#test-coverage-report-badge), or ways to contact the project maintainers. diff --git a/doc/user/project/clusters/add_eks_clusters.md b/doc/user/project/clusters/add_eks_clusters.md index b3b1b51a543..65416d73f06 100644 --- a/doc/user/project/clusters/add_eks_clusters.md +++ b/doc/user/project/clusters/add_eks_clusters.md @@ -43,7 +43,7 @@ For example, the following policy document allows assuming a role whose name sta Generate an access key for the IAM user, and configure GitLab with the credentials: -1. Navigate to **Admin Area > Settings > Integrations** and expand the **Amazon EKS** section. +1. Navigate to **Admin Area > Settings > General** and expand the **Amazon EKS** section. 1. Check **Enable Amazon EKS integration**. 1. Enter the account ID and access key credentials into the respective `Account ID`, `Access key ID` and `Secret access key` fields. @@ -61,22 +61,13 @@ To create and add a new Kubernetes cluster to your project, group, or instance: - **Admin Area > Kubernetes**, for an instance-level cluster. 1. Click **Add Kubernetes cluster**. 1. Under the **Create new cluster** tab, click **Amazon EKS**. You will be provided with an - `Account ID` and `External ID` to use in the next step. -1. In the [IAM Management Console](https://console.aws.amazon.com/iam/home), create an EKS management IAM role. - To do so, follow the [Amazon EKS cluster IAM role](https://docs.aws.amazon.com/eks/latest/userguide/service_IAM_role.html) instructions - to create a IAM role suitable for managing the AWS EKS cluster's resources on your behalf. - In addition to the policies that guide suggests, you must also include the `AmazonEKSClusterPolicy` - policy for this role in order for GitLab to manage the EKS cluster correctly. -1. In the [IAM Management Console](https://console.aws.amazon.com/iam/home), create an IAM role: - 1. From the left panel, select **Roles**. - 1. Click **Create role**. - 1. Under `Select type of trusted entity`, select **Another AWS account**. - 1. Enter the Account ID from GitLab into the `Account ID` field. - 1. Check **Require external ID**. - 1. Enter the External ID from GitLab into the `External ID` field. - 1. Click **Next: Permissions**. - 1. Click **Create Policy**, which will open a new window. - 1. Select the **JSON** tab, and paste in the following snippet in place of the existing content: + `Account ID` and `External ID` needed for later steps. +1. In the [IAM Management Console](https://console.aws.amazon.com/iam/home), create an IAM policy: + 1. From the left panel, select **Policies**. + 1. Click **Create Policy**, which opens a new window. + 1. Select the **JSON** tab, and paste the following snippet in place of the + existing content. These permissions give GitLab the ability to create + resources, but not delete them: ```json { @@ -123,15 +114,26 @@ To create and add a new Kubernetes cluster to your project, group, or instance: } ``` - NOTE: **Note:** - These permissions give GitLab the ability to create resources, but not delete them. - This means that if an error is encountered during the creation process, changes will + If an error is encountered during the creation process, changes will not be rolled back and you must remove resources manually. You can do this by deleting the relevant [CloudFormation stack](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/cfn-console-delete-stack.html) 1. Click **Review policy**. 1. Enter a suitable name for this policy, and click **Create Policy**. You can now close this window. - 1. Switch back to the "Create role" window, and select the policy you just created. + +1. In the [IAM Management Console](https://console.aws.amazon.com/iam/home), create an EKS management IAM role. + To do so, follow the [Amazon EKS cluster IAM role](https://docs.aws.amazon.com/eks/latest/userguide/service_IAM_role.html) instructions + to create a IAM role suitable for managing the AWS EKS cluster's resources on your behalf. + In addition to the policies that guide suggests, you must also include the `AmazonEKSClusterPolicy` + policy for this role in order for GitLab to manage the EKS cluster correctly. +1. In the [IAM Management Console](https://console.aws.amazon.com/iam/home), create an IAM role: + 1. From the left panel, select **Roles**. + 1. Click **Create role**. + 1. Under `Select type of trusted entity`, select **Another AWS account**. + 1. Enter the Account ID from GitLab into the `Account ID` field. + 1. Check **Require external ID**. + 1. Enter the External ID from GitLab into the `External ID` field. + 1. Click **Next: Permissions**, and select the policy you just created. 1. Click **Next: Tags**, and optionally enter any tags you wish to associate with this role. 1. Click **Next: Review**. 1. Enter a role name and optional description into the fields provided. diff --git a/doc/user/project/clusters/add_remove_clusters.md b/doc/user/project/clusters/add_remove_clusters.md index 18d9fa67ee1..094f4bcf6ba 100644 --- a/doc/user/project/clusters/add_remove_clusters.md +++ b/doc/user/project/clusters/add_remove_clusters.md @@ -19,9 +19,12 @@ and learn how to spin up a Kubernetes cluster managed by Google Cloud Platform ( in a few clicks. TIP: **Tip:** -Every new Google Cloud Platform (GCP) account receives [$300 in credit upon sign up](https://console.cloud.google.com/freetrial), -and in partnership with Google, GitLab is able to offer an additional $200 for new GCP accounts to get started with GitLab's -Google Kubernetes Engine Integration. All you have to do is [follow this link](https://cloud.google.com/partners/partnercredit/?pcn_code=0014M00001h35gDQAQ#contact-form) and apply for credit. +Every new Google Cloud Platform (GCP) account receives +[$300 in credit upon sign up](https://console.cloud.google.com/freetrial). +In partnership with Google, GitLab is able to offer an additional $200 for new GCP +accounts to get started with GitLab's Google Kubernetes Engine Integration. +[Follow this link](https://cloud.google.com/partners/partnercredit/?pcn_code=0014M00001h35gDQAQ#contact-form) +to apply for credit. ## Before you begin @@ -30,7 +33,7 @@ Before [adding a Kubernetes cluster](#create-new-cluster) using GitLab, you need - GitLab itself. Either: - A [GitLab.com account](https://about.gitlab.com/pricing/#gitlab-com). - A [self-managed installation](https://about.gitlab.com/pricing/#self-managed) with GitLab version - 12.5 or later. This will ensure the GitLab UI can be used for cluster creation. + 12.5 or later. This ensures the GitLab UI can be used for cluster creation. - The following GitLab access: - [Maintainer access to a project](../../permissions.md#project-members-permissions) for a project-level cluster. @@ -41,28 +44,25 @@ Before [adding a Kubernetes cluster](#create-new-cluster) using GitLab, you need ## Access controls -When creating a cluster in GitLab, you will be asked if you would like to create either: +> - Restricted service account for deployment was [introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/51716) in GitLab 11.5. -- A [Role-based access control (RBAC)](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) cluster. -- An [Attribute-based access control (ABAC)](https://kubernetes.io/docs/reference/access-authn-authz/abac/) cluster. +When creating a cluster in GitLab, you are asked if you would like to create either: -NOTE: **Note:** -[RBAC](#rbac-cluster-resources) is recommended and the GitLab default. +- A [Role-based access control (RBAC)](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) + cluster, which is the GitLab default and recommended option. +- An [Attribute-based access control (ABAC)](https://kubernetes.io/docs/reference/access-authn-authz/abac/) cluster. GitLab creates the necessary service accounts and privileges to install and run [GitLab managed applications](index.md#installing-applications). When GitLab creates the cluster, a `gitlab` service account with `cluster-admin` privileges is created in the `default` namespace to manage the newly created cluster. -NOTE: **Note:** -Restricted service account for deployment was [introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/51716) in GitLab 11.5. - The first time you install an application into your cluster, the `tiller` service account is created with `cluster-admin` privileges in the -`gitlab-managed-apps` namespace. This service account will be used by Helm to +`gitlab-managed-apps` namespace. This service account is used by Helm to install and run [GitLab managed applications](index.md#installing-applications). -Helm will also create additional service accounts and other resources for each +Helm also creates additional service accounts and other resources for each installed application. Consult the documentation of the Helm charts for each application for details. @@ -77,7 +77,7 @@ Note the following about access controls: - Environment-specific resources are only created if your cluster is [managed by GitLab](index.md#gitlab-managed-clusters). -- If your cluster was created before GitLab 12.2, it will use a single namespace for all project +- If your cluster was created before GitLab 12.2, it uses a single namespace for all project environments. ### RBAC cluster resources @@ -151,11 +151,12 @@ Amazon Elastic Kubernetes Service (EKS) at the project, group, or instance level ## Add existing cluster -If you have an existing Kubernetes cluster, you can add it to a project, group, or instance. +If you have an existing Kubernetes cluster, you can add it to a project, group, +or instance. -NOTE: **Note:** -Kubernetes integration is not supported for arm64 clusters. See the issue -[Helm Tiller fails to install on arm64 cluster](https://gitlab.com/gitlab-org/gitlab/-/issues/29838) for details. +Kubernetes integration isn't supported for arm64 clusters. See the issue +[Helm Tiller fails to install on arm64 cluster](https://gitlab.com/gitlab-org/gitlab/-/issues/29838) +for details. ### Existing Kubernetes cluster @@ -181,7 +182,7 @@ To add a Kubernetes cluster to your project, group, or instance: kubectl cluster-info | grep 'Kubernetes master' | awk '/http/ {print $NF}' ``` - 1. **CA certificate** (required) - A valid Kubernetes certificate is needed to authenticate to the cluster. We will use the certificate created by default. + 1. **CA certificate** (required) - A valid Kubernetes certificate is needed to authenticate to the cluster. We use the certificate created by default. 1. List the secrets with `kubectl get secrets`, and one should be named similar to `default-token-xxxxx`. Copy that token name for use below. 1. Get the certificate by running this command: @@ -190,9 +191,20 @@ To add a Kubernetes cluster to your project, group, or instance: kubectl get secret <secret name> -o jsonpath="{['data']['ca\.crt']}" | base64 --decode ``` - NOTE: **Note:** - If the command returns the entire certificate chain, you need copy the *root ca* - certificate at the bottom of the chain. + If the command returns the entire certificate chain, you must copy the Root CA + certificate and any intermediate certificates at the bottom of the chain. + A chain file has following structure: + + ```plaintext + -----BEGIN MY CERTIFICATE----- + -----END MY CERTIFICATE----- + -----BEGIN INTERMEDIATE CERTIFICATE----- + -----END INTERMEDIATE CERTIFICATE----- + -----BEGIN INTERMEDIATE CERTIFICATE----- + -----END INTERMEDIATE CERTIFICATE----- + -----BEGIN ROOT CERTIFICATE----- + -----END ROOT CERTIFICATE----- + ``` 1. **Token** - GitLab authenticates against Kubernetes using service tokens, which are @@ -229,10 +241,10 @@ To add a Kubernetes cluster to your project, group, or instance: kubectl apply -f gitlab-admin-service-account.yaml ``` - You will need the `container.clusterRoleBindings.create` permission + You need the `container.clusterRoleBindings.create` permission to create cluster-level roles. If you do not have this permission, you can alternatively enable Basic Authentication and then run the - `kubectl apply` command as an admin: + `kubectl apply` command as an administrator: ```shell kubectl apply -f gitlab-admin-service-account.yaml --username=admin --password=<password> @@ -257,7 +269,7 @@ To add a Kubernetes cluster to your project, group, or instance: Copy the `<authentication_token>` value from the output: - ```yaml + ```plaintext Name: gitlab-token-b5zv4 Namespace: kube-system Labels: <none> @@ -274,7 +286,7 @@ To add a Kubernetes cluster to your project, group, or instance: ``` NOTE: **Note:** - For GKE clusters, you will need the + For GKE clusters, you need the `container.clusterRoleBindings.create` permission to create a cluster role binding. You can follow the [Google Cloud documentation](https://cloud.google.com/iam/docs/granting-changing-revoking-access) @@ -283,7 +295,7 @@ To add a Kubernetes cluster to your project, group, or instance: 1. **GitLab-managed cluster** - Leave this checked if you want GitLab to manage namespaces and service accounts for this cluster. See the [Managed clusters section](index.md#gitlab-managed-clusters) for more information. 1. **Project namespace** (optional) - You don't have to fill it in; by leaving - it blank, GitLab will create one for you. Also: + it blank, GitLab creates one for you. Also: - Each project should have a unique namespace. - The project namespace is not necessarily the namespace of the secret, if you're using a secret with broader permissions, like the secret from `default`. @@ -294,21 +306,21 @@ To add a Kubernetes cluster to your project, group, or instance: 1. Finally, click the **Create Kubernetes cluster** button. -After a couple of minutes, your cluster will be ready to go. You can now proceed +After a couple of minutes, your cluster is ready. You can now proceed to install some [pre-defined applications](index.md#installing-applications). #### Disable Role-Based Access Control (RBAC) (optional) When connecting a cluster via GitLab integration, you may specify whether the -cluster is RBAC-enabled or not. This will affect how GitLab interacts with the +cluster is RBAC-enabled or not. This affects how GitLab interacts with the cluster for certain operations. If you did *not* check the **RBAC-enabled cluster** -checkbox at creation time, GitLab will assume RBAC is disabled for your cluster +checkbox at creation time, GitLab assumes RBAC is disabled for your cluster when interacting with it. If so, you must disable RBAC on your cluster for the integration to work properly. - + -NOTE: **Note:** +CAUTION: **Caution:** Disabling RBAC means that any application running in the cluster, or user who can authenticate to the cluster, has full API access. This is a [security concern](index.md#security-implications), and may not be desirable. @@ -356,3 +368,12 @@ When removing the cluster integration, note: To learn more on automatically deploying your applications, read about [Auto DevOps](../../../topics/autodevops/index.md). + +## Troubleshooting + +### There was a problem authenticating with your cluster. Please ensure your CA Certificate and Token are valid + +If you encounter this error while adding a Kubernetes cluster, ensure you're +properly pasting the service token. Some shells may add a line break to the +service token, making it invalid. Ensure that there are no line breaks by +pasting your token into an editor and removing any additional spaces. diff --git a/doc/user/project/clusters/index.md b/doc/user/project/clusters/index.md index 8d188f00ceb..459ba144186 100644 --- a/doc/user/project/clusters/index.md +++ b/doc/user/project/clusters/index.md @@ -1,6 +1,6 @@ --- stage: Monitor -group: APM +group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- @@ -12,8 +12,6 @@ info: To determine the technical writer assigned to the Stage/Group associated w > - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/39840) in > GitLab 11.11 for [instances](../../instance/clusters/index.md). -## Overview - Using the GitLab project Kubernetes integration, you can: - Use [Review Apps](../../../ci/review_apps/index.md). @@ -31,6 +29,11 @@ Besides integration at the project level, Kubernetes clusters can also be integrated at the [group level](../../group/clusters/index.md) or [GitLab instance level](../../instance/clusters/index.md). +To view your project level Kubernetes clusters, navigate to **Operations > Kubernetes** +from your project. On this page, you can [add a new cluster](#adding-and-removing-clusters) +and view information about your existing clusters, such as nodes count and rough estimates +of memory and CPU usage. + ## Setting up ### Supported cluster versions @@ -49,10 +52,9 @@ Currently, GitLab supports the following Kubernetes versions: - 1.17 - 1.16 - 1.15 -- 1.14 +- 1.14 (deprecated, support ends on December 22, 2020) - 1.13 (deprecated, support ends on November 22, 2020) -NOTE: **Note:** Some GitLab features may support versions outside the range provided here. ### Adding and removing clusters @@ -192,7 +194,6 @@ To clear the cache: > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/24580) in GitLab 11.8. -NOTE: **Note:** You do not need to specify a base domain on cluster settings when using GitLab Serverless. The domain in that case will be specified as part of the Knative installation. See [Installing Applications](#installing-applications). @@ -220,13 +221,11 @@ Auto DevOps automatically detects, builds, tests, deploys, and monitors your applications. To make full use of Auto DevOps (Auto Deploy, Auto Review Apps, and -Auto Monitoring) you will need the Kubernetes project integration enabled. +Auto Monitoring) you will need the Kubernetes project integration enabled, but +Kubernetes clusters can be used without Auto DevOps. [Read more about Auto DevOps](../../../topics/autodevops/index.md) -NOTE: **Note:** -Kubernetes clusters can be used without Auto DevOps. - ## Deploying to a Kubernetes cluster A Kubernetes cluster can be the destination for a deployment job. If @@ -249,36 +248,51 @@ GitLab CI/CD build environment. | Variable | Description | | -------- | ----------- | | `KUBE_URL` | Equal to the API URL. | -| `KUBE_TOKEN` | The Kubernetes token of the [environment service account](add_remove_clusters.md#access-controls). | -| `KUBE_NAMESPACE` | The namespace associated with the project's deployment service account. In the format `<project_name>-<project_id>-<environment>`. For GitLab-managed clusters, a matching namespace is automatically created by GitLab in the cluster. | +| `KUBE_TOKEN` | The Kubernetes token of the [environment service account](add_remove_clusters.md#access-controls). Prior to GitLab 11.5, `KUBE_TOKEN` was the Kubernetes token of the main service account of the cluster integration. | +| `KUBE_NAMESPACE` | The namespace associated with the project's deployment service account. In the format `<project_name>-<project_id>-<environment>`. For GitLab-managed clusters, a matching namespace is automatically created by GitLab in the cluster. If your cluster was created before GitLab 12.2, the default `KUBE_NAMESPACE` is set to `<project_name>-<project_id>`. | | `KUBE_CA_PEM_FILE` | Path to a file containing PEM data. Only present if a custom CA bundle was specified. | | `KUBE_CA_PEM` | (**deprecated**) Raw PEM data. Only if a custom CA bundle was specified. | -| `KUBECONFIG` | Path to a file containing `kubeconfig` for this deployment. CA bundle would be embedded if specified. This config also embeds the same token defined in `KUBE_TOKEN` so you likely will only need this variable. This variable name is also automatically picked up by `kubectl` so you won't actually need to reference it explicitly if using `kubectl`. | +| `KUBECONFIG` | Path to a file containing `kubeconfig` for this deployment. CA bundle would be embedded if specified. This configuration also embeds the same token defined in `KUBE_TOKEN` so you likely will only need this variable. This variable name is also automatically picked up by `kubectl` so you won't actually need to reference it explicitly if using `kubectl`. | | `KUBE_INGRESS_BASE_DOMAIN` | From GitLab 11.8, this variable can be used to set a domain per cluster. See [cluster domains](#base-domain) for more information. | -NOTE: **Note:** -Prior to GitLab 11.5, `KUBE_TOKEN` was the Kubernetes token of the main -service account of the cluster integration. - -NOTE: **Note:** -If your cluster was created before GitLab 12.2, default `KUBE_NAMESPACE` will be set to `<project_name>-<project_id>`. - ### Custom namespace -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/27630) in GitLab 12.6. - -The Kubernetes integration defaults to project-environment-specific namespaces -of the form `<project_name>-<project_id>-<environment>` (see [Deployment -variables](#deployment-variables)). - -For **non**-GitLab-managed clusters, the namespace can be customized using -[`environment:kubernetes:namespace`](../../../ci/environments/index.md#configuring-kubernetes-deployments) -in `.gitlab-ci.yml`. - -NOTE: **Note:** -When using a [GitLab-managed cluster](#gitlab-managed-clusters), the -namespaces are created automatically prior to deployment and [can not be -customized](https://gitlab.com/gitlab-org/gitlab/-/issues/38054). +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/27630) in GitLab 12.6. +> - An option to use project-wide namespaces [was added](https://gitlab.com/gitlab-org/gitlab/-/issues/38054) in GitLab 13.5. + +The Kubernetes integration provides a `KUBECONFIG` with an auto-generated namespace +to deployment jobs. It defaults to using project-environment specific namespaces +of the form `<prefix>-<environment>`, where `<prefix>` is of the form +`<project_name>-<project_id>`. To learn more, read [Deployment variables](#deployment-variables). + +You can customize the deployment namespace in a few ways: + +- You can choose between a **namespace per [environment](../../../ci/environments/index.md)** + or a **namespace per project**. A namespace per environment is the default and recommended + setting, as it prevents the mixing of resources between production and non-production environments. +- When using a project-level cluster, you can additionally customize the namespace prefix. + When using namespace-per-environment, the deployment namespace is `<prefix>-<environment>`, + but otherwise just `<prefix>`. +- For **non-managed** clusters, the auto-generated namespace is set in the `KUBECONFIG`, + but the user is responsible for ensuring its existence. You can fully customize + this value using + [`environment:kubernetes:namespace`](../../../ci/environments/index.md#configuring-kubernetes-deployments) + in `.gitlab-ci.yml`. + +When you customize the namespace, existing environments remain linked to their current +namespaces until you [clear the cluster cache](#clearing-the-cluster-cache). + +CAUTION: **Warning:** +By default, anyone who can create a deployment job can access any CI variable within +an environment's deployment job. This includes `KUBECONFIG`, which gives access to +any secret available to the associated service account in your cluster. +To keep your production credentials safe, consider using +[Protected Environments](../../../ci/environments/protected_environments.md), +combined with either + +- a GitLab-managed cluster and namespace per environment, +- *or*, an environment-scoped cluster per protected environment. The same cluster + can be added multiple times with multiple restricted service accounts. ### Integrations diff --git a/doc/user/project/clusters/kubernetes_pod_logs.md b/doc/user/project/clusters/kubernetes_pod_logs.md index afb6d016f45..2e224208eb8 100644 --- a/doc/user/project/clusters/kubernetes_pod_logs.md +++ b/doc/user/project/clusters/kubernetes_pod_logs.md @@ -1,6 +1,6 @@ --- stage: Monitor -group: APM +group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- @@ -28,7 +28,6 @@ above the log file data, depending on your configuration: <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> To learn more about the Log Explorer, see [APM - Log Explorer](https://www.youtube.com/watch?v=hWclZHA7Dgw). -NOTE: **Note:** [Learn more about Kubernetes + GitLab](https://about.gitlab.com/solutions/kubernetes/). Everything you need to build, test, deploy, and run your application at scale. diff --git a/doc/user/project/clusters/runbooks/index.md b/doc/user/project/clusters/runbooks/index.md index 360b02efb69..c1e4e821efd 100644 --- a/doc/user/project/clusters/runbooks/index.md +++ b/doc/user/project/clusters/runbooks/index.md @@ -115,9 +115,7 @@ the components outlined above and the pre-loaded demo runbook. VARIABLE_VALUE = project.variables.get('PRIVATE_TOKEN').value ``` -1. To configure the operation of a runbook, create and configure variables: - - NOTE: **Note:** +1. To configure the operation of a runbook, create and configure variables. For this example, we are using the **Run SQL queries in Notebook** section in the sample runbook to query a PostgreSQL database. The first four lines of the following code block define the variables that are required for this query to function: diff --git a/doc/user/project/clusters/securing.md b/doc/user/project/clusters/securing.md index a15660051f7..bed01ff4d58 100644 --- a/doc/user/project/clusters/securing.md +++ b/doc/user/project/clusters/securing.md @@ -85,7 +85,7 @@ Host Security (Falco) are deployed with this model. To deploy GitLab Managed Apps to your cluster, you must first [add your cluster](add_remove_clusters.md) -to GitLab. Then [install](../../clusters/applications.md#installing-applications) +to GitLab. Then [install](../../clusters/applications.md#install-with-one-click) the Web Application Firewall from the project or group Kubernetes page. Note that your project doesn't have to be hosted or deployed through GitLab. You can manage a diff --git a/doc/user/project/clusters/serverless/aws.md b/doc/user/project/clusters/serverless/aws.md index 543ffdbce8f..db91f78fc20 100644 --- a/doc/user/project/clusters/serverless/aws.md +++ b/doc/user/project/clusters/serverless/aws.md @@ -136,8 +136,8 @@ This example code does the following: In order to interact with your AWS account, the GitLab CI/CD pipelines require both `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` to be defined in your GitLab settings under **Settings > CI/CD > Variables**. For more information please see [Create a custom variable in the UI](../../../../ci/variables/README.md#create-a-custom-variable-in-the-ui). -NOTE: **Note:** - The AWS credentials you provide must include IAM policies that provision correct access control to AWS Lambda, API Gateway, CloudFormation, and IAM resources. + The AWS credentials you provide must include IAM policies that provision correct + access control to AWS Lambda, API Gateway, CloudFormation, and IAM resources. #### Deploying your function @@ -154,9 +154,7 @@ endpoints: #### Manually testing your function Running the following `curl` command should trigger your function. - -NOTE: **Note:** -Your URL should be the one retrieved from the GitLab deploy stage log. +Your URL should be the one retrieved from the GitLab deploy stage log: ```shell curl https://u768nzby1j.execute-api.us-east-1.amazonaws.com/production/hello @@ -222,7 +220,8 @@ the environment of the deployed function: ```yaml provider: - ... + # Other configuration omitted + # ... environment: A_VARIABLE: ${env:A_VARIABLE} ``` @@ -245,10 +244,10 @@ functions: hello: handler: src/handler.hello events: - - http: # Rewrite this part to enable CORS + - http: # Rewrite this part to enable CORS path: hello method: get - cors: true # <-- CORS here + cors: true # <-- CORS here ``` You also need to return CORS specific headers in your function response: @@ -378,7 +377,6 @@ To set these: `AWS_SECRET_ACCESS_KEY`. 1. Mask the credentials so they do not show in logs using the **Masked** toggle. -NOTE: **Note:** The AWS credentials you provide must include IAM policies that provision correct access control to AWS Lambda, API Gateway, CloudFormation, and IAM resources. diff --git a/doc/user/project/clusters/serverless/index.md b/doc/user/project/clusters/serverless/index.md index 1157c2c5632..603c4bd73b1 100644 --- a/doc/user/project/clusters/serverless/index.md +++ b/doc/user/project/clusters/serverless/index.md @@ -75,8 +75,8 @@ To run Knative on GitLab, you will need: ## Installing Knative via GitLab's Kubernetes integration -NOTE: **Note:** -The minimum recommended cluster size to run Knative is 3-nodes, 6 vCPUs, and 22.50 GB memory. **RBAC must be enabled.** +The minimum recommended cluster size to run Knative is 3-nodes, 6 vCPUs, and 22.50 GB +memory. **RBAC must be enabled.** 1. [Add a Kubernetes cluster](../add_remove_clusters.md). 1. Select the **Applications** tab and scroll down to the Knative app section. Enter the domain to be used with @@ -99,22 +99,19 @@ The minimum recommended cluster size to run Knative is 3-nodes, 6 vCPUs, and 22.  -NOTE: **Note:** You can deploy either [functions](#deploying-functions) or [serverless applications](#deploying-serverless-applications) -on a given project but not both. The current implementation makes use of a `serverless.yml` file to signal a FaaS project. +on a given project, but not both. The current implementation makes use of a +`serverless.yml` file to signal a FaaS project. ## Using an existing installation of Knative > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/58941) in GitLab 12.0. -NOTE: **Note:** -The "invocations" monitoring feature of GitLab serverless will not work when +The _invocations_ monitoring feature of GitLab serverless won't work when adding an existing installation of Knative. -It is also possible to use GitLab Serverless with an existing Kubernetes -cluster which already has Knative installed. - -You must do the following: +It's also possible to use GitLab Serverless with an existing Kubernetes cluster +which already has Knative installed. You must do the following: 1. Follow the steps to [add an existing Kubernetes @@ -453,16 +450,16 @@ To run a function locally: > Introduced in GitLab 11.5. +12345678901234567890123456789012345678901234567890123456789012345678901234567890 Serverless applications are an alternative to [serverless functions](#deploying-functions). -They are useful in scenarios where an existing runtime does not meet the needs of an application, -such as one written in a language that has no runtime available. Note though that serverless -applications should be stateless! - -NOTE: **Note:** -You can reference and import the sample [Knative Ruby App](https://gitlab.com/knative-examples/knative-ruby-app) to get started. +They're useful in scenarios where an existing runtime does not meet the needs of +an application, such as one written in a language that has no runtime available. +Note though that serverless applications should be stateless. -Add the following `.gitlab-ci.yml` to the root of your repository -(you may skip this step if you've previously cloned the sample [Knative Ruby App](https://gitlab.com/knative-examples/knative-ruby-app) mentioned above): +You can reference and import the sample [Knative Ruby App](https://gitlab.com/knative-examples/knative-ruby-app) +to get started. Add the following `.gitlab-ci.yml` to the root of your repository +(you may skip this step if you've previously cloned the previously mentioned, +sample [Knative Ruby App](https://gitlab.com/knative-examples/knative-ruby-app)): ```yaml include: @@ -561,14 +558,18 @@ Or: ## Enabling TLS for Knative services -By default, a GitLab serverless deployment will be served over `http`. In order to serve over `https` you -must manually obtain and install TLS certificates. +By default, a GitLab serverless deployment will be served over `http`. To serve +over `https`, you must manually obtain and install TLS certificates. -The simplest way to accomplish this is to -use [Certbot to manually obtain Let's Encrypt certificates](https://knative.dev/docs/serving/using-a-tls-cert/#using-certbot-to-manually-obtain-let-s-encrypt-certificates). Certbot is a free, open source software tool for automatically using Let’s Encrypt certificates on manually-administrated websites to enable HTTPS. +12345678901234567890123456789012345678901234567890123456789012345678901234567890 +The simplest way to accomplish this is to use Certbot to +[manually obtain Let's Encrypt certificates](https://knative.dev/docs/serving/using-a-tls-cert/#using-certbot-to-manually-obtain-let-s-encrypt-certificates). +Certbot is a free, open source software tool for automatically using Let’s Encrypt +certificates on manually-administered websites to enable HTTPS. -NOTE: **Note:** -The instructions below relate to installing and running Certbot on a Linux server that has Python 3 installed and may not work on other operating systems or with other versions of Python. +The following instructions relate to installing and running Certbot on a Linux +server that has Python 3 installed, and may not work on other operating systems +or with other versions of Python. 1. Install Certbot by running the [`certbot-auto` wrapper script](https://certbot.eff.org/docs/install.html#certbot-auto). @@ -788,7 +789,7 @@ The instructions below relate to installing and running Certbot on a Linux serve kubectl edit gateway knative-ingress-gateway --namespace knative-serving ``` - Update the gateway to include the following tls: section and configuration: + Update the gateway to include the following `tls:` section and configuration: ```shell tls: diff --git a/doc/user/project/code_intelligence.md b/doc/user/project/code_intelligence.md index f56673e69b7..d0c5a24826a 100644 --- a/doc/user/project/code_intelligence.md +++ b/doc/user/project/code_intelligence.md @@ -58,9 +58,9 @@ relevant language. | Language | Implementation | |---|---| -| Go | [sourcegraph/lsif-go](https://github.com/sourcegraph/lsif-go) | -| JavaScript | [sourcegraph/lsif-node](https://github.com/sourcegraph/lsif-node) | -| TypeScript | [sourcegraph/lsif-node](https://github.com/sourcegraph/lsif-node) | +| Go | [`sourcegraph/lsif-go`](https://github.com/sourcegraph/lsif-go) | +| JavaScript | [`sourcegraph/lsif-node`](https://github.com/sourcegraph/lsif-node) | +| TypeScript | [`sourcegraph/lsif-node`](https://github.com/sourcegraph/lsif-node) | View a complete list of [available LSIF indexers](https://lsif.dev/#implementations-server) on their website and refer to their documentation to see how to generate an LSIF file for your specific language. diff --git a/doc/user/project/code_owners.md b/doc/user/project/code_owners.md index 730a9ada428..4ae3d5ec032 100644 --- a/doc/user/project/code_owners.md +++ b/doc/user/project/code_owners.md @@ -75,7 +75,6 @@ be used for merge request approvals: - As [merge request eligible approvers](merge_requests/merge_request_approvals.md#code-owners-as-eligible-approvers). - As required approvers for [protected branches](protected_branches.md#protected-branches-approval-by-code-owners). **(PREMIUM)** -NOTE: **Note:** Developer or higher [permissions](../permissions.md) are required in order to approve a merge request. @@ -93,12 +92,14 @@ to specify the actual owners and granular permissions. Using Code Owners in conjunction with [Protected Branches](protected_branches.md#protected-branches-approval-by-code-owners) will prevent any user who is not specified in the `CODEOWNERS` file from pushing -changes for the specified files/paths, even if their role is included in the +changes for the specified files/paths, except those included in the **Allowed to push** column. This allows for a more inclusive push strategy, as administrators don't have to restrict developers from pushing directly to the protected branch, but can restrict pushing to certain files where a review by Code Owners is required. +[Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/35097) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.5, users and groups who are allowed to push to protected branches do not require a merge request to merge their feature branches. Thus, they can skip merge request approval rules, Code Owners included. + ## The syntax of Code Owners files Files can be specified using the same kind of patterns you would use @@ -158,7 +159,7 @@ file.md @group-x @group-x/subgroup-y ### Code Owners Sections **(PREMIUM)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12137) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.2 behind a feature flag, enabled by default. -> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/42389) in GitLab 13.4. +> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/42389) in GitLab 13.4. Code Owner rules can be grouped into named sections. This allows for better organization of broader categories of Code Owner rules to be applied. diff --git a/doc/user/project/deploy_boards.md b/doc/user/project/deploy_boards.md index 8146f39ef87..3c6494d5f1a 100644 --- a/doc/user/project/deploy_boards.md +++ b/doc/user/project/deploy_boards.md @@ -85,7 +85,7 @@ To display the Deploy Boards for a specific [environment](../../ci/environments/ [`kubernetes`](https://docs.gitlab.com/runner/executors/kubernetes.html) executor. 1. Configure the [Kubernetes integration](clusters/index.md) in your project for the cluster. The Kubernetes namespace is of particular note as you will need it - for your deployment scripts (exposed by the `KUBE_NAMESPACE` env variable). + for your deployment scripts (exposed by the `KUBE_NAMESPACE` environment variable). 1. Ensure Kubernetes annotations of `app.gitlab.com/env: $CI_ENVIRONMENT_SLUG` and `app.gitlab.com/app: $CI_PROJECT_PATH_SLUG` are applied to the deployments, replica sets, and pods, where `$CI_ENVIRONMENT_SLUG` and @@ -106,7 +106,7 @@ To display the Deploy Boards for a specific [environment](../../ci/environments/ be done automatically and no action is necessary. If you are using GCP to manage clusters, you can see the deployment details in GCP itself by going to **Workloads > deployment name > Details**: - +  Once all of the above are set up and the pipeline has run at least once, diff --git a/doc/user/project/deploy_keys/index.md b/doc/user/project/deploy_keys/index.md index 81c9008c5b3..4f344554016 100644 --- a/doc/user/project/deploy_keys/index.md +++ b/doc/user/project/deploy_keys/index.md @@ -12,7 +12,7 @@ more repositories, by importing an SSH public key to your GitLab instance. This is useful for cloning repositories to your Continuous Integration (CI) server. By using deploy keys, you don't have to set up a -dummy user account. +fake user account. There are two types of deploy keys: diff --git a/doc/user/project/description_templates.md b/doc/user/project/description_templates.md index aa5987bf5f9..e0c4097d1c5 100644 --- a/doc/user/project/description_templates.md +++ b/doc/user/project/description_templates.md @@ -49,8 +49,8 @@ To create a Markdown file: 1. Click the `+` button next to `master` and click **New file**. 1. Add the name of your issue template to the **File name** text field next to `master`. - Make sure words are separated with underscores and that your file has the `.md` extension, for - example `feature_request.md`. + Make sure that your file has the `.md` extension, for + example `feature_request.md` or `Feature Request.md`. 1. Commit and push to your default branch. If you don't have a `.gitlab/issue_templates` directory in your repository, you'll need to create it. @@ -79,6 +79,9 @@ This will enable the `Bug` dropdown option when creating or editing issues. When to the issue description field. The 'Reset template' button will discard any changes you made after picking the template and return it to its initial status. +TIP: **Tip:** +You can create short-cut links to create an issue using a designated template. For example: `https://gitlab.com/gitlab-org/gitlab/-/issues/new?issuable_template=Feature%20proposal`. +  ## Setting a default template for merge requests and issues **(STARTER)** diff --git a/doc/user/project/file_lock.md b/doc/user/project/file_lock.md index 6fd33901621..46c2e211d57 100644 --- a/doc/user/project/file_lock.md +++ b/doc/user/project/file_lock.md @@ -69,7 +69,7 @@ brew install git-lfs ``` Once installed, **open your local repository in a terminal window** and -install Git LFS in your repo. If you're sure that LFS is already installed, +install Git LFS in your repository. If you're sure that LFS is already installed, you can skip this step. If you're unsure, re-installing it won't do any harm: ```shell @@ -159,7 +159,7 @@ command line interface, file locks can be created for any file. ### View exclusively-locked files To list all the files locked with LFS locally, open a terminal window in your -repo and run: +repository and run: ```shell git lfs locks @@ -189,7 +189,7 @@ Suggested workflow for shared projects: 1. Lock the file. 1. Edit the file. 1. Commit your changes. -1. Push to the repo. +1. Push to the repository. 1. Get your changes reviewed, approved, and merged. 1. Unlock the file. diff --git a/doc/user/project/img/issue_board_default_lists_v13_4.png b/doc/user/project/img/issue_board_default_lists_v13_4.png Binary files differnew file mode 100644 index 00000000000..23cdc9b4e22 --- /dev/null +++ b/doc/user/project/img/issue_board_default_lists_v13_4.png diff --git a/doc/user/project/img/issue_board_welcome_message.png b/doc/user/project/img/issue_board_welcome_message.png Binary files differdeleted file mode 100644 index 357dff42488..00000000000 --- a/doc/user/project/img/issue_board_welcome_message.png +++ /dev/null diff --git a/doc/user/project/img/labels_key_value_v12_1.png b/doc/user/project/img/labels_key_value_v12_1.png Binary files differdeleted file mode 100644 index ccda944a647..00000000000 --- a/doc/user/project/img/labels_key_value_v12_1.png +++ /dev/null diff --git a/doc/user/project/img/labels_key_value_v13_5.png b/doc/user/project/img/labels_key_value_v13_5.png Binary files differnew file mode 100644 index 00000000000..4264eb3211e --- /dev/null +++ b/doc/user/project/img/labels_key_value_v13_5.png diff --git a/doc/user/project/img/labels_prioritized_v12_1.png b/doc/user/project/img/labels_prioritized_v12_1.png Binary files differdeleted file mode 100644 index 512c5d59a5a..00000000000 --- a/doc/user/project/img/labels_prioritized_v12_1.png +++ /dev/null diff --git a/doc/user/project/img/labels_prioritized_v13_5.png b/doc/user/project/img/labels_prioritized_v13_5.png Binary files differnew file mode 100644 index 00000000000..04ffd67a59f --- /dev/null +++ b/doc/user/project/img/labels_prioritized_v13_5.png diff --git a/doc/user/project/img/labels_subscriptions_v12_1.png b/doc/user/project/img/labels_subscriptions_v12_1.png Binary files differdeleted file mode 100644 index fa83b7db414..00000000000 --- a/doc/user/project/img/labels_subscriptions_v12_1.png +++ /dev/null diff --git a/doc/user/project/img/labels_subscriptions_v13_5.png b/doc/user/project/img/labels_subscriptions_v13_5.png Binary files differnew file mode 100644 index 00000000000..a2a4e9e50cc --- /dev/null +++ b/doc/user/project/img/labels_subscriptions_v13_5.png diff --git a/doc/user/project/import/bitbucket.md b/doc/user/project/import/bitbucket.md index 89130d5822f..56266718d12 100644 --- a/doc/user/project/import/bitbucket.md +++ b/doc/user/project/import/bitbucket.md @@ -76,6 +76,3 @@ If you've accidentally started the import process with the wrong account, follow 1. Revoke GitLab access to your Bitbucket account, essentially reversing the process in the following procedure: [Import your Bitbucket repositories](#import-your-bitbucket-repositories). 1. Sign out of the Bitbucket account. Follow the procedure linked from the previous step. - -NOTE: **Note:** -To import a repository including LFS objects from a Bitbucket server repository, use the [Repo by URL](../import/repo_by_url.md) importer. diff --git a/doc/user/project/import/bitbucket_server.md b/doc/user/project/import/bitbucket_server.md index d0499730bfe..ac5be2b46a4 100644 --- a/doc/user/project/import/bitbucket_server.md +++ b/doc/user/project/import/bitbucket_server.md @@ -37,12 +37,7 @@ Import your projects from Bitbucket Server to GitLab with minimal effort. empty changes. 1. Attachments in Markdown are currently not imported. 1. Task lists are not imported. -1. Emoji reactions are not imported. -1. [LFS objects](../../../topics/git/lfs/index.md) are not imported. - - NOTE: **Note:** - To import a repository including LFS objects from a Bitbucket server repository, use the [Repo by URL](../import/repo_by_url.md) importer. - +1. Emoji reactions are not imported 1. Project filtering does not support fuzzy search (only `starts with` or `full match strings` are currently supported) @@ -69,20 +64,43 @@ namespace that started the import process. #### User assignment by username -Alternatively, user assignment by username is available behind a `bitbucket_server_user_mapping_by_username` feature flag. -The importer will try to find a user in the GitLab user database using author's `username` or `slug` or `displayName`. -Falls back to author's `email` if user is not found by username. -Similarly to user assignment by email, if no such user is available, the project creator is set as the author. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/218609) in GitLab 13.4. +> - It's [deployed behind a feature flag](../../feature_flags.md), disabled by default. +> - It's disabled on GitLab.com. +> - It's not recommended for production use. +> - To use it in GitLab self-managed instances, ask a GitLab administrator to enable it. + +CAUTION: **Warning:** +This feature might not be available to you. Check the **version history** note above for details. + +If you've enabled this feature, the importer tries to find a user in the GitLab user database with +the author's: + +- `username` +- `slug` +- `displayName` + +If the user is not found by any of these properties, the search falls back to the author's +`email` address. -To enable or disable user assignment by username: +Alternatively, if there is also no email address, the project creator is set as the author. -Start a [Rails console](../../../administration/troubleshooting/debug.md#starting-a-rails-console-session). +##### Enable or disable User assignment by username + +User assignment by username is under development and not ready for production use. It is +deployed behind a feature flag that is **disabled by default**. +[GitLab administrators with access to the GitLab Rails console](../../../administration/feature_flags.md) +can enable it. + +To enable it: ```ruby -# Enable Feature.enable(:bitbucket_server_user_mapping_by_username) +``` -# Disable +To disable it: + +```ruby Feature.disable(:bitbucket_server_user_mapping_by_username) ``` diff --git a/doc/user/project/import/gemnasium.md b/doc/user/project/import/gemnasium.md index f21ec26bdef..2d0caa7d46e 100644 --- a/doc/user/project/import/gemnasium.md +++ b/doc/user/project/import/gemnasium.md @@ -96,7 +96,7 @@ back to both GitLab and GitHub when completed. The mirroring is pull-only by default, so you may create or update the file on GitHub: -  +  1. Once your file has been committed, a new pipeline will be automatically triggered if your file is valid: diff --git a/doc/user/project/import/github.md b/doc/user/project/import/github.md index 4cd0c9e02c7..6c0105aaded 100644 --- a/doc/user/project/import/github.md +++ b/doc/user/project/import/github.md @@ -35,25 +35,20 @@ The namespace is a user or group in GitLab, such as `gitlab.com/janedoe` or `git This process does not migrate or import any types of groups or organizations from GitHub to GitLab. -### If you're using GitLab.com +### Use cases -If you're using GitLab.com, you can alternatively import -GitHub repositories using a [personal access token](#using-a-github-token), -but we don't recommend this method because it can't associate all user activity -(such as issues and pull requests) with matching GitLab users. +The steps you take depend on whether you are importing from GitHub.com or GitHub Enterprise, as well as whether you are importing to GitLab.com or self-managed GitLab instance. -### If you're importing from GitLab Enterprise - -If you're importing from GitHub Enterprise, you must enable [GitHub integration][gh-import]. - -### If you're using a self-managed GitLab instance - -If you're an administrator of a self-managed GitLab instance, you must enable -[GitHub integration][gh-import]. - -If you're an administrator of a self-managed GitLab instance, you can also use the -[GitHub Rake task](../../../administration/raketasks/github_import.md) to import projects from -GitHub without the constraints of a Sidekiq worker. +- If you're importing to GitLab.com, you can alternatively import GitHub repositories + using a [personal access token](#using-a-github-token). We do not recommend + this method, as it does not associate all user activity (such as issues and + pull requests) with matching GitLab users. +- If you're importing to a self-managed GitLab instance, you can alternatively use the + [GitHub Rake task](../../../administration/raketasks/github_import.md) to import + projects without the constraints of a [Sidekiq](../../../development/sidekiq_style_guide.md) worker. +- If you're importing from GitHub Enterprise to your self-managed GitLab instance, you must first enable + [GitHub integration](../../../integration/github.md). However, you cannot import projects from GitHub Enterprise to GitLab.com. +- If you're importing from GitHub.com to your self-managed GitLab instance, you do not need to set up GitHub integration. ## How it works @@ -106,7 +101,7 @@ If you are using a self-managed GitLab instance or if you are importing from Git 1. From the top navigation bar, click **+** and select **New project**. 1. Select the **Import project** tab and then select **GitHub**. 1. Select the first button to **List your GitHub repositories**. You are redirected to a page on [GitHub](https://github.com) to authorize the GitLab application. -1. Click **Authorize gitlabhq**. You are redirected back to GitLab's Import page and all of your GitHub repositories are listed. +1. Click **Authorize GitlabHQ**. You are redirected back to GitLab's Import page and all of your GitHub repositories are listed. 1. Continue on to [selecting which repositories to import](#selecting-which-repositories-to-import). ### Using a GitHub token @@ -124,7 +119,7 @@ If you are not using the GitHub integration, you can still perform an authorizat 1. Go to <https://github.com/settings/tokens/new> 1. Enter a token description. -1. Select the repo scope. +1. Select the repository scope. 1. Click **Generate token**. 1. Copy the token hash. 1. Go back to GitLab and provide the token to the GitHub importer. @@ -141,10 +136,10 @@ your GitHub repositories are listed. 1. Select the **Import** button next to any number of repositories, or select **Import all repositories**. Additionally, you can filter projects by name. If filter is applied, **Import all repositories** only imports matched repositories. 1. The **Status** column shows the import status of each repository. You can choose to leave the page open and it will - update in realtime or you can return to it later. + update in real-time or you can return to it later. 1. Once a repository has been imported, click its GitLab path to open its GitLab URL. - + ## Mirroring and pipeline status sharing @@ -154,7 +149,7 @@ your imported repository in sync with its GitHub copy. Additionally, you can configure GitLab to send pipeline status updates back GitHub with the [GitHub Project Integration](../integrations/github.md). **(PREMIUM)** -If you import your project using [CI/CD for external repo](../../../ci/ci_cd_for_external_repos/index.md), then both +If you import your project using [CI/CD for external repository](../../../ci/ci_cd_for_external_repos/index.md), then both of the above are automatically configured. **(PREMIUM)** ## Improving the speed of imports on self-managed instances diff --git a/doc/user/project/import/img/manifest_status_v13_3.png b/doc/user/project/import/img/manifest_status_v13_3.png Binary files differindex 3f0063e6715..c1a55ba1f50 100644 --- a/doc/user/project/import/img/manifest_status_v13_3.png +++ b/doc/user/project/import/img/manifest_status_v13_3.png diff --git a/doc/user/project/import/index.md b/doc/user/project/import/index.md index 86b671c8371..a1c28cfa2b7 100644 --- a/doc/user/project/import/index.md +++ b/doc/user/project/import/index.md @@ -18,7 +18,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w 1. [From Perforce](perforce.md) 1. [From SVN](svn.md) 1. [From TFVC](tfvc.md) -1. [From repo by URL](repo_by_url.md) +1. [From repository by URL](repo_by_url.md) 1. [By uploading a manifest file (AOSP)](manifest.md) 1. [From Gemnasium](gemnasium.md) 1. [From Phabricator](phabricator.md) @@ -32,7 +32,7 @@ There is also the option of [connecting your external repository to get CI/CD be ## Migrating from self-managed GitLab to GitLab.com -If you only need to migrate Git repos, you can [import each project by URL](repo_by_url.md). Issues and merge requests can't be imported. +If you only need to migrate Git repositories, you can [import each project by URL](repo_by_url.md). Issues and merge requests can't be imported. If you want to retain all metadata like issues and merge requests, you can use the [import/export feature](../settings/import_export.md) to export projects from self-managed GitLab and import those projects into GitLab.com. diff --git a/doc/user/project/import/manifest.md b/doc/user/project/import/manifest.md index 60524f3cc69..ba1e2011d08 100644 --- a/doc/user/project/import/manifest.md +++ b/doc/user/project/import/manifest.md @@ -56,7 +56,7 @@ You can start the import with: 1. From your GitLab dashboard click **New project** 1. Switch to the **Import project** tab 1. Click on the **Manifest file** button -1. Provide GitLab with a manifest xml file +1. Provide GitLab with a manifest XML file 1. Select a group you want to import to (you need to create a group first if you don't have one) 1. Click **List available repositories**. At this point, you will be redirected to the import status page with projects list based on the manifest file. diff --git a/doc/user/project/import/perforce.md b/doc/user/project/import/perforce.md index dbc1c491493..4ccc34efe30 100644 --- a/doc/user/project/import/perforce.md +++ b/doc/user/project/import/perforce.md @@ -20,7 +20,7 @@ Git: it creates an integration record in their proprietary database for every file in the branch, regardless how many were actually changed. Whereas Git was implemented with a different architecture so that a single SHA acts as a pointer - to the state of the whole repo after the changes, making it very easy to branch. + to the state of the whole repository after the changes, making it very easy to branch. This is what made feature branching workflows so easy to adopt with Git. 1. Also, context switching between branches is much easier in Git. If your manager said 'You need to stop work on that new feature and fix this security diff --git a/doc/user/project/import/repo_by_url.md b/doc/user/project/import/repo_by_url.md index 9b5e43aae79..5c53b6eaf06 100644 --- a/doc/user/project/import/repo_by_url.md +++ b/doc/user/project/import/repo_by_url.md @@ -5,7 +5,7 @@ group: Import info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# Import project from repo by URL +# Import project from repository by URL You can import your existing repositories by providing the Git URL: @@ -16,4 +16,4 @@ You can import your existing repositories by providing the Git URL: 1. Click **Create project** to begin the import process 1. Once complete, you will be redirected to your newly created project - + diff --git a/doc/user/project/index.md b/doc/user/project/index.md index c79f2be1d3f..a00f93bac9c 100644 --- a/doc/user/project/index.md +++ b/doc/user/project/index.md @@ -34,7 +34,7 @@ When you create a project in GitLab, you'll have access to a large number of - [Protected tags](protected_tags.md): Control over who has permission to create tags, and prevent accidental update or deletion - [Repository mirroring](repository/repository_mirroring.md) - - [Signing commits](gpg_signed_commits/index.md): use GPG to sign your commits + - [Signing commits](repository/gpg_signed_commits/index.md): use GPG to sign your commits - [Deploy tokens](deploy_tokens/index.md): Manage project-based deploy tokens that allow permanent access to the repository and Container Registry. - [Web IDE](web_ide/index.md) - [CVE ID Requests](../application_security/cve_id_request.md): Request a CVE identifier to track a @@ -96,9 +96,9 @@ When you create a project in GitLab, you'll have access to a large number of - [Wiki](wiki/index.md): document your GitLab project in an integrated Wiki. - [Snippets](../snippets.md): store, share and collaborate on code snippets. -- [Value Stream Analytics](cycle_analytics.md): review your development lifecycle. +- [Value Stream Analytics](../analytics/value_stream_analytics.md): review your development lifecycle. - [Insights](insights/index.md): configure the Insights that matter for your projects. **(ULTIMATE)** -- [Security Dashboard](security_dashboard.md): Security Dashboard. **(ULTIMATE)** +- [Security Dashboard](../application_security/security_dashboard/index.md): Security Dashboard. **(ULTIMATE)** - [Syntax highlighting](highlighting.md): an alternative to customize your code blocks, overriding GitLab's default choice of language. - [Badges](badges.md): badges for the project overview. diff --git a/doc/user/project/integrations/irker.md b/doc/user/project/integrations/irker.md index 443ca11be27..bb4a5b2b97f 100644 --- a/doc/user/project/integrations/irker.md +++ b/doc/user/project/integrations/irker.md @@ -14,8 +14,8 @@ See the project homepage for further information: <https://gitlab.com/esr/irker> ## Needed setup -You will first need an Irker daemon. You can download the Irker code from its -repository on <https://gitlab.com/esr/irker>: +You will first need an Irker daemon. You can download the Irker code +[from its repository](https://gitlab.com/esr/irker): ```shell git clone https://gitlab.com/esr/irker.git @@ -55,6 +55,6 @@ case, `Aorimn` is treated as a nick and no more as a channel name. Irker can also join password-protected channels. Users need to append `?key=thesecretpassword` to the channel name. When using this feature remember to **not** put the `#` sign in front of the channel name; failing to do so will -result on irker joining a channel literally named `#chan?key=password` henceforth +result on Irker joining a channel literally named `#chan?key=password` henceforth leaking the channel key through the `/whois` IRC command (depending on IRC server -configuration). This is due to a long standing irker bug. +configuration). This is due to a long standing Irker bug. diff --git a/doc/user/project/integrations/jira.md b/doc/user/project/integrations/jira.md index 3e0b6492477..b4e02bbd5f3 100644 --- a/doc/user/project/integrations/jira.md +++ b/doc/user/project/integrations/jira.md @@ -23,7 +23,7 @@ Features include: - **View a list of Jira issues directly in GitLab** **(PREMIUM)** For additional features, you can install the -[Jira Development Panel integration](../../../integration/jira_development_panel.md) **(PREMIUM)**. +[Jira Development Panel integration](../../../integration/jira_development_panel.md). This enables you to: - In a Jira issue, display relevant GitLab information in the [development panel](https://support.atlassian.com/jira-software-cloud/docs/view-development-information-for-an-issue/), including related branches, commits, and merge requests. @@ -122,6 +122,8 @@ By now you should have [configured Jira](#configuring-jira) and enabled the you should be able to reference and close Jira issues by just mentioning their ID in GitLab commits and merge requests. +Jira issue IDs must be formatted in uppercase for the integration to work. + ### Reference Jira issues When GitLab project has Jira issue tracker configured and enabled, mentioning diff --git a/doc/user/project/integrations/overview.md b/doc/user/project/integrations/overview.md index 7a1f757c138..a502dfbf320 100644 --- a/doc/user/project/integrations/overview.md +++ b/doc/user/project/integrations/overview.md @@ -50,7 +50,7 @@ Click on the service links to see further configuration instructions and details | [Mattermost slash commands](mattermost_slash_commands.md) | Mattermost chat and ChatOps slash commands | No | | [Mattermost Notifications](mattermost.md) | Receive event notifications in Mattermost | No | | [Microsoft teams](microsoft_teams.md) | Receive notifications for actions that happen on GitLab into a room on Microsoft Teams using Office 365 Connectors | No | -| Packagist | Update your project on Packagist, the main Composer repository | Yes | +| Packagist | Update your projects on Packagist, the main Composer repository | Yes | | Pipelines emails | Email the pipeline status to a list of recipients | No | | [Slack Notifications](slack.md) | Send GitLab events (for example, an issue was created) to Slack as notifications | No | | [Slack slash commands](slack_slash_commands.md) **(CORE ONLY)** | Use slash commands in Slack to control GitLab | No | diff --git a/doc/user/project/integrations/prometheus.md b/doc/user/project/integrations/prometheus.md index a19b819c823..28a9afa5bb0 100644 --- a/doc/user/project/integrations/prometheus.md +++ b/doc/user/project/integrations/prometheus.md @@ -1,6 +1,6 @@ --- stage: Monitor -group: APM +group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- @@ -58,6 +58,43 @@ CPU and Memory consumption is monitored, but requires [naming conventions](prome The [NGINX Ingress](../clusters/index.md#installing-applications) that is deployed by GitLab to clusters, is automatically annotated for monitoring providing key response metrics: latency, throughput, and error rates. +##### Example of Kubernetes service annotations and labels + +As an example, to activate Prometheus monitoring of a service: + +1. Add at least this annotation: `prometheus.io/scrape: 'true'`. +1. Add two labels so GitLab can retrieve metrics dynamically for any environment: + - `application: ${CI_ENVIRONMENT_SLUG}` + - `release: ${CI_ENVIRONMENT_SLUG}` +1. Create a dynamic PromQL query. For example, a query like + `temperature{application="{{ci_environment_slug}}",release="{{ci_environment_slug}}"}` to either: + - Add [custom metrics](../../../operations/metrics/index.md#adding-custom-metrics). + - Add [custom dashboards](../../../operations/metrics/dashboards/index.md). + +The following is a service definition to accomplish this: + +```yaml +--- +# Service +apiVersion: v1 +kind: Service +metadata: + name: service-${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG} + # === Prometheus annotations === + annotations: + prometheus.io/scrape: 'true' + labels: + application: ${CI_ENVIRONMENT_SLUG} + release: ${CI_ENVIRONMENT_SLUG} + # === End of Prometheus === +spec: + selector: + app: ${CI_PROJECT_NAME} + ports: + - port: ${EXPOSED_PORT} + targetPort: ${CONTAINER_PORT} +``` + ### Manual configuration of Prometheus #### Requirements @@ -136,10 +173,7 @@ one of them will be used: > - GitLab 9.3 added the [numeric comparison](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/27439) of the 30 minute averages. Developers can view the performance impact of their changes within the merge -request workflow. - -NOTE: **Note:** -Requires [Kubernetes](prometheus_library/kubernetes.md) metrics. +request workflow. This feature requires [Kubernetes](prometheus_library/kubernetes.md) metrics. When a source branch has been deployed to an environment, a sparkline and numeric comparison of the average memory consumption will appear. On the diff --git a/doc/user/project/integrations/prometheus_library/cloudwatch.md b/doc/user/project/integrations/prometheus_library/cloudwatch.md index e278c7eb664..70f8a55bb07 100644 --- a/doc/user/project/integrations/prometheus_library/cloudwatch.md +++ b/doc/user/project/integrations/prometheus_library/cloudwatch.md @@ -1,6 +1,6 @@ --- stage: Monitor -group: APM +group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- diff --git a/doc/user/project/integrations/prometheus_library/haproxy.md b/doc/user/project/integrations/prometheus_library/haproxy.md index 712805b75f2..0fbc49ddad7 100644 --- a/doc/user/project/integrations/prometheus_library/haproxy.md +++ b/doc/user/project/integrations/prometheus_library/haproxy.md @@ -1,6 +1,6 @@ --- stage: Monitor -group: APM +group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- diff --git a/doc/user/project/integrations/prometheus_library/index.md b/doc/user/project/integrations/prometheus_library/index.md index 6f2c2477eee..35b111ab2b2 100644 --- a/doc/user/project/integrations/prometheus_library/index.md +++ b/doc/user/project/integrations/prometheus_library/index.md @@ -1,6 +1,6 @@ --- stage: Monitor -group: APM +group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- diff --git a/doc/user/project/integrations/prometheus_library/kubernetes.md b/doc/user/project/integrations/prometheus_library/kubernetes.md index 29efe08e53d..43c2d305437 100644 --- a/doc/user/project/integrations/prometheus_library/kubernetes.md +++ b/doc/user/project/integrations/prometheus_library/kubernetes.md @@ -1,6 +1,6 @@ --- stage: Monitor -group: APM +group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- @@ -12,7 +12,7 @@ GitLab has support for automatically detecting and monitoring Kubernetes metrics ## Requirements -The [Prometheus](../prometheus.md) and [Kubernetes](../kubernetes.md) +The [Prometheus](../prometheus.md) and [Kubernetes](../../clusters/index.md) integration services must be enabled. ## Metrics supported diff --git a/doc/user/project/integrations/prometheus_library/nginx.md b/doc/user/project/integrations/prometheus_library/nginx.md index 0d3042463c9..1757378fb70 100644 --- a/doc/user/project/integrations/prometheus_library/nginx.md +++ b/doc/user/project/integrations/prometheus_library/nginx.md @@ -1,6 +1,6 @@ --- stage: Monitor -group: APM +group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- diff --git a/doc/user/project/integrations/prometheus_library/nginx_ingress.md b/doc/user/project/integrations/prometheus_library/nginx_ingress.md index 7bebe7b1e65..fdea800c410 100644 --- a/doc/user/project/integrations/prometheus_library/nginx_ingress.md +++ b/doc/user/project/integrations/prometheus_library/nginx_ingress.md @@ -1,6 +1,6 @@ --- stage: Monitor -group: APM +group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- @@ -8,11 +8,11 @@ info: To determine the technical writer assigned to the Stage/Group associated w > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/22133) in GitLab 11.7. +GitLab has support for automatically detecting and monitoring the Kubernetes NGINX Ingress controller. This is provided by leveraging the built-in Prometheus metrics included with Kubernetes NGINX Ingress controller [version 0.16.0](https://github.com/kubernetes/ingress-nginx/blob/master/Changelog.md#0160) onward. + NOTE: **Note:** NGINX Ingress versions prior to 0.16.0 offer an included [VTS Prometheus metrics exporter](nginx_ingress_vts.md), which exports metrics different than the built-in metrics. -GitLab has support for automatically detecting and monitoring the Kubernetes NGINX Ingress controller. This is provided by leveraging the built-in Prometheus metrics included with Kubernetes NGINX Ingress controller [version 0.16.0](https://github.com/kubernetes/ingress-nginx/blob/master/Changelog.md#0160) onward. - ## Requirements [Prometheus integration](../prometheus.md) must be active. diff --git a/doc/user/project/integrations/prometheus_library/nginx_ingress_vts.md b/doc/user/project/integrations/prometheus_library/nginx_ingress_vts.md index 326931e9790..ec7b1ee6d10 100644 --- a/doc/user/project/integrations/prometheus_library/nginx_ingress_vts.md +++ b/doc/user/project/integrations/prometheus_library/nginx_ingress_vts.md @@ -1,6 +1,6 @@ --- stage: Monitor -group: APM +group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- diff --git a/doc/user/project/integrations/services_templates.md b/doc/user/project/integrations/services_templates.md index 688643a85a7..abb072c9a0a 100644 --- a/doc/user/project/integrations/services_templates.md +++ b/doc/user/project/integrations/services_templates.md @@ -6,20 +6,52 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Service templates -Using a service template, GitLab administrators can provide default values for configuring integrations at the project level. +Using a service template, GitLab administrators can: -When you enable a service template, the defaults are applied to **all** projects that do not -already have the integration enabled or do not otherwise have custom values saved. -The values are pre-filled on each project's configuration page for the applicable integration. +- Provide default values for configuring integrations when creating new projects. +- Bulk configure all existing projects in one step. -If you disable the template, these values no longer appear as defaults, while -any values already saved for an integration remain unchanged. +When you enable a service template: + +- The defaults are applied to **all** existing projects that either: + - Don't already have the integration enabled. + - Don't have custom values stored for already enabled integrations. +- Values are populated on each project's configuration page for the applicable + integration. +- Settings are stored at the project level. + +If you disable the template: + +- GitLab default values again become the default values for integrations on + new projects. +- Projects previously configured using the template will continue to use + those settings. + +If you change the template, the revised values are applied to new projects. This feature +does not provide central administration of integration settings. + +## Central administration of project integrations + +A new set of features is being introduced in GitLab to provide more control over +how integrations are configured at the instance, group, and project level. + +[Read more about setting up project integration management](../../admin_area/settings/project_integration_management.md) +(introduced in GitLab 13.3) and [our plans for managing integrations](https://gitlab.com/groups/gitlab-org/-/epics/2137). ## Enable a service template Navigate to the **Admin Area > Service Templates** and choose the service template you wish to create. +Recommendation: + +- Test the settings on some projects individually before enabling a template. +- Copy the working settings from a project to the template. + +There is no "Test settings" option when enabling templates. If the settings do not work, +these incorrect settings will be applied to all existing projects that do not already have +the integration configured. Fixing the integration then needs to be done project-by-project. + ## Service for external issue trackers The following image shows an example service template for Redmine. diff --git a/doc/user/project/integrations/slack.md b/doc/user/project/integrations/slack.md index 03ff5f845b6..e6f12c2532f 100644 --- a/doc/user/project/integrations/slack.md +++ b/doc/user/project/integrations/slack.md @@ -64,7 +64,7 @@ The following triggers are available for Slack notifications: - **Tag push**: Triggered when a new tag is pushed to the repository. - **Pipeline**: Triggered when a pipeline status changes. - **Wiki page**: Triggered when a wiki page is created or updated. -- **Deployment**: Triggered when a deployment finishes. +- **Deployment**: Triggered when a deployment starts or finishes. - **Alert**: Triggered when a new, unique alert is recorded. ## Troubleshooting diff --git a/doc/user/project/integrations/webhooks.md b/doc/user/project/integrations/webhooks.md index 800eb1d3359..7adea5ebcd6 100644 --- a/doc/user/project/integrations/webhooks.md +++ b/doc/user/project/integrations/webhooks.md @@ -1303,7 +1303,12 @@ Note that `commit.id` is the ID of the pipeline, not the ID of the commit. ### Deployment events -Triggered when deployment is finished/failed/canceled. +Triggered when a deployment: + +- Starts ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/41214) in GitLab 13.5.) +- Succeeds +- Fails +- Is cancelled **Request Header**: diff --git a/doc/user/project/issue_board.md b/doc/user/project/issue_board.md index f8172a0f988..bce40e9a838 100644 --- a/doc/user/project/issue_board.md +++ b/doc/user/project/issue_board.md @@ -8,14 +8,12 @@ info: To determine the technical writer assigned to the Stage/Group associated w > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/5554) in [GitLab 8.11](https://about.gitlab.com/releases/2016/08/22/gitlab-8-11-released/#issue-board). -## Overview - The GitLab Issue Board is a software project management tool used to plan, organize, and visualize a workflow for a feature or product release. It can be used as a [Kanban](https://en.wikipedia.org/wiki/Kanban_(development)) or a [Scrum](https://en.wikipedia.org/wiki/Scrum_(software_development)) board. -It pairs issue tracking and project management, keeping everything in the same place, +It pairs issue tracking and project management, keeping everything together, so that you don't need to jump between different platforms to organize your workflow. Issue boards build on the existing [issue tracking functionality](issues/index.md#issues-list) and @@ -26,8 +24,8 @@ Issue boards help you to visualize and manage your entire process in GitLab. You add your labels, and then create the corresponding list for your existing issues. When you're ready, you can drag your issue cards from one step to another one. -An issue board can show you what issues your team is working on, who is assigned to each, -and where in the workflow those issues are. +An issue board can show you the issues your team is working on, who is assigned to each, +and where the issues are in the workflow. To let your team members organize their own workflows, use [multiple issue boards](#use-cases-for-multiple-issue-boards). This allows creating multiple issue @@ -60,8 +58,8 @@ Here are some common use cases for issue boards. ### Use cases for a single issue board -With the GitLab Workflow you can discuss proposals in issues, categorize them -with labels, and from there, organize and prioritize them with issue boards. +With the GitLab Workflow you can discuss proposals in issues, label +them, and organize and prioritize them with issue boards. For example, let's consider this simplified development workflow: @@ -155,7 +153,7 @@ card includes: ## Permissions Users with the [Reporter and higher roles](../permissions.md) can use all the functionality of the -Issue Board feature to create or delete lists and drag issues from one list to another. +Issue Board feature to create or delete lists. They can also drag issues from one list to another. ## How GitLab orders issues in a list @@ -164,20 +162,19 @@ that order by dragging the issues. The changed order is saved, so that anybody w board later sees the reordering, with some exceptions. The first time a given issue appears in any board (that is, the first time a user -loads a board containing that issue), it is ordered in relation to other issues in that list -according to [label priority](labels.md#label-priority). +loads a board containing that issue), it is ordered in relation to other issues in that list. +The order is done according to [label priority](labels.md#label-priority). At this point, that issue is assigned a relative order value by the system, -representing its relative order with respect to the other issues in the list. Any time -you reorder that issue by dragging, its relative order value changes accordingly. +with respect to the other issues in the list. Any time +you drag and reorder the issue, its relative order value changes accordingly. -Also, any time that issue appears in any board when it's loaded by a user, -the updated relative order value is used for the ordering. It's only the first -time an issue appears that it takes from the priority order mentioned above. This means that -if issue `A` is reordered by dragging to be above issue `B` by any user in -a given board inside your GitLab instance, any time those two issues are subsequently -loaded in any board in the same instance (could be a different project board or a different group -board, for example), that ordering is maintained. +Also, any time that issue appears in any board, the ordering is done according to +the updated relative order value. It's only the first +time an issue appears that it takes from the priority order mentioned above. If a user in your GitLab instance +drags issue `A` above issue `B`, the ordering is maintained when these two issues are subsequently +loaded in any board in the same instance. This could be a different project board or a different group +board, for example. This ordering also affects [issue lists](issues/sorting_issue_lists.md). Changing the order in an issue board changes the ordering in an issue list, @@ -195,8 +192,7 @@ advanced functionality is present in [higher tiers only](https://about.gitlab.co > - Multiple issue boards per group are available in [GitLab Premium](https://about.gitlab.com/pricing/). Multiple issue boards allow for more than one issue board for a given project or group. -This is great for large projects with more than one team or in situations where a repository is used -to host the code of multiple products. +This is great for large projects with more than one team or when a repository hosts the code of multiple products. Using the search box at the top of the menu, you can filter the listed boards. @@ -230,13 +226,13 @@ To delete the currently active issue board: An issue board can be associated with a GitLab [Milestone](milestones/index.md#milestones), [Labels](labels.md), Assignee and Weight -which will automatically filter the Board issues according to these fields. +which automatically filter the board issues accordingly. This allows you to create unique boards according to your team's need.  You can define the scope of your board when creating it or by clicking the "Edit board" button. -Once a milestone, assignee or weight is assigned to an issue board, you will no longer be able to +Once a milestone, assignee or weight is assigned to an issue board, you can no longer filter through these in the search bar. In order to do that, you need to remove the desired scope (for example, milestone, assignee, or weight) from the issue board. @@ -274,24 +270,22 @@ especially in combination with [assignee lists](#assignee-lists). ### Group issue boards **(PREMIUM)** -> [Introduced](https://about.gitlab.com/releases/2017/09/22/gitlab-10-0-released/#group-issue-boards) in [GitLab Premium](https://about.gitlab.com/pricing/) 10.0. +> - One group issue board per group introduced in GitLab 10.6. +> - Multiple group issue boards [introduced](https://about.gitlab.com/releases/2017/09/22/gitlab-10-0-released/#group-issue-boards) in [GitLab Premium](https://about.gitlab.com/pricing/) 10.0. -Accessible at the group navigation level, a group issue board offers the same features as a project-level board, -but it can display issues from all projects in that +Accessible at the group navigation level, a group issue board offers the same features as a project-level board. +It can display issues from all projects in that group and its descendant subgroups. Similarly, you can only filter by group labels for these boards. When updating milestones and labels for an issue through the sidebar update mechanism, again only group-level objects are available. -NOTE: **Note:** -Multiple group issue boards were originally [introduced](https://about.gitlab.com/releases/2017/09/22/gitlab-10-0-released/#group-issue-boards) in [GitLab Premium](https://about.gitlab.com/pricing/) 10.0, and one group issue board per group was made available in GitLab Core 10.6. -  ### Assignee lists **(PREMIUM)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/5784) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.0. -Like in a regular list that shows all issues with a chosen label, you can add +As in a regular list showing all issues with a chosen label, you can add an assignee list that shows all issues assigned to a user. You can have a board with both label lists and assignee lists. To add an assignee list: @@ -317,7 +311,7 @@ milestone, giving you more freedom and visibility on the issue board. To add a m 1. Select the **Milestone** tab. 1. Search and click the milestone. -Similar to the assignee lists, you're now able to [drag issues](#drag-issues-between-lists) +Like the assignee lists, you're able to [drag issues](#drag-issues-between-lists) to and from a milestone list to manipulate the milestone of the dragged issues. As in other list types, click the trash icon to remove a list. @@ -333,7 +327,7 @@ You cannot set a WIP limit on the default lists (**Open** and **Closed**). Examples: -- You have a list with four issues, and a limit of five, the header will show **4/5**. +- When you have a list with four issues and a limit of five, the header shows **4/5**. If you exceed the limit, the current number of issues is shown in red. - You have a list with five issues with a limit of five. When you move another issue to that list, the list's header displays **6/5**, with the six shown in red. @@ -374,29 +368,24 @@ If you're not able to do some of the things above, make sure you have the right ### First time using an issue board -The first time you open an issue board, you are presented with -the default lists (**Open** and **Closed**) and a welcome message that gives -you two options. You can either: +> The automatic creation of the **To Do** and **Doing** lists was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/202144) in GitLab 13.5. -- Create a predefined set of labels (by default: **To Do** and **Doing**) and create their - corresponding lists to the issue board. -- Opt-out and use your own lists. +The first time you open an issue board, you are presented with the default lists +(**Open**, **To Do**, **Doing**, and **Closed**). - +If the **To Do** and **Doing** labels don't exist in the project or group, they are created, and +their lists appear as empty. If any of them already exists, the list is filled with the issues that +have that label. -If you choose to use and create the predefined lists, they will appear as empty -because the labels associated to them will not exist up until that moment, -which means the system has no way of populating them automatically. That's of -course if the predefined labels don't already exist. If any of them does exist, -the list will be created and filled with the issues that have that label. + ### Create a new list Create a new list by clicking the **Add list** button in the upper right corner of the issue board. - + -Then, choose the label or user to create the list from. The new list will be inserted +Then, choose the label or user to create the list from. The new list is inserted at the end of the lists, before **Done**. Moving and reordering lists is as easy as dragging them around. @@ -407,15 +396,15 @@ You can now choose it to create a list. ### Delete a list To delete a list from the issue board, use the small trash icon present -in the list's heading. A confirmation dialog will appear for you to confirm. +in the list's heading. A confirmation dialog appears for you to confirm. -Deleting a list doesn't have any effect in issues and labels, it's just the -list view that is removed. You can always add it back later if you need. +Deleting a list doesn't have any effect on issues and labels, as it's just the +list view that's removed. You can always restore it later if you need. ### Add issues to a list You can add issues to a list by clicking the **Add issues** button -present in the upper right corner of the issue board. This will open up a modal +present in the upper right corner of the issue board. This opens up a modal window where you can see all the issues that do not belong to any list. Select one or more issues by clicking the cards and then click **Add issues** @@ -435,7 +424,7 @@ respective label is removed. ### Filter issues You should be able to use the filters on top of your issue board to show only -the results you want. This is similar to the filtering used in the issue tracker +the results you want. It's similar to the filtering used in the issue tracker since the metadata from the issues and labels are re-used in the issue board. You can filter by author, assignee, milestone, and label. @@ -469,12 +458,12 @@ For example, you can create a list based on the label of **Frontend** and one fo worked on by the designers. Then, once they're done, all they have to do is -drag it to the next list, **Backend**, where a backend developer can +drag it to the next list, **Backend**. Then, a backend developer can eventually pick it up. Once they’re done, they move it to **Done**, to close the issue. -This process can be seen clearly when visiting an issue since with every move -to another list the label changes and a system note is recorded. +This process can be seen clearly when visiting an issue. With every move +to another list, the label changes and a system note is recorded.  diff --git a/doc/user/project/issues/crosslinking_issues.md b/doc/user/project/issues/crosslinking_issues.md index c721bef8f4d..2a75f8ad837 100644 --- a/doc/user/project/issues/crosslinking_issues.md +++ b/doc/user/project/issues/crosslinking_issues.md @@ -33,7 +33,7 @@ Of course, you can replace `gitlab.com` with the URL of your own GitLab instance NOTE: **Note:** Linking your first commit to your issue is going to be relevant -for tracking your process with [GitLab Cycle Analytics](https://about.gitlab.com/stages-devops-lifecycle/value-stream-analytics/). +for tracking your process with [GitLab Value Stream Analytics](https://about.gitlab.com/stages-devops-lifecycle/value-stream-analytics/). It will measure the time taken for planning the implementation of that issue, which is the time between creating an issue and making the first commit. diff --git a/doc/user/project/issues/design_management.md b/doc/user/project/issues/design_management.md index 7c9278c8403..6f57487fccd 100644 --- a/doc/user/project/issues/design_management.md +++ b/doc/user/project/issues/design_management.md @@ -4,14 +4,12 @@ group: Knowledge info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" --- -# Design Management +# Design Management **(CORE)** > - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/660) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.2. > - Support for SVGs was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12771) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.4. > - Design Management was [moved](https://gitlab.com/gitlab-org/gitlab/-/issues/212566) to GitLab Core in 13.0. -## Overview - Design Management allows you to upload design assets (wireframes, mockups, etc.) to GitLab issues and keep them stored in one single place, accessed by the Design Management's page within an issue, giving product designers, product managers, and engineers a @@ -58,8 +56,7 @@ Support for [PDF](https://gitlab.com/gitlab-org/gitlab/issues/32811) is planned - From GitLab 13.1, Design filenames are limited to 255 characters. - Design Management data [isn't deleted when a project is destroyed](https://gitlab.com/gitlab-org/gitlab/-/issues/13429) yet. -- Design Management data [won't be moved](https://gitlab.com/gitlab-org/gitlab/-/issues/13426) - when an issue is moved, nor [deleted](https://gitlab.com/gitlab-org/gitlab/-/issues/13427) +- Design Management data [won't be deleted](https://gitlab.com/gitlab-org/gitlab/-/issues/13427) when an issue is deleted. - From GitLab 12.7, Design Management data [can be replicated](../../../administration/geo/replication/datatypes.md#limitations-on-replicationverification) by Geo but [not verified](https://gitlab.com/gitlab-org/gitlab/-/issues/32467). @@ -114,9 +111,6 @@ Designs with the same filename as an existing uploaded design will create a new of the design, and will replace the previous version. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/34353) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.9, dropping a design on an existing uploaded design will also create a new version, provided the filenames are the same. -Designs cannot be added if the issue has been moved, or its -[discussion is locked](../../discussions/#lock-discussions). - ### Skipped designs Designs with the same filename as an existing uploaded design _and_ whose content has not changed will be skipped. @@ -185,9 +179,7 @@ viewed by browsing previous versions. > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/34382) in GitLab 13.3. -You can change the order of designs by dragging them to a new position: - - +You can change the order of designs by dragging them to a new position. ## Starting discussions on designs @@ -231,47 +223,19 @@ Note that your resolved comment pins will disappear from the Design to free up s However, if you need to revisit or find a resolved discussion, all of your resolved threads will be available in the **Resolved Comment** area at the bottom of the right sidebar. -## Add To-Do for Designs +## Add to dos for designs > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/198439) in GitLab 13.4. -> - It's [deployed behind a feature flag](../../feature_flags.md), enabled by default. -> - It's enabled on GitLab.com. -> - It's recommended for production use. -> - For GitLab self-managed instances, GitLab administrators can opt to [disable it](#enable-or-disable-the-design-to-do-button). **(CORE ONLY)** - -CAUTION: **Warning:** -This feature might not be available to you. Check the **version history** note above for details. - -Add a to-do for a design by clicking **Add a To-Do** on the design sidebar: - - - -### Enable or disable the design To-Do button **(CORE ONLY)** - -The **Add a To-Do** button for Designs is under development but ready for production use. It is -deployed behind a feature flag that is **enabled by default**. -[GitLab administrators with access to the GitLab Rails console](../../../administration/feature_flags.md) -can enable it. - -To enable it: - -```ruby -Feature.enable(:design_management_todo_button) -``` +> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/245074) in GitLab 13.5. -To disable it: +Add a to do for a design by clicking **Add a To Do** on the design sidebar: -```ruby -Feature.disable(:design_management_todo_button) -``` + ## Referring to designs in Markdown > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/217160) in **GitLab 13.1**. -> - It is deployed behind a feature flag, disabled by default. -> - It is disabled on GitLab.com. -> - It is not recommended for production use. -> - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enable-or-disable-design-references). **(CORE ONLY)** +> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/258662) in **GitLab 13.5** We support referring to designs in [Markdown](../../markdown.md), which is available throughout the application, including in merge request and issue descriptions, in discussions and comments, and in wiki pages. @@ -287,25 +251,6 @@ This will be rendered as: > See [#123[homescreen.png]](https://gitlab.com/your-group/your-project/-/issues/123/designs/homescreen.png) -### Enable or disable design references **(CORE ONLY)** - -Design reference parsing is -deployed behind a feature flag that is **enabled by default**. -[GitLab administrators with access to the GitLab Rails console](../../../administration/feature_flags.md) -can disable it for your instance. - -To disable it: - -```ruby -Feature.disable(:design_management_reference_filter_gfm_pipeline) -``` - -To re-enable it: - -```ruby -Feature.enable(:design_management_reference_filter_gfm_pipeline) -``` - ## Design activity records > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/33051) in GitLab 13.1. diff --git a/doc/user/project/issues/due_dates.md b/doc/user/project/issues/due_dates.md index 55b45bf9d3d..b3ebefadef0 100644 --- a/doc/user/project/issues/due_dates.md +++ b/doc/user/project/issues/due_dates.md @@ -46,7 +46,7 @@ the icon and the date colored red. You can sort issues by those that are Due dates also appear in your [to-do list](../../todos.md). - + The day before an open issue is due, an email will be sent to all participants of the issue. Like the due date, the "day before the due date" is determined by the diff --git a/doc/user/project/issues/img/design_todo_button_v13_4.png b/doc/user/project/issues/img/design_todo_button_v13_4.png Binary files differdeleted file mode 100644 index 62bbecf4ed9..00000000000 --- a/doc/user/project/issues/img/design_todo_button_v13_4.png +++ /dev/null diff --git a/doc/user/project/issues/img/design_todo_button_v13_5.png b/doc/user/project/issues/img/design_todo_button_v13_5.png Binary files differnew file mode 100644 index 00000000000..970161a6097 --- /dev/null +++ b/doc/user/project/issues/img/design_todo_button_v13_5.png diff --git a/doc/user/project/issues/img/designs_reordering_v13_3.gif b/doc/user/project/issues/img/designs_reordering_v13_3.gif Binary files differdeleted file mode 100644 index 496eea532e2..00000000000 --- a/doc/user/project/issues/img/designs_reordering_v13_3.gif +++ /dev/null diff --git a/doc/user/project/issues/index.md b/doc/user/project/issues/index.md index 060266a478f..434af3a4a49 100644 --- a/doc/user/project/issues/index.md +++ b/doc/user/project/issues/index.md @@ -156,12 +156,15 @@ collaborate with your team. efficiently and with less effort by tracking groups of issues that share a theme, across projects and milestones. -### Related issues **(STARTER)** +### Related issues You can mark two issues as related, so that when viewing one, the other is always listed in its [Related Issues](related_issues.md) section. This can help display important context, such as past work, dependencies, or duplicates. +Users on [GitLab Starter, GitLab Bronze, and higher tiers](https://about.gitlab.com/pricing/), can +also mark issues as blocking or blocked by another issue. + ### Crosslinking issues You can [cross-link issues](crosslinking_issues.md) by referencing an issue from another diff --git a/doc/user/project/issues/issue_data_and_actions.md b/doc/user/project/issues/issue_data_and_actions.md index 5356e6aeb40..003444e0ed8 100644 --- a/doc/user/project/issues/issue_data_and_actions.md +++ b/doc/user/project/issues/issue_data_and_actions.md @@ -36,7 +36,7 @@ You can find all the information for that issue on one screen. - **15.** [Edit](#edit) - **16.** [Description](#description) - **17.** [Mentions](#mentions) -- **18.** [Related Issues **(STARTER)**](#related-issues) +- **18.** [Related Issues](#related-issues) - **19.** [Related Merge Requests](#related-merge-requests) - **20.** [Award emoji](#award-emoji) - **21.** [Show all activity](#show-all-activity) @@ -80,7 +80,7 @@ The button to do this has a different label depending on whether the issue is al List or not. If the issue is: - Already on your To-Do List: The button is labeled **Mark as done**. Click the button to remove the issue from your To-Do List. -- Not on your To-Do List: The button is labeled **Add a To Do**. Click the button to add the issue to your To-Do List. +- Not on your To-Do List: The button is labeled **Add a to do**. Click the button to add the issue to your To-Do List. ### Assignee @@ -191,7 +191,7 @@ The plain text title and description of the issue fill the top center of the iss The description fully supports [GitLab Flavored Markdown](../../markdown.md#gitlab-flavored-markdown-gfm), allowing many formatting options. -> [Since GitLab 12.6](https://gitlab.com/gitlab-org/gitlab/-/issues/10103), changes to an issue's description are listed in the [issue history](#issue-history).**(STARTER)** +> [In GitLab 12.6](https://gitlab.com/gitlab-org/gitlab/-/issues/10103) and later, changes to an issue's description are listed in the [issue history](#issue-history).**(STARTER)** ### Mentions @@ -208,7 +208,7 @@ TIP: **Tip:** Avoid mentioning `@all` in issues and merge requests, as it sends an email notification to all the members of that project's group, which can be interpreted as spam. -### Related Issues **(STARTER)** +### Related Issues Issues that were mentioned as [related issues](related_issues.md) are listed here. You can also click the `+` to add more related issues. @@ -242,7 +242,7 @@ and selecting either: Also: - You can mention a user or a group present in your GitLab instance with - `@username` or `@groupname` and they will be notified via To-Do items + `@username` or `@groupname` and they will be notified via to-do items and email, unless they have [disabled all notifications](#notifications) in their profile settings. - Mentions for yourself (the current logged in user), will be highlighted diff --git a/doc/user/project/issues/managing_issues.md b/doc/user/project/issues/managing_issues.md index 0e0731528be..b033dc79dcc 100644 --- a/doc/user/project/issues/managing_issues.md +++ b/doc/user/project/issues/managing_issues.md @@ -127,6 +127,7 @@ field). | title | `issue[title]` | | | description | `issue[description]` | | | description template | `issuable_template` | | +| issue type | `issue[issue_type]` | Either `incident` or `issue` | | confidential | `issue[confidential]` | Parameter value must be `true` to set to confidential | Follow these examples to form your new issue URL with prefilled fields. diff --git a/doc/user/project/issues/multiple_assignees_for_issues.md b/doc/user/project/issues/multiple_assignees_for_issues.md index c11977f5bee..b1806460c08 100644 --- a/doc/user/project/issues/multiple_assignees_for_issues.md +++ b/doc/user/project/issues/multiple_assignees_for_issues.md @@ -8,8 +8,6 @@ info: To determine the technical writer assigned to the Stage/Group associated w > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/1904) in [GitLab Starter 9.2](https://about.gitlab.com/releases/2017/05/22/gitlab-9-2-released/#multiple-assignees-for-issues). -## Overview - In large teams, where there is shared ownership of an issue, it can be difficult to track who is working on it, who already completed their contributions, who didn't even start yet. diff --git a/doc/user/project/issues/related_issues.md b/doc/user/project/issues/related_issues.md index 954e3771722..b040bcf3b03 100644 --- a/doc/user/project/issues/related_issues.md +++ b/doc/user/project/issues/related_issues.md @@ -4,33 +4,40 @@ group: Project Management info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# Related issues **(STARTER)** +# Related issues **(CORE)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/1797) in [GitLab Starter](https://about.gitlab.com/pricing/) 9.4. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/1797) in [GitLab Starter](https://about.gitlab.com/pricing/) 9.4. +> - The simple "relates to" relationship [moved](https://gitlab.com/gitlab-org/gitlab/-/issues/212329) to [GitLab Core](https://about.gitlab.com/pricing/) in 13.4. Related issues are a bi-directional relationship between any two issues and appear in a block below the issue description. Issues can be across groups and projects. +You can set any issue as: + +- Related to another issue +- Blocking another issue **(STARTER)** +- Blocked by another issue **(STARTER)** + The relationship only shows up in the UI if the user can see both issues. When you try to close an issue that has open blockers, a warning is displayed. TIP: **Tip:** -To manage related issues through our API, see the [API documentation](../../../api/issue_links.md). +To manage related issues through our API, visit the [issue links API documentation](../../../api/issue_links.md). ## Adding a related issue > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/2035) in [GitLab Starter](https://about.gitlab.com/pricing/) 12.8. > - [Improved](https://gitlab.com/gitlab-org/gitlab/-/issues/34239) to warn when attempting to close an issue that is blocked by others in [GitLab Starter](https://about.gitlab.com/pricing/) 13.0. -> When you try to close an issue with open blockers, you'll see a warning that you can dismiss. +> When you try to close an issue with open blockers, you see a warning that you can dismiss. 1. Relate one issue to another by clicking the related issues "+" button in the header of the related issue block. 1. Input the issue reference number or paste in the full URL of the issue. -1. Select whether the current issue relates to, blocks, or is blocked by the issues being entered. +1. **(STARTER)** Select whether the current issue relates to, blocks, or is blocked by the issues being entered.  @@ -42,11 +49,11 @@ in the header of the related issue block. - same group: `project#44` - different group: `group/project#44` - Valid references will be added to a temporary list that you can review. + Valid references are added to a temporary list that you can review. 1. When you have added all the related issues, click **Add** to submit. -When you have finished adding all related issues, you will be able to see +When you have finished adding all related issues, you can see them categorized so their relationships can be better understood visually.  @@ -56,7 +63,7 @@ them categorized so their relationships can be better understood visually. In the related issues block, click the "x" icon on the right-side of each issue token that you wish to remove. -Due to the bi-directional relationship, it will no longer appear in either issue. +Due to the bi-directional relationship, it no longer appears in either issue.  diff --git a/doc/user/project/labels.md b/doc/user/project/labels.md index 0d02b89be7e..4f0354f86af 100644 --- a/doc/user/project/labels.md +++ b/doc/user/project/labels.md @@ -6,8 +6,6 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Labels -## Overview - As your count of issues, merge requests, and epics grows in GitLab, it's more and more challenging to keep track of those items. Especially as your organization grows from just a few people to hundreds or thousands. This is where labels come in. They help you organize and tag your work @@ -32,18 +30,25 @@ There are two types of labels in GitLab: ## Assign and unassign labels -Every issue, merge request and epic can be assigned any number of labels. The labels are +> Unassigning labels with the **X** button [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/216881) in GitLab 13.5. + +Every issue, merge request, and epic can be assigned any number of labels. The labels are managed in the right sidebar, where you can assign or unassign labels as needed. -To assign a label to an issue, merge request or epic: +To assign or unassign a label: -1. In the label section of the sidebar, click **Edit**, then: - - In the list, click the labels you want. Each label is flagged with a checkmark. - - Find labels by entering a search query and clicking search (**{search}**), then - click on them. You can search repeatedly and add more labels. -1. Click **X** or anywhere outside the label section and the labels are applied. +1. In the **Labels** section of the sidebar, click **Edit**. +1. In the **Assign labels** list, search for labels by typing their names. + You can search repeatedly to add more labels. + The selected labels are marked with a checkmark. +1. Click the labels you want to assign or unassign. +1. To apply your changes to labels, click **X** next to **Assign labels** or anywhere outside the + label section. -You can also assign a label with the [`/label ~label1 ~label2` quick action](quick_actions.md). +Alternatively, to unassign a label, click the **X** on the label you want to unassign. + +You can also assign a label with the `/label` [quick action](quick_actions.md), +remove labels with `/unlabel`, and reassign labels (remove all and assign new ones) with `/relabel`. ## Label management @@ -52,10 +57,13 @@ and edit labels. ### Project labels -View the project labels list by going to the project and clicking **Issues > Labels**. +> Showing all inherited labels [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/241990) in 13.5. + +To view the project labels list, navigate to the project and click **Issues > Labels**. The list includes all labels that are defined at the project level, as well as all -labels inherited from the immediate parent group. You can filter the list by entering a search -query at the top and clicking search (**{search}**). +labels defined by its ancestor groups. +For each label, you can see the project or group path from where it was created. +You can filter the list by entering a search query at the top and clicking search (**{search}**). To create a new project label: @@ -83,19 +91,19 @@ a label by clicking the three dots (**{ellipsis_v}**) next to the **Subscribe** and selecting **Delete**. CAUTION: **Caution:** -If you delete a label, it is permanently deleted. You will not be able to undo the deletion, and all references to the label will be removed from the system. +If you delete a label, it is permanently deleted. All references to the label are removed from the system and you cannot undo the deletion. #### Promote a project label to a group label If you previously created a project label and now want to make it available for other projects within the same group, you can promote it to a group label. -If other projects in the same group have a label with the same title, they will all be -merged with the new group label. If a group label with the same title exists, it will -also be merged. +If other projects in the same group have a label with the same title, they are all +merged with the new group label. If a group label with the same title exists, it is +also merged. All issues, merge requests, issue board lists, issue board filters, and label subscriptions -with the old labels will be assigned to the new group label. +with the old labels are assigned to the new group label. CAUTION: **Caution:** Promoting a label is a permanent action, and cannot be reversed. @@ -108,7 +116,7 @@ To promote a project label to a group label: ### Group labels -View the group labels list by going to the group and clicking **Issues > Labels**. +To view the group labels list, navigate to the group and click **Issues > Labels**. The list includes all labels that are defined at the group level only. It does not list any labels that are defined in projects. You can filter the list by entering a search query at the top and clicking search (**{search}**). @@ -118,15 +126,15 @@ follow the same process as [creating a project label](#project-labels). #### Create group labels from epics **(ULTIMATE)** -You can create group labels from the Epic sidebar. The labels you create will +You can create group labels from the epic sidebar. The labels you create belong to the immediate group to which the epic belongs. The process is the same as creating a [project label from an issue or merge request](#project-labels). ### Generate default labels If a project or group has no labels, you can generate a default set of project or group -labels from the label list page. The page will show a **Generate a default set of labels** -button if the list is empty, and clicking it will add the following default labels +labels from the label list page. The page shows a **Generate a default set of labels** +button if the list is empty. Select the button to add the following default labels to the project: - `bug` @@ -149,13 +157,13 @@ by preventing certain labels from being used together. A label is scoped when it uses a special double-colon (`::`) syntax in the label’s title, for example: - + An issue, merge request or epic cannot have two scoped labels, of the form `key::value`, -with the same `key`. Adding a new label with the same `key`, but a different `value` will -cause the previous `key` label to be replaced with the new label. +with the same `key`. Adding a new label with the same `key`, but a different `value` +causes the previous `key` label to be replaced with the new label. -Example use case: +For example: 1. An issue is identified as being low priority, and a `priority::low` project label is added to it. @@ -188,7 +196,7 @@ This functionality is demonstrated in a video regarding ### Scoped labels with nested scopes You can create a label with a nested scope by using multiple double colons `::` when creating -it. In this case, everything before the last `::` will be the scope. +it. In this case, everything before the last `::` is the scope. For example, `workflow::backend::review` and `workflow::backend::development` are valid scoped labels, but they **can't** exist on the same issue at the same time, as they @@ -202,13 +210,13 @@ both have different scopes, `workflow::frontend` and `workflow::backend`. From the project label list page and the group label list page, you can click **Subscribe** to the right of any label to enable [notifications](../profile/notifications.md) for that -label. You will be notified whenever the label is assigned to an epic, +label. You are notified whenever the label is assigned to an epic, issue, or merge request. If you are subscribing to a group label from within a project, you can select to subscribe to label notifications for the project only, or the whole group. - + ## Label priority @@ -222,7 +230,7 @@ from the group label list. From the project label list page, star a label to indicate that it has a priority. - + Drag starred labels up and down the list to change their priority, where higher in the list means higher priority. diff --git a/doc/user/project/merge_requests/accessibility_testing.md b/doc/user/project/merge_requests/accessibility_testing.md index f3a0aac9ff4..a07a155745e 100644 --- a/doc/user/project/merge_requests/accessibility_testing.md +++ b/doc/user/project/merge_requests/accessibility_testing.md @@ -55,7 +55,7 @@ include: ``` creates an `a11y` job in your CI/CD pipeline, runs -Pa11y against the webpages defined in `a11y_urls`, and builds an HTML report for each. +Pa11y against the web pages defined in `a11y_urls`, and builds an HTML report for each. The report for each URL is saved as an artifact that can be [viewed directly in your browser](../../../ci/pipelines/job_artifacts.md#browsing-artifacts). diff --git a/doc/user/project/merge_requests/allow_collaboration.md b/doc/user/project/merge_requests/allow_collaboration.md index 60d247ccc19..4ba0b50a3cf 100644 --- a/doc/user/project/merge_requests/allow_collaboration.md +++ b/doc/user/project/merge_requests/allow_collaboration.md @@ -28,11 +28,12 @@ source project and only lasts while the merge request is open. Once enabled, upstream members will also be able to retry the pipelines and jobs of the merge request: -1. Enable the contribution while creating or editing a merge request. +1. While creating or editing a merge request, select the checkbox **Allow + commits from members who can merge to the target branch**.  -1. Once the merge request is created, you'll see that commits from members who +1. Once the merge request is created, you can see that commits from members who can merge to the target branch are allowed.  diff --git a/doc/user/project/merge_requests/getting_started.md b/doc/user/project/merge_requests/getting_started.md index a0be32e0708..462b8f68ece 100644 --- a/doc/user/project/merge_requests/getting_started.md +++ b/doc/user/project/merge_requests/getting_started.md @@ -111,6 +111,48 @@ It is also possible to manage multiple assignees: - When creating a merge request. - Using [quick actions](../quick_actions.md#quick-actions-for-issues-merge-requests-and-epics). +## Reviewer + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/216054) in GitLab 13.5. +> - It's [deployed behind a feature flag](../../../user/feature_flags.md), enabled by default. +> - It's disabled on GitLab.com. +> - It's not recommended for production use. +> - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enable-or-disable-merge-request-reviewers). **(CORE ONLY)** + +CAUTION: **Warning:** +This feature might not be available to you. Check the **version history** note above for details. + +Requesting a code review is an important part of contributing code. However, deciding who should review +your code and asking for a review are no easy tasks. Using the "assignee" field for both authors and +reviewers makes it hard for others to determine who's doing what on a merge request. + +GitLab's Merge Request Reviewers easily allow authors to request a review as well as see the status of the +review. By selecting one or more users from the **Reviewers** field in the merge request's right-hand +sidebar, the assigned reviewers will receive a notification of the request to review the merge request. + +This makes it easy to determine the relevant roles for the users involved in the merge request, as well as formally requesting a review from a peer. + +To request it, open the **Reviewers** drop-down box to search for the user you wish to get a review from. + +### Enable or disable Merge Request Reviewers **(CORE ONLY)** + +Merge Request Reviewers is under development and not ready for production use. It is +deployed behind a feature flag that is **disabled by default**. +[GitLab administrators with access to the GitLab Rails console](../../../administration/feature_flags.md) +can enable it. + +To enable it: + +```ruby +Feature.enable(:merge_request_reviewers) +``` + +To disable it: + +```ruby +Feature.disable(:merge_request_reviewers) +``` + ### Merge requests to close issues If the merge request is being created to resolve an issue, you can diff --git a/doc/user/project/merge_requests/img/commit_nav_v13_4.png b/doc/user/project/merge_requests/img/commit_nav_v13_4.png Binary files differnew file mode 100644 index 00000000000..0ae6ce32693 --- /dev/null +++ b/doc/user/project/merge_requests/img/commit_nav_v13_4.png diff --git a/doc/user/project/merge_requests/load_performance_testing.md b/doc/user/project/merge_requests/load_performance_testing.md index daebd71e14f..2675f509eed 100644 --- a/doc/user/project/merge_requests/load_performance_testing.md +++ b/doc/user/project/merge_requests/load_performance_testing.md @@ -164,8 +164,8 @@ For example: 1. Capture the dynamic URL and save it into a `.env` file, e.g. `echo "ENVIRONMENT_URL=$CI_ENVIRONMENT_URL" >> review.env`. 1. Set the `.env` file to be a [job artifact](../../../ci/pipelines/job_artifacts.md#job-artifacts). 1. In the `load_performance` job: - 1. Set it to depend on the review job, so it inherits the env file. - 1. Set the `K6_DOCKER_OPTIONS` variable with the [Docker cli option for env files](https://docs.docker.com/engine/reference/commandline/run/#set-environment-variables--e---env---env-file), for example `--env-file review.env`. + 1. Set it to depend on the review job, so it inherits the environment file. + 1. Set the `K6_DOCKER_OPTIONS` variable with the [Docker CLI option for environment files](https://docs.docker.com/engine/reference/commandline/run/#set-environment-variables--e---env---env-file), for example `--env-file review.env`. 1. Configure the k6 test script to use the environment variable in it's steps. Your `.gitlab-ci.yml` file might be similar to: diff --git a/doc/user/project/merge_requests/merge_request_approvals.md b/doc/user/project/merge_requests/merge_request_approvals.md index 185ab0e6298..4b4c930c7af 100644 --- a/doc/user/project/merge_requests/merge_request_approvals.md +++ b/doc/user/project/merge_requests/merge_request_approvals.md @@ -5,13 +5,16 @@ info: "To determine the technical writer assigned to the Stage/Group associated type: reference, concepts --- -# Merge Request Approvals +# Merge Request Approvals **(CORE)** + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/580) in GitLab Enterprise Edition 7.2. Available in GitLab Core and higher tiers. +> - Redesign [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/1979) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.8 and [feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/10685) in 12.0. Code review is an essential practice of every successful project, and giving your approval once a merge request is in good shape is an important part of the review process, as it clearly communicates the ability to merge the change. -## Optional Approvals **(CORE ONLY)** +## Optional Approvals > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/27426) in GitLab 13.2. @@ -335,26 +338,6 @@ of your security team when a vulnerability would be introduced by a merge reques For more information, see [Security approvals in merge requests](../../application_security/index.md#security-approvals-in-merge-requests). -### Enabling the new approvals interface - -Since [GitLab v12.0](https://gitlab.com/gitlab-org/gitlab/-/issues/10685), an updated approvals -interface is available by default. In versions older than 12.0, the updated interface is not -available unless the `approval_rules` feature flag is enabled, which can be done from -the Rails console by instance administrators. - -Use these commands to start the Rails console: - -```shell -# Omnibus GitLab -gitlab-rails console - -# Installation from source -cd /home/git/gitlab -sudo -u git -H bin/rails console -e production -``` - -Then run `Feature.enable(:approval_rules)` to enable the updated interface. - <!-- ## Troubleshooting Include any troubleshooting steps that you can foresee. If you know beforehand what issues diff --git a/doc/user/project/merge_requests/revert_changes.md b/doc/user/project/merge_requests/revert_changes.md index bc4fee749e8..6b302b0ff02 100644 --- a/doc/user/project/merge_requests/revert_changes.md +++ b/doc/user/project/merge_requests/revert_changes.md @@ -14,7 +14,7 @@ by clicking the **Revert** button in merge requests and commit details. NOTE: **Note:** The **Revert** button will only be available for merge requests -created since GitLab 8.5. However, you can still revert a merge request +created in GitLab 8.5 and later. However, you can still revert a merge request by reverting the merge commit from the list of Commits page. NOTE: **Note:** diff --git a/doc/user/project/merge_requests/reviewing_and_managing_merge_requests.md b/doc/user/project/merge_requests/reviewing_and_managing_merge_requests.md index 3a18cacde64..aef68e0e771 100644 --- a/doc/user/project/merge_requests/reviewing_and_managing_merge_requests.md +++ b/doc/user/project/merge_requests/reviewing_and_managing_merge_requests.md @@ -84,7 +84,7 @@ Click **Expand file** on any file to view the changes for that file. > - For GitLab self-managed instances, GitLab administrators can opt to [disable it](#enable-or-disable-file-by-file-diff-navigation). For larger merge requests it might sometimes be useful to review single files at a time. To enable, -from your avatar on the top-right navbar, click **Settings**, and go to **Preferences** on the left +from your avatar on the top-right navigation bar, click **Settings**, and go to **Preferences** on the left sidebar. Scroll down to the **Behavior** section and select **Show one file at a time on merge request's Changes tab**. Click **Save changes** to apply. @@ -122,6 +122,8 @@ the commits to open the single-commit view. From there, you can navigate among t by clicking the **Prev** and **Next** buttons on the top-right of the page or by using the <kbd>X</kbd> and <kbd>C</kbd> keyboard shortcuts. + + ### Incrementally expand merge request diffs By default, the diff shows only the parts of a file which are changed. diff --git a/doc/user/project/merge_requests/squash_and_merge.md b/doc/user/project/merge_requests/squash_and_merge.md index 69a0dd6e84f..68f5478038a 100644 --- a/doc/user/project/merge_requests/squash_and_merge.md +++ b/doc/user/project/merge_requests/squash_and_merge.md @@ -65,8 +65,8 @@ meaningful commit messages and: ## Enabling squash for a merge request Anyone who can create or edit a merge request can choose for it to be squashed -on the merge request form. Users can select or unselect the checkbox at the moment -they are creating the merge request: +on the merge request form. Users can select or clear the check box when they +create the merge request:  diff --git a/doc/user/project/merge_requests/test_coverage_visualization.md b/doc/user/project/merge_requests/test_coverage_visualization.md index 56b5774f15b..bded1839f97 100644 --- a/doc/user/project/merge_requests/test_coverage_visualization.md +++ b/doc/user/project/merge_requests/test_coverage_visualization.md @@ -5,13 +5,10 @@ info: To determine the technical writer assigned to the Stage/Group associated w type: reference, howto --- -# Test Coverage Visualization **(CORE ONLY)** +# Test Coverage Visualization > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/3708) in GitLab 12.9. -> - [Feature flag enabled](https://gitlab.com/gitlab-org/gitlab/-/issues/211410) in GitLab 13.4. -> - It's enabled on GitLab.com. -> - It can be disabled per-project. -> - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enabling-the-feature). **(CORE ONLY)** +> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/249811) in GitLab 13.5. With the help of [GitLab CI/CD](../../../ci/README.md), you can collect the test coverage information of your favorite testing or coverage-analysis tool, and visualize @@ -58,7 +55,9 @@ NOTE: **Note:** The Cobertura XML parser currently does not support the `sources` element and ignores it. It is assumed that the `filename` of a `class` element contains the full path relative to the project root. -## Example test coverage configuration +## Example test coverage configurations + +### JavaScript example The following [`gitlab-ci.yml`](../../../ci/yaml/README.md) example uses [Mocha](https://mochajs.org/) JavaScript testing and [NYC](https://github.com/istanbuljs/nyc) coverage-tooling to @@ -74,26 +73,84 @@ test: cobertura: coverage/cobertura-coverage.xml ``` -## Enabling the feature +### Java examples + +#### Maven example -This feature comes with the `:coverage_report_view` feature flag enabled by -default. It is enabled on GitLab.com. [GitLab administrators with access to the GitLab Rails console](../../../administration/feature_flags.md) -can disable it for your instance. Test coverage visualization can be enabled or disabled per-project. +The following [`gitlab-ci.yml`](../../../ci/yaml/README.md) example for Java uses [Maven](https://maven.apache.org/) +to build the project and [Jacoco](https://www.eclemma.org/jacoco/) coverage-tooling to +generate the coverage artifact. +You can check the [Docker image configuration and scripts](https://gitlab.com/haynes/jacoco2cobertura) if you want to build your own image. -To enable it: +GitLab expects the artifact in the Cobertura format, so you have to execute a few +scripts before uploading it. The `test-jdk11` job tests the code and generates an +XML artifact. The `coverage-jdk-11` job converts the artifact into a Cobertura report: -```ruby -# Instance-wide -Feature.enable(:coverage_report_view) -# or by project -Feature.enable(:coverage_report_view, Project.find(<project id>)) +```yaml +test-jdk11: + stage: test + image: maven:3.6.3-jdk-11 + script: + - 'mvn $MAVEN_CLI_OPTS clean org.jacoco:jacoco-maven-plugin:prepare-agent test jacoco:report' + artifacts: + paths: + - target/site/jacoco/jacoco.xml + +coverage-jdk11: + # Must be in a stage later than test-jdk11's stage. + # The `visualize` stage does not exist by default. + # Please define it first, or chose an existing stage like `deploy`. + stage: visualize + image: haynes/jacoco2cobertura:1.0.3 + script: + # convert report from jacoco to cobertura + - 'python /opt/cover2cover.py target/site/jacoco/jacoco.xml src/main/java > target/site/cobertura.xml' + # read the <source></source> tag and prepend the path to every filename attribute + - 'python /opt/source2filename.py target/site/cobertura.xml' + needs: ["test-jdk11"] + dependencies: + - test-jdk11 + artifacts: + reports: + cobertura: target/site/cobertura.xml ``` -To disable it: +#### Gradle example -```ruby -# Instance-wide -Feature.disable(:coverage_report_view) -# or by project -Feature.disable(:coverage_report_view, Project.find(<project id>)) +The following [`gitlab-ci.yml`](../../../ci/yaml/README.md) example for Java uses [Gradle](https://gradle.org/) +to build the project and [Jacoco](https://www.eclemma.org/jacoco/) coverage-tooling to +generate the coverage artifact. +You can check the [Docker image configuration and scripts](https://gitlab.com/haynes/jacoco2cobertura) if you want to build your own image. + +GitLab expects the artifact in the Cobertura format, so you have to execute a few +scripts before uploading it. The `test-jdk11` job tests the code and generates an +XML artifact. The `coverage-jdk-11` job converts the artifact into a Cobertura report: + +```yaml +test-jdk11: + stage: test + image: gradle:6.6.1-jdk11 + script: + - 'gradle test jacocoTestReport' # jacoco must be configured to create an xml report + artifacts: + paths: + - build/jacoco/jacoco.xml + +coverage-jdk11: + # Must be in a stage later than test-jdk11's stage. + # The `visualize` stage does not exist by default. + # Please define it first, or chose an existing stage like `deploy`. + stage: visualize + image: haynes/jacoco2cobertura:1.0.3 + script: + # convert report from jacoco to cobertura + - 'python /opt/cover2cover.py build/jacoco/jacoco.xml src/main/java > build/cobertura.xml' + # read the <source></source> tag and prepend the path to every filename attribute + - 'python /opt/source2filename.py build/cobertura.xml' + needs: ["test-jdk11"] + dependencies: + - test-jdk11 + artifacts: + reports: + cobertura: build/cobertura.xml ``` diff --git a/doc/user/project/merge_requests/work_in_progress_merge_requests.md b/doc/user/project/merge_requests/work_in_progress_merge_requests.md index d1833318a85..e7abf6e5fb6 100644 --- a/doc/user/project/merge_requests/work_in_progress_merge_requests.md +++ b/doc/user/project/merge_requests/work_in_progress_merge_requests.md @@ -5,7 +5,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w type: reference, concepts --- -# Draft merge requests +# Draft merge requests **(CORE)** If a merge request is not yet ready to be merged, perhaps due to continued development or open threads, you can prevent it from being accepted before it's ready by flagging @@ -16,10 +16,12 @@ being merged, and it will stay disabled until the "Draft" flag has been removed. ## Adding the "Draft" flag to a merge request -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/32692) in GitLab 13.2, Work-In-Progress (WIP) merge requests were renamed to **Draft**. Support for using **WIP** will be removed in GitLab 14.0. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/32692) in GitLab 13.2, Work-In-Progress (WIP) merge requests were renamed to **Draft**. Support for using **WIP** will be removed in GitLab 14.0. +> - **Mark as draft** and **Mark as ready** buttons [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/227421) in GitLab 13.5. There are several ways to flag a merge request as a Draft: +- Click the **Mark as draft** button on the top-right corner of the merge request's page. - Add `[Draft]`, `Draft:` or `(Draft)` to the start of the merge request's title. Clicking on **Start the title with Draft:**, under the title box, when editing the merge request's description will have the same effect. @@ -28,15 +30,16 @@ There are several ways to flag a merge request as a Draft: - Add the `/wip` [quick action](../quick_actions.md#quick-actions-for-issues-merge-requests-and-epics) in a comment in the merge request. This is a toggle, and can be repeated to change the status back. Note that any other text in the comment will be discarded. -- Add `draft:` or `Draft:` to the start of a commit message targeting the merge request's - source branch. This is not a toggle, and doing it again in another commit will have - no effect. +- Add `draft:`, `Draft:`, `fixup!`, or `Fixup!` to the beginning of a commit message targeting the + merge request's source branch. This is not a toggle, and doing it again in another + commit will have no effect. ## Removing the "Draft" flag from a merge request Similar to above, when a Merge Request is ready to be merged, you can remove the `Draft` flag in several ways: +- Click the **Mark as ready** button on the top-right corner of the merge request's page. - Remove `[Draft]`, `Draft:` or `(Draft)` from the start of the merge request's title. Clicking on **Remove the Draft: prefix from the title**, under the title box, when editing the merge request's description, will have the same effect. diff --git a/doc/user/project/milestones/burndown_and_burnup_charts.md b/doc/user/project/milestones/burndown_and_burnup_charts.md new file mode 100644 index 00000000000..327a52a05ab --- /dev/null +++ b/doc/user/project/milestones/burndown_and_burnup_charts.md @@ -0,0 +1,142 @@ +--- +type: reference +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Burndown and burnup charts **(STARTER)** + +[Burndown](#burndown-charts) and [burnup](#burnup-charts) charts show the progress of completing a milestone. + + + +## Burndown charts + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/1540) in [GitLab Starter](https://about.gitlab.com/pricing/) 9.1 for project milestones. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/5354) in [GitLab Premium](https://about.gitlab.com/pricing/) 10.8 for group milestones. +> - [Added](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/6495) to [GitLab Starter](https://about.gitlab.com/pricing/) 11.2 for group milestones. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/6903) [fixed burndown charts](#fixed-burndown-charts) in [GitLab Starter](https://about.gitlab.com/pricing/) 13.5. + +Burndown charts show the number of issues over the course of a milestone. + + + +At a glance, you see the current state for the completion a given milestone. +Without them, you would have to organize the data from the milestone and plot it +yourself to have the same sense of progress. + +GitLab plots it for you and presents it in a clear and beautiful chart. + +<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> +For an overview, check the video demonstration on [Mapping work versus time with burndown charts](https://www.youtube.com/watch?v=zJU2MuRChzs). + +To view a project's burndown chart: + +1. In a project, navigate to **Issues > Milestones**. +1. Select a milestone from the list. + +To view a group's burndown chart: + +1. In a group, navigate to **Issues > Milestones**. +1. Select a milestone from the list. + +### Use cases for burndown charts + +Burndown charts are generally used for tracking and analyzing the completion of +a milestone. Therefore, their use cases are tied to the +[use you are assigning your milestone to](index.md). + +For example, suppose you lead a team of developers in a large company, +and you follow this workflow: + +- Your company set the goal for the quarter to deliver 10 new features for your app + in the upcoming major release. +- You create a milestone, and remind your team to assign that milestone to every new issue + and merge request that's part of the launch of your app. +- Every week, you open the milestone, visualize the progress, identify the gaps, + and help your team to get their work done. +- Every month, you check in with your supervisor, and show the progress of that milestone + from the burndown chart. +- By the end of the quarter, your team successfully delivered 100% of that milestone, as + it was taken care of closely throughout the whole quarter. + +### How burndown charts work + +A burndown chart is available for every project or group milestone that has been attributed a **start +date** and a **due date**. + +NOTE: **Note:** +You're able to [promote project](index.md#promoting-project-milestones-to-group-milestones) to group milestones and still see the **burndown chart** for them, respecting license limitations. + +The chart indicates the project's progress throughout that milestone (for issues assigned to it). + +In particular, it shows how many issues were or are still open for a given day in the +milestone's corresponding period. + +The burndown chart can also be toggled to display the cumulative open issue +weight for a given day. When using this feature, make sure issue weights have +been properly assigned, since an open issue with no weight adds zero to the +cumulative value. + +### Fixed burndown charts + +For milestones created before GitLab 13.5, burndown charts have an additional toggle to +switch between Legacy and Fixed views. + +| Legacy | Fixed | +| ----- | ----- | +|  |  | + +**Fixed burndown** charts track the full history of milestone activity, from its creation until the +milestone expires. After the milestone due date passes, issues removed from the milestone no longer +affect the chart. + +**Legacy burndown** charts track when issues were created and when they were last closed, not their +full history. For each day, a legacy burndown chart takes the number of open issues and the issues +created that day, and subtracts the number of issues closed that day. +Issues that were created and assigned a milestone before its start date (and remain open as of the +start date) are considered as having been opened on the start date. +Therefore, when the milestone start date is changed, the number of opened issues on each day may +change. +Reopened issues are considered as having been opened on the day after they were last closed. + +## Burnup charts + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/6903) in [GitLab Starter](https://about.gitlab.com/pricing/) 13.5. + +Burnup charts show the assigned and completed work for a milestone. + + + +To view a project's burnup chart: + +1. In a project, navigate to **Issues > Milestones**. +1. Select a milestone from the list. + +To view a group's burnup chart: + +1. In a group, navigate to **Issues > Milestones**. +1. Select a milestone from the list. + +### How burnup charts work + +Burnup charts have separate lines for total work and completed work. The total line +shows when scope is reduced or added to a milestone. The completed work is a count +of issues closed. + +Burnup charts can show either the total number of issues or total weight for each +day of the milestone. Use the toggle above the charts to switch between total +and weight. + +<!-- ## Troubleshooting + +Include any troubleshooting steps that you can foresee. If you know beforehand what issues +one might have when setting this up, or when something is changed, or on upgrading, it's +important to describe those, too. Think of things that may go wrong and include them here. +This is important to minimize requests for support, and to avoid doc comments with +questions that you know someone might ask. + +Each scenario can be a third-level heading, e.g. `### Getting error message X`. +If you have none to add when creating a doc, leave this section in place +but commented out to help encourage others to add to it in the future. --> diff --git a/doc/user/project/milestones/burndown_charts.md b/doc/user/project/milestones/burndown_charts.md index 0c8bba831a9..5aa5534dd37 100644 --- a/doc/user/project/milestones/burndown_charts.md +++ b/doc/user/project/milestones/burndown_charts.md @@ -1,89 +1,5 @@ --- -type: reference -stage: Plan -group: Project Management -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +redirect_to: './burndown_and_burnup_charts.md' --- -# Burndown Charts **(STARTER)** - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/1540) in [GitLab Starter](https://about.gitlab.com/pricing/) 9.1 for project milestones. -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/5354) in [GitLab Premium](https://about.gitlab.com/pricing/) 10.8 for group milestones. -> - [Added](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/6495) to [GitLab Starter](https://about.gitlab.com/pricing/) 11.2 for group milestones. -> - Closed or reopened issues prior to GitLab 9.1 won't have a `closed_at` -> value, so the burndown chart considers them as closed on the milestone -> `start_date`. In that case, a warning will be displayed. - -## Overview - -Burndown Charts are visual representations of the progress of completing a milestone. - - - -At a glance, you see the current state for the completion a given milestone. -Without them, you would have to organize the data from the milestone and plot it -yourself to have the same sense of progress. - -GitLab Starter plots it for you and presents it in a clear and beautiful chart. - -<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> -For an overview, check the video demonstration on [Mapping work versus time with Burndown Charts](https://www.youtube.com/watch?v=zJU2MuRChzs). - -## Use cases - -Burndown Charts are generally used for tracking and analyzing the completion of -a milestone. Therefore, their use cases are tied to the -[use you are assigning your milestone to](index.md). - -For example, suppose you lead a team of developers in a large company, -and you follow this workflow: - -- Your company set the goal for the quarter to deliver 10 new features for your app - in the upcoming major release. -- You create a milestone, and remind your team to assign that milestone to every new issue - and merge request that's part of the launch of your app. -- Every week, you open the milestone, visualize the progress, identify the gaps, - and help your team to get their work done. -- Every month, you check in with your supervisor, and show the progress of that milestone - from the Burndown Chart. -- By the end of the quarter, your team successfully delivered 100% of that milestone, as - it was taken care of closely throughout the whole quarter. - -## How it works - -A Burndown Chart is available for every project or group milestone that has been attributed a **start -date** and a **due date**. - -Find your project's **Burndown Chart** under **Project > Issues > Milestones**, -and select a milestone from your current ones, while for group's, access the **Groups** dashboard, -select a group, and go through **Issues > Milestones** on the sidebar. - -NOTE: **Note:** -You're able to [promote project](index.md#promoting-project-milestones-to-group-milestones) to group milestones and still see the **Burndown Chart** for them, respecting license limitations. - -The chart indicates the project's progress throughout that milestone (for issues assigned to it). - -In particular, it shows how many issues were or are still open for a given day in the -milestone's corresponding period. - -The Burndown Chart tracks when issues were created and when they were last closed—not their full history. For each day, it takes the number of issues still open and issues created that day and subtracts the number of issues closed that day. -**Issues that were created and assigned a milestone before its start date—and remain open as of the start date—are considered as having been opened on the start date**. Therefore, when the milestone start date is changed the number of opened issues on each day may change. -Reopened issues are -considered as having been opened on the day after they were last closed. - -The Burndown Chart can also be toggled to display the cumulative open issue -weight for a given day. When using this feature, make sure issue weights have -been properly assigned, since an open issue with no weight adds zero to the -cumulative value. - -<!-- ## Troubleshooting - -Include any troubleshooting steps that you can foresee. If you know beforehand what issues -one might have when setting this up, or when something is changed, or on upgrading, it's -important to describe those, too. Think of things that may go wrong and include them here. -This is important to minimize requests for support, and to avoid doc comments with -questions that you know someone might ask. - -Each scenario can be a third-level heading, e.g. `### Getting error message X`. -If you have none to add when creating a doc, leave this section in place -but commented out to help encourage others to add to it in the future. --> +This document was moved to [another location](./burndown_and_burnup_charts.md). diff --git a/doc/user/project/milestones/img/burndown_and_burnup_charts_v13_5.png b/doc/user/project/milestones/img/burndown_and_burnup_charts_v13_5.png Binary files differnew file mode 100644 index 00000000000..8d6ba1d4fa7 --- /dev/null +++ b/doc/user/project/milestones/img/burndown_and_burnup_charts_v13_5.png diff --git a/doc/user/project/milestones/img/burndown_chart_fixed_v13_5.png b/doc/user/project/milestones/img/burndown_chart_fixed_v13_5.png Binary files differnew file mode 100644 index 00000000000..a532bfeeca0 --- /dev/null +++ b/doc/user/project/milestones/img/burndown_chart_fixed_v13_5.png diff --git a/doc/user/project/milestones/img/burndown_chart_legacy_v13_5.png b/doc/user/project/milestones/img/burndown_chart_legacy_v13_5.png Binary files differnew file mode 100644 index 00000000000..5824fc59ce5 --- /dev/null +++ b/doc/user/project/milestones/img/burndown_chart_legacy_v13_5.png diff --git a/doc/user/project/milestones/img/burndown_chart.png b/doc/user/project/milestones/img/burndown_chart_v13_5.png Binary files differindex e06b24f9907..e06b24f9907 100644 --- a/doc/user/project/milestones/img/burndown_chart.png +++ b/doc/user/project/milestones/img/burndown_chart_v13_5.png diff --git a/doc/user/project/milestones/img/burnup_chart_v13_5.png b/doc/user/project/milestones/img/burnup_chart_v13_5.png Binary files differnew file mode 100644 index 00000000000..a850caba348 --- /dev/null +++ b/doc/user/project/milestones/img/burnup_chart_v13_5.png diff --git a/doc/user/project/milestones/index.md b/doc/user/project/milestones/index.md index 9d02a22f91e..8cbed3de1c6 100644 --- a/doc/user/project/milestones/index.md +++ b/doc/user/project/milestones/index.md @@ -7,8 +7,6 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Milestones -## Overview - Milestones in GitLab are a way to track issues and merge requests created to achieve a broader goal in a certain period of time. Milestones allow you to organize issues and merge requests into a cohesive group, with an optional start date and an optional due date. @@ -152,7 +150,7 @@ There are also tabs below these that show the following: For project milestones in [GitLab Starter](https://about.gitlab.com/pricing/), a [burndown chart](burndown_charts.md) is in the milestone view, showing the progress of completing a milestone. - + ### Group Burndown Charts **(STARTER)** diff --git a/doc/user/project/new_ci_build_permissions_model.md b/doc/user/project/new_ci_build_permissions_model.md index c3825371030..a7a72ca4d82 100644 --- a/doc/user/project/new_ci_build_permissions_model.md +++ b/doc/user/project/new_ci_build_permissions_model.md @@ -223,7 +223,7 @@ This is how an example usage can look like: ```yaml test: script: - - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - docker pull $CI_REGISTRY/group/other-project:latest - docker run $CI_REGISTRY/group/other-project:latest ``` @@ -236,5 +236,4 @@ to projects and their project permissions. ### API -GitLab API cannot be used via `CI_JOB_TOKEN` but there is a [proposal](https://gitlab.com/gitlab-org/gitlab/-/issues/35067) -to support it. +GitLab API can be used via `CI_JOB_TOKEN`, see [the relevant documentation](../../api/README.md#gitlab-ci-job-token). diff --git a/doc/user/project/pages/custom_domains_ssl_tls_certification/dns_concepts.md b/doc/user/project/pages/custom_domains_ssl_tls_certification/dns_concepts.md index 735d27ec04d..810538ab460 100644 --- a/doc/user/project/pages/custom_domains_ssl_tls_certification/dns_concepts.md +++ b/doc/user/project/pages/custom_domains_ssl_tls_certification/dns_concepts.md @@ -30,6 +30,8 @@ to do it for you. To help you out, we've gathered some instructions on how to do that for the most popular hosting services: +<!-- vale gitlab.Spelling = NO --> + - [Amazon](https://docs.aws.amazon.com/AmazonS3/latest/dev/website-hosting-custom-domain-walkthrough.html) - [Bluehost](https://www.bluehost.com/help/article/dns-management-add-edit-or-delete-dns-entries) - [Cloudflare](https://support.cloudflare.com/hc/en-us/articles/201720164-Creating-a-Cloudflare-account-and-adding-a-website) @@ -41,6 +43,8 @@ for the most popular hosting services: - [Media Temple](https://mediatemple.net/community/products/dv/204403794/how-can-i-change-the-dns-records-for-my-domain) - [Microsoft](https://docs.microsoft.com/en-us/previous-versions/windows/it-pro/windows-2000-server/bb727018(v=technet.10)) +<!-- vale gitlab.Spelling = YES --> + If your hosting service is not listed above, you can just try to search the web for `how to add dns record on <my hosting service>`. diff --git a/doc/user/project/pages/custom_domains_ssl_tls_certification/index.md b/doc/user/project/pages/custom_domains_ssl_tls_certification/index.md index badafa478ef..9b43dd58afe 100644 --- a/doc/user/project/pages/custom_domains_ssl_tls_certification/index.md +++ b/doc/user/project/pages/custom_domains_ssl_tls_certification/index.md @@ -61,7 +61,6 @@ according to the type of domain you want to use with your Pages site: - [For subdomains](#for-subdomains), `subdomain.example.com`. - [For both](#for-both-root-and-subdomains). -NOTE: **Note:** You can [configure IPv6 on self-managed instances](../../../../administration/pages/index.md#advanced-configuration), but IPv6 is not currently configured for Pages on GitLab.com. Follow [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/214718) for details. @@ -250,8 +249,8 @@ You can use any certificate satisfying the following requirements: - **A private key**, it's an encrypted key which validates your PEM against your domain. -NOTE: **Note:** -[Cloudflare certificates](https://about.gitlab.com/blog/2017/02/07/setting-up-gitlab-pages-with-cloudflare-certificates/), for example, meet these requirements. +For example, [Cloudflare certificates](https://about.gitlab.com/blog/2017/02/07/setting-up-gitlab-pages-with-cloudflare-certificates/) +meet these requirements. #### Steps @@ -269,7 +268,6 @@ NOTE: **Note:** just jumping a line between them. 1. Copy your private key and paste it in the last field. -NOTE: **Note:** **Do not** open certificates or encryption keys in regular text editors. Always use code editors (such as Sublime Text, Atom, Dreamweaver, Brackets, etc). @@ -290,8 +288,8 @@ To enable this setting: 1. Navigate to your project's **Settings > Pages**. 1. Tick the checkbox **Force HTTPS (requires valid certificates)**. -NOTE: **Note:** -If you use Cloudflare CDN in front of GitLab Pages, make sure to set the SSL connection setting to `full` instead of `flexible`. For more details, see the [Cloudflare CDN directions](https://support.cloudflare.com/hc/en-us/articles/200170416-End-to-end-HTTPS-with-Cloudflare-Part-3-SSL-options#h_4e0d1a7c-eb71-4204-9e22-9d3ef9ef7fef). +If you use Cloudflare CDN in front of GitLab Pages, make sure to set the SSL connection setting to +`full` instead of `flexible`. For more details, see the [Cloudflare CDN directions](https://support.cloudflare.com/hc/en-us/articles/200170416-End-to-end-HTTPS-with-Cloudflare-Part-3-SSL-options#h_4e0d1a7c-eb71-4204-9e22-9d3ef9ef7fef). <!-- ## Troubleshooting diff --git a/doc/user/project/pages/custom_domains_ssl_tls_certification/lets_encrypt_integration.md b/doc/user/project/pages/custom_domains_ssl_tls_certification/lets_encrypt_integration.md index 4b4b430b663..24b202dfdbd 100644 --- a/doc/user/project/pages/custom_domains_ssl_tls_certification/lets_encrypt_integration.md +++ b/doc/user/project/pages/custom_domains_ssl_tls_certification/lets_encrypt_integration.md @@ -25,7 +25,7 @@ This feature covers only certificates for **custom domains**, not the wildcard c Before you can enable automatic provisioning of an SSL certificate for your domain, make sure you have: -- Created a [project](../getting_started_part_two.md) in GitLab +- Created a [project](../index.md#getting-started) in GitLab containing your website's source code. - Acquired a domain (`example.com`) and added a [DNS entry](index.md) pointing it to your Pages website. @@ -33,7 +33,6 @@ Before you can enable automatic provisioning of an SSL certificate for your doma and verified your ownership. - Verified your website is up and running, accessible through your custom domain. -NOTE: **Note:** GitLab's Let's Encrypt integration is enabled and available on GitLab.com. For **self-managed** GitLab instances, make sure your administrator has [enabled it](../../../../administration/pages/index.md#lets-encrypt-integration). diff --git a/doc/user/project/pages/getting_started/new_or_existing_website.md b/doc/user/project/pages/getting_started/new_or_existing_website.md index 86f36447b93..f19334a1764 100644 --- a/doc/user/project/pages/getting_started/new_or_existing_website.md +++ b/doc/user/project/pages/getting_started/new_or_existing_website.md @@ -2,4 +2,4 @@ redirect_to: 'pages_ci_cd_template.md' --- -This document was moved to [pages_ci_cd_template.md](pages_ci_cd_template.md). +This document was moved to [another location](pages_ci_cd_template.md). diff --git a/doc/user/project/pages/getting_started/pages_forked_sample_project.md b/doc/user/project/pages/getting_started/pages_forked_sample_project.md index de9bd97b262..7dc3d2197b5 100644 --- a/doc/user/project/pages/getting_started/pages_forked_sample_project.md +++ b/doc/user/project/pages/getting_started/pages_forked_sample_project.md @@ -53,4 +53,4 @@ You can take some **optional** further steps:  - Now go to your SSG's configuration file and change the [base URL](../getting_started_part_one.md#urls-and-baseurls) - from `"project-name"` to `""`. The project name setting varies by SSG and may not be in the config file. + from `"project-name"` to `""`. The project name setting varies by SSG and may not be in the configuration file. diff --git a/doc/user/project/pages/getting_started_part_one.md b/doc/user/project/pages/getting_started_part_one.md index 9272b1f9093..5ef0c4cc7b9 100644 --- a/doc/user/project/pages/getting_started_part_one.md +++ b/doc/user/project/pages/getting_started_part_one.md @@ -11,12 +11,9 @@ according to your intended website's URL. ## GitLab Pages default domain names -NOTE: **Note:** -If you use your own GitLab instance to deploy your -site with GitLab Pages, check with your sysadmin what's your -Pages wildcard domain. This guide is valid for any GitLab instance, -you just need to replace Pages wildcard domain on GitLab.com -(`*.gitlab.io`) with your own. +If you use your own GitLab instance to deploy your site with GitLab Pages, verify your Pages +wildcard domain with your sysadmin. This guide is valid for any GitLab instance, provided that you +replace the Pages wildcard domain on GitLab.com (`*.gitlab.io`) with your own. If you set up a GitLab Pages project on GitLab, it will automatically be accessible under a diff --git a/doc/user/project/pages/getting_started_part_three.md b/doc/user/project/pages/getting_started_part_three.md index 9bc9fe97fd3..4bf5300aa13 100644 --- a/doc/user/project/pages/getting_started_part_three.md +++ b/doc/user/project/pages/getting_started_part_three.md @@ -1 +1,5 @@ +--- +redirect_to: 'custom_domains_ssl_tls_certification/index.md' +--- + This document was moved to [another location](custom_domains_ssl_tls_certification/index.md). diff --git a/doc/user/project/pages/index.md b/doc/user/project/pages/index.md index 6c3b911d033..4f389716f08 100644 --- a/doc/user/project/pages/index.md +++ b/doc/user/project/pages/index.md @@ -124,3 +124,24 @@ If you are running a self-managed instance of GitLab (GitLab Community Edition a [follow the administration steps](../../../administration/pages/index.md) to configure Pages. <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> Watch a [video tutorial](https://www.youtube.com/watch?v=dD8c7WNcc6s) about how to get started with GitLab Pages administration. + +## Security for GitLab Pages + +If your username is `foo`, your GitLab Pages website is located at `foo.gitlab.io`. +GitLab allows usernames to contain a `.`, so a user named `bar.foo` could create +a GitLab Pages website `bar.foo.gitlab.io` that effectively is a subdomain of your +`foo.gitlab.io` website. Be careful if you use JavaScript to set cookies for your website. +The safe way to manually set cookies with JavaScript is to not specify the `domain` at all: + +```javascript +// Safe: This cookie is only visible to foo.gitlab.io +document.cookie = "key=value"; + +// Unsafe: This cookie is visible to foo.gitlab.io and its subdomains, +// regardless of the presence of the leading dot. +document.cookie = "key=value;domain=.foo.gitlab.io"; +document.cookie = "key=value;domain=foo.gitlab.io"; +``` + +This issue doesn't affect users with a custom domain, or users who don't set any +cookies manually with JavaScript. diff --git a/doc/user/project/pages/introduction.md b/doc/user/project/pages/introduction.md index cea6bab1a50..b97d5328c07 100644 --- a/doc/user/project/pages/introduction.md +++ b/doc/user/project/pages/introduction.md @@ -173,7 +173,7 @@ Most modern browsers support downloading files in a compressed format. This speeds up downloads by reducing the size of files. Before serving an uncompressed file, Pages will check whether the same file -exists with a `.gz` extension. If it does, and the browser supports receiving +exists with a `.br` or `.gz` extension. If it does, and the browser supports receiving compressed files, it will serve that version instead of the uncompressed one. To take advantage of this feature, the artifact you upload to the Pages should @@ -182,14 +182,17 @@ have this structure: ```plaintext public/ ├─┬ index.html +│ | index.html.br │ └ index.html.gz │ ├── css/ │ └─┬ main.css +│ | main.css.br │ └ main.css.gz │ └── js/ └─┬ main.js + | main.js.br └ main.js.gz ``` @@ -202,6 +205,7 @@ pages: script: # Build the public/ directory first - find public -type f -regex '.*\.\(htm\|html\|txt\|text\|js\|css\)$' -exec gzip -f -k {} \; + - find public -type f -regex '.*\.\(htm\|html\|txt\|text\|js\|css\)$' -exec brotli -f -k {} \; ``` By pre-compressing the files and including both versions in the artifact, Pages @@ -255,9 +259,8 @@ instead. Here are some examples of what will happen given the above Pages site: | `/other/index` | `200 OK` | `public/other/index.html` | | `/other/index.html` | `200 OK` | `public/other/index.html` | -NOTE: **Note:** -When `public/data/index.html` exists, it takes priority over the `public/data.html` -file for both the `/data` and `/data/` URL paths. +Note that when `public/data/index.html` exists, it takes priority over the `public/data.html` file +for both the `/data` and `/data/` URL paths. ## Frequently Asked Questions diff --git a/doc/user/project/pages/lets_encrypt_for_gitlab_pages.md b/doc/user/project/pages/lets_encrypt_for_gitlab_pages.md index 708d886b352..02d1dd7898a 100644 --- a/doc/user/project/pages/lets_encrypt_for_gitlab_pages.md +++ b/doc/user/project/pages/lets_encrypt_for_gitlab_pages.md @@ -33,9 +33,8 @@ To follow along with this tutorial, we assume you already have: Once you have the requirements addressed, follow the instructions below to learn how to obtain the certificate. -NOTE: **Note:** -The instructions below were tested on macOS Mojave. For other -operating systems the steps might be slightly different. Follow the +Note that these instructions were tested on macOS Mojave. For other operating systems the steps +might be slightly different. Follow the [CertBot instructions](https://certbot.eff.org/) according to your OS. 1. On your computer, open a terminal and navigate to your repository's diff --git a/doc/user/project/pages/pages_access_control.md b/doc/user/project/pages/pages_access_control.md index b6b881b961e..b3705a5835a 100644 --- a/doc/user/project/pages/pages_access_control.md +++ b/doc/user/project/pages/pages_access_control.md @@ -20,11 +20,9 @@ on your GitLab instance. When enabled, only For a demonstration, see [Pages access controls](https://www.youtube.com/watch?v=tSPAr5mQYc8). 1. Navigate to your project's **Settings > General** and expand **Visibility, project features, permissions**. -1. Toggle the **Pages** button to enable the access control. - NOTE: **Note:** - If you don't see the toggle button, that means that it's not enabled. - Ask your administrator to [enable it](../../../administration/pages/index.md#access-control). +1. Toggle the **Pages** button to enable the access control. If you don't see the toggle button, + that means it isn't enabled. Ask your administrator to [enable it](../../../administration/pages/index.md#access-control). 1. The Pages access control dropdown allows you to set who can view pages hosted with GitLab Pages, depending on your project's visibility: @@ -48,9 +46,10 @@ can access the website. ## Terminating a Pages session -If you want to log out from your Pages website, -you can do so by revoking application access token for GitLab Pages: +To sign out of your GitLab Pages website, revoke the application access token +for GitLab Pages: -1. Navigate to your profile's **Settings > Applications**. -1. Find **Authorized applications** at the bottom of the page. -1. Find **GitLab Pages** and press the **Revoke** button. +1. In the top menu, select your profile, and then select **Settings**. +1. In the left sidebar, select **Applications**. +1. Scroll to the **Authorized applications** section, find the **GitLab Pages** + entry, and select its **Revoke** button. diff --git a/doc/user/project/pages/redirects.md b/doc/user/project/pages/redirects.md index ae7b1b4fa6e..60fbf368061 100644 --- a/doc/user/project/pages/redirects.md +++ b/doc/user/project/pages/redirects.md @@ -6,15 +6,15 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Create redirects for GitLab Pages -> - [Introduced](https://gitlab.com/gitlab-org/gitlab-pages/-/issues/24) in GitLab Pages 1.25.0 and GitLab 13.4. -> - It's [deployed behind a feature flag](#enable-or-disable-redirects), disabled by default. -> - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enable-or-disable-redirects). +> - [Introduced](https://gitlab.com/gitlab-org/gitlab-pages/-/issues/24) in GitLab Pages 1.25.0 and GitLab 13.4 behind a feature flag, disabled by default. +> - [Became enabled by default](https://gitlab.com/gitlab-org/gitlab-pages/-/merge_requests/367) in GitLab 13.5. CAUTION: **Warning:** This feature might not be available to you. Check the **version history** note above for details. -In GitLab Pages, you can [enable](#enable-or-disable-redirects) the redirects feature to configure rules to forward one URL to another using HTTP redirects. GitLab Pages uses -[Netlify style redirects](https://docs.netlify.com/routing/redirects/#syntax-for-the-redirects-file). +In GitLab Pages, you can configure rules to forward one URL to another using +[Netlify style](https://docs.netlify.com/routing/redirects/#syntax-for-the-redirects-file) +HTTP redirects. ## Supported features @@ -22,8 +22,10 @@ GitLab Pages only supports the [`_redirects` plain text file syntax](https://docs.netlify.com/routing/redirects/#syntax-for-the-redirects-file), and `.toml` files are not supported. -Redirects are only supported at a basic level, and GitLab Pages doesn't support all -[special options offered by Netlify](https://docs.netlify.com/routing/redirects/redirect-options/): +Redirects are only supported at a basic level. GitLab Pages doesn't support all +[special options offered by Netlify](https://docs.netlify.com/routing/redirects/redirect-options/). + +Note that supported paths must start with a forward slash `/`. | Feature | Supported | Example | | ------- | --------- | ------- | @@ -37,12 +39,9 @@ Redirects are only supported at a basic level, and GitLab Pages doesn't support | Redirect by country or language | **{dotted-circle}** No | `/ /anz 302 Country=au,nz` | | Redirect by role | **{dotted-circle}** No | `/admin/* 200! Role=admin` | -NOTE: **Note:** -Supported paths must start with a forward slash `/`. - ## Create redirects -To create redirects after [enabling](#enable-or-disable-redirects) the feature, +To create redirects, create a configuration file named `_redirects` in the `public/` directory of your GitLab Pages site. @@ -78,8 +77,7 @@ is ignored because `hello.html` exists: /projectname/hello.html /projectname/world.html 302 ``` -NOTE: **Note:** -GitLab does not support Netlify's +GitLab doesn't support Netlify's [force option](https://docs.netlify.com/routing/redirects/rewrites-proxies/#shadowing) to change this behavior. @@ -105,19 +103,19 @@ rule 10: valid rule 11: valid ``` -## Enable or disable redirects +## Disable redirects -Redirects in GitLab Pages is under development and not ready for production use. It is -deployed behind a feature flag that is **disabled by default**. +Redirects in GitLab Pages is under development, and is deployed behind a feature flag +that is **enabled by default**. -For [Omnibus installations](../../../administration/pages/index.md), define the +To disable redirects, for [Omnibus installations](../../../administration/pages/index.md), define the `FF_ENABLE_REDIRECTS` environment variable in the [global settings](../../../administration/pages/index.md#global-settings). Add the following line to `/etc/gitlab/gitlab.rb` and [reconfigure the instance](../../../administration/restart_gitlab.md#omnibus-gitlab-reconfigure). ```ruby -gitlab_pages['env']['FF_ENABLE_REDIRECTS'] = 'true' +gitlab_pages['env']['FF_ENABLE_REDIRECTS'] = 'false' ``` For [source installations](../../../administration/pages/source.md), define the @@ -125,6 +123,6 @@ For [source installations](../../../administration/pages/source.md), define the [restart GitLab](../../../administration/restart_gitlab.md#installations-from-source): ```shell -export FF_ENABLE_REDIRECTS="true" +export FF_ENABLE_REDIRECTS="false" /path/to/pages/bin/gitlab-pages -config gitlab-pages.conf ``` diff --git a/doc/user/project/protected_branches.md b/doc/user/project/protected_branches.md index 3d0cb1bf3a5..7265fd330e3 100644 --- a/doc/user/project/protected_branches.md +++ b/doc/user/project/protected_branches.md @@ -48,7 +48,7 @@ that the `master` branch is protected by default. > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/5081) in GitLab 8.11. -Since GitLab 8.11, we added another layer of branch protection which provides +In GitLab 8.11 and later, we added another layer of branch protection which provides more granular management of protected branches. The "Developers can push" option was replaced by an "Allowed to push" setting which can be set to allow/prohibit Maintainers and/or Developers to push to a protected branch. @@ -185,6 +185,8 @@ When enabled, all merge requests targeting these branches will require approval by a Code Owner per matched rule before they can be merged. Additionally, direct pushes to the protected branch are denied if a rule is matched. +[Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/35097) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.5, users and groups who are allowed to push to protected branches do not require a merge request to merge their feature branches. Thus, they can skip merge request approval rules. + ## Running pipelines on protected branches The permission to merge or push to protected branches is used to define if a user can diff --git a/doc/user/project/quick_actions.md b/doc/user/project/quick_actions.md index 518cf472b50..2f1b05f481e 100644 --- a/doc/user/project/quick_actions.md +++ b/doc/user/project/quick_actions.md @@ -39,22 +39,22 @@ The following quick actions are applicable to descriptions, discussions and thre | `/copy_metadata <!merge_request>` | ✓ | ✓ | | Copy labels and milestone from another merge request in the project. | | `/copy_metadata <#issue>` | ✓ | ✓ | | Copy labels and milestone from another issue in the project. | | `/create_merge_request <branch name>` | ✓ | | | Create a new merge request starting from the current issue. | -| `/done` | ✓ | ✓ | ✓ | Mark To-Do as done. | +| `/done` | ✓ | ✓ | ✓ | Mark to do as done. | | `/due <date>` | ✓ | | | Set due date. Examples of valid `<date>` include `in 2 days`, `this Friday` and `December 31st`. | | `/duplicate <#issue>` | ✓ | | | Close this issue and mark as a duplicate of another issue. **(CORE)** Also, mark both as related. **(STARTER)** | | `/epic <epic>` | ✓ | | | Add to epic `<epic>`. The `<epic>` value should be in the format of `&epic`, `group&epic`, or a URL to an epic. **(PREMIUM)** | | `/estimate <<W>w <DD>d <hh>h <mm>m>` | ✓ | ✓ | | Set time estimate. For example, `/estimate 1w 3d 2h 14m`. | | `/iteration *iteration:"iteration name"` | ✓ | | | Set iteration. For example, to set the `Late in July` iteration: `/iteration *iteration:"Late in July"` ([introduced in GitLab 13.1](https://gitlab.com/gitlab-org/gitlab/-/issues/196795)). **(STARTER)** | | `/label ~label1 ~label2` | ✓ | ✓ | ✓ | Add one or more labels. Label names can also start without a tilde (`~`), but mixed syntax is not supported. | -| `/lock` | ✓ | ✓ | | Lock the thread. | +| `/lock` | ✓ | ✓ | | Lock the discussions. | | `/merge` | | ✓ | | Merge changes. Depending on the project setting, this may be [when the pipeline succeeds](merge_requests/merge_when_pipeline_succeeds.md), adding to a [Merge Train](../../ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md), etc. | | `/milestone %milestone` | ✓ | ✓ | | Set milestone. | | `/move <path/to/project>` | ✓ | | | Move this issue to another project. | | `/parent_epic <epic>` | | | ✓ | Set parent epic to `<epic>`. The `<epic>` value should be in the format of `&epic`, `group&epic`, or a URL to an epic ([introduced in GitLab 12.1](https://gitlab.com/gitlab-org/gitlab/-/issues/10556)). **(ULTIMATE)** | | `/promote` | ✓ | | | Promote issue to epic. **(PREMIUM)** | | `/publish` | ✓ | | | Publish issue to an associated [Status Page](../../operations/incident_management/status_page.md) ([Introduced in GitLab 13.0](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/30906)) **(ULTIMATE)** | -| `/reassign @user1 @user2` | ✓ | ✓ | | Change assignee. **(STARTER)** | -| `/relabel ~label1 ~label2` | ✓ | ✓ | ✓ | Replace existing labels with those specified. | +| `/reassign @user1 @user2` | ✓ | ✓ | | Replace current assignees with those specified. **(STARTER)** | +| `/relabel ~label1 ~label2` | ✓ | ✓ | ✓ | Replace current labels with those specified. | | `/relate #issue1 #issue2` | ✓ | | | Mark issues as related. **(STARTER)** | | `/remove_child_epic <epic>` | | | ✓ | Remove child epic from `<epic>`. The `<epic>` value should be in the format of `&epic`, `group&epic`, or a URL to an epic ([introduced in GitLab 12.0](https://gitlab.com/gitlab-org/gitlab/-/issues/7330)). **(ULTIMATE)** | | `/remove_due_date` | ✓ | | | Remove due date. | @@ -74,11 +74,12 @@ The following quick actions are applicable to descriptions, discussions and thre | `/tableflip <comment>` | ✓ | ✓ | ✓ | Append the comment with `(╯°□°)╯︵ ┻━┻`. | | `/target_branch <local branch name>` | | ✓ | | Set target branch. | | `/title <new title>` | ✓ | ✓ | ✓ | Change title. | -| `/todo` | ✓ | ✓ | ✓ | Add a To-Do. | +| `/todo` | ✓ | ✓ | ✓ | Add a to do. | | `/unassign @user1 @user2` | ✓ | ✓ | | Remove specific assignees. **(STARTER)** | | `/unassign` | ✓ | ✓ | | Remove all assignees. | -| `/unlabel ~label1 ~label2` or `/remove_label ~label1 ~label2` | ✓ | ✓ | ✓ | Remove all or specific labels. | -| `/unlock` | ✓ | ✓ | | Unlock the thread. | +| `/unlabel ~label1 ~label2` or `/remove_label ~label1 ~label2` | ✓ | ✓ | ✓ | Remove specified labels. | +| `/unlabel` or `/remove_label` | ✓ | ✓ | ✓ | Remove all labels. | +| `/unlock` | ✓ | ✓ | | Unlock the discussions. | | `/unsubscribe` | ✓ | ✓ | ✓ | Unsubscribe from notifications. | | `/weight <value>` | ✓ | | | Set weight. Valid options for `<value>` include `0`, `1`, `2`, and so on. **(STARTER)** | | `/wip` | | ✓ | | Toggle the Work In Progress status. | diff --git a/doc/user/project/releases/index.md b/doc/user/project/releases/index.md index 6c8aacd12b3..962d612cac1 100644 --- a/doc/user/project/releases/index.md +++ b/doc/user/project/releases/index.md @@ -47,22 +47,20 @@ To view a list of releases: > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/32812) in GitLab 12.9. Releases can be created directly in the GitLab UI. -NOTE: **Note:** -Only users with Developer permissions or higher can create releases. -Read more about [Release permissions](../../../user/permissions.md#project-members-permissions). - You can create a release in the user interface, or by using the [Releases API](../../../api/releases/index.md#create-a-release). We recommend using the API to create releases as one of the last steps in your CI/CD pipeline. +Only users with Developer permissions or higher can create releases. +Read more about [Release permissions](../../../user/permissions.md#project-members-permissions). + To create a new release through the GitLab UI: 1. Navigate to **Project overview > Releases** and click the **New release** button. 1. In the [**Tag name**](#tag-name) box, enter a name. - NOTE: **Note:** Creating a release based on an existing tag using the user interface is not yet supported. However, this is possible using the [Releases API](../../../api/releases/index.md#create-a-release). @@ -88,7 +86,6 @@ release tag. When the `released_at` date and time has passed, the badge is autom > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/26016) in GitLab 12.6. Asset link editing was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9427) in GitLab 12.10. -NOTE: **Note:** Only users with Developer permissions or higher can edit releases. Read more about [Release permissions](../../../user/permissions.md#project-members-permissions). @@ -225,7 +222,6 @@ The release title can be customized using the **Release title** field when creating or editing a release. If no title is provided, the release's tag name is used instead. -NOTE: **Note:** Guest users of private projects are allowed to view the **Releases** page but are _not_ allowed to view details about the Git repository (in particular, tag names). Because of this, release titles are replaced with a generic @@ -254,7 +250,6 @@ Every release has a description. You can add any text you like, but we recommend including a changelog to describe the content of your release. This helps users quickly scan the differences between each release you publish. -NOTE: **Note:** [Git's tagging messages](https://git-scm.com/book/en/v2/Git-Basics-Tagging) and Release note descriptions are unrelated. Description supports [Markdown](../../markdown.md). @@ -334,8 +329,7 @@ generate release evidence for an existing release. Because of this, each release can have multiple release evidence snapshots. You can view the release evidence and its details on the Releases page. -NOTE: **Note:** -When the issue tracker is disabled, release evidence [cannot be downloaded](https://gitlab.com/gitlab-org/gitlab/-/issues/208397). +When the issue tracker is disabled, release evidence [can't be downloaded](https://gitlab.com/gitlab-org/gitlab/-/issues/208397). Here is an example of a release evidence object: @@ -431,10 +425,13 @@ ruby: junit: rspec.xml ``` -If the pipeline ran successfully, when you create your release, the `rspec.xml` file is saved as release evidence. +If the pipeline ran successfully, when you create your release, the `rspec.xml` file is saved as +release evidence. -NOTE: **Note:** -If you [schedule release evidence collection](#schedule-release-evidence-collection), some artifacts may already be expired by the time of evidence collection. To avoid this you can use the [`artifacts:expire_in`](../../../ci/yaml/README.md#artifactsexpire_in) keyword. Learn more in [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/222351). +If you [schedule release evidence collection](#schedule-release-evidence-collection), +some artifacts may already be expired by the time of evidence collection. To avoid this you can use +the [`artifacts:expire_in`](../../../ci/yaml/README.md#artifactsexpire_in) +keyword. Learn more in [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/222351). ### Schedule release evidence collection @@ -449,21 +446,6 @@ In the API: - If you do not specify a `released_at` date, release evidence is collected on the date the release is created. -### Disable release evidence display **(CORE ONLY)** - -The `:release_evidence_collection` feature flag is enabled by default in GitLab -self-managed instances. To turn it off, ask a GitLab administrator with Rails console -access to run the following command: - -```ruby -Feature.disable(:release_evidence_collection) -``` - -NOTE: **Note:** -Release evidence is collected regardless of this feature flag, -which only enables or disables the display of the data on the -Releases page. - ## GitLab Releaser > [Introduced](https://gitlab.com/gitlab-org/gitlab-releaser/-/merge_requests/6) in GitLab 12.10. diff --git a/doc/user/project/repository/forking_workflow.md b/doc/user/project/repository/forking_workflow.md index e90f0a7354c..b0aa7569579 100644 --- a/doc/user/project/repository/forking_workflow.md +++ b/doc/user/project/repository/forking_workflow.md @@ -14,7 +14,7 @@ can create a fork. A fork is a personal copy of the repository and all its branches, which you create in a namespace of your choice. This way you can make changes in your own fork and -submit them through a merge request to the repo you don't have access to. +submit them through a merge request to the repository you don't have access to. ## Creating a fork diff --git a/doc/user/project/repository/img/repository_mirroring_push_settings.png b/doc/user/project/repository/img/repository_mirroring_push_settings.png Binary files differindex d055cc580c4..9fc25dd3b25 100644 --- a/doc/user/project/repository/img/repository_mirroring_push_settings.png +++ b/doc/user/project/repository/img/repository_mirroring_push_settings.png diff --git a/doc/user/project/repository/index.md b/doc/user/project/repository/index.md index 536cae263b8..5473439a162 100644 --- a/doc/user/project/repository/index.md +++ b/doc/user/project/repository/index.md @@ -86,7 +86,7 @@ according to the markup language. | [reStructuredText](https://docutils.sourceforge.io/rst.html) | `rst` | | [AsciiDoc](../../asciidoc.md) | `adoc`, `ad`, `asciidoc` | | [Textile](https://textile-lang.com/) | `textile` | -| [rdoc](http://rdoc.sourceforge.net/doc/index.html) | `rdoc` | +| [Rdoc](http://rdoc.sourceforge.net/doc/index.html) | `rdoc` | | [Org mode](https://orgmode.org/) | `org` | | [creole](http://www.wikicreole.org/) | `creole` | | [MediaWiki](https://www.mediawiki.org/wiki/MediaWiki) | `wiki`, `mediawiki` | @@ -234,7 +234,7 @@ lock your files to prevent any conflicting changes. ## Repository's API -You can access your repos via [repository API](../../../api/repositories.md). +You can access your repositories via [repository API](../../../api/repositories.md). ## Clone in Apple Xcode diff --git a/doc/user/project/repository/reducing_the_repo_size_using_git.md b/doc/user/project/repository/reducing_the_repo_size_using_git.md index 28fdda07b05..ad79fd8a8f9 100644 --- a/doc/user/project/repository/reducing_the_repo_size_using_git.md +++ b/doc/user/project/repository/reducing_the_repo_size_using_git.md @@ -19,7 +19,7 @@ over [`git filter-branch`](https://git-scm.com/docs/git-filter-branch) and [BFG](https://rtyley.github.io/bfg-repo-cleaner/). DANGER: **Danger:** -Rewriting repository history is a destructive operation. Make sure to backup your repository before +Rewriting repository history is a destructive operation. Make sure to back up your repository before you begin. The best way back up a repository is to [export the project](../settings/import_export.md#exporting-a-project-and-its-data). @@ -230,6 +230,7 @@ This will: - Run `git gc` against the repository to remove unreferenced objects. Repacking your repository will temporarily cause the size of your repository to increase significantly, because the old pack files are not removed until the new pack files have been created. +- Unlink any unused LFS objects currently attached to your project, freeing up storage space. - Recalculate the size of your repository on disk. You will receive an email notification with the recalculated repository size after the cleanup has completed. @@ -266,21 +267,20 @@ You can still: - Create new issues. - Clone the project. -If you exceed the repository size limit, you might try to: +If you exceed the repository size limit, you can: 1. Remove some data. 1. Make a new commit. 1. Push back to the repository. -Perhaps you might also: +If these actions are insufficient, you can also: - Move some blobs to LFS. - Remove some old dependency updates from history. -Unfortunately, this workflow won't work. Deleting files in a commit doesn't actually reduce the size -of the repository because the earlier commits and blobs still exist. - -What you need to do is rewrite history. We recommend the open-source community-maintained tool +Unfortunately, this workflow doesn't work. Deleting files in a commit doesn't actually reduce the +size of the repository, because the earlier commits and blobs still exist. Instead, you must rewrite +history. We recommend the open-source community-maintained tool [`git filter-repo`](https://github.com/newren/git-filter-repo). NOTE: **Note:** diff --git a/doc/user/project/repository/repository_mirroring.md b/doc/user/project/repository/repository_mirroring.md index e1d2c20850b..188699e0c77 100644 --- a/doc/user/project/repository/repository_mirroring.md +++ b/doc/user/project/repository/repository_mirroring.md @@ -56,6 +56,7 @@ The following are some possible use cases for repository mirroring: > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/249) in GitLab Enterprise Edition 8.7. > - [Moved to GitLab Core](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/18715) in 10.8. +> - [LFS support over HTTPS added](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/40137) in 13.5 For an existing project, you can set up push mirroring as follows: @@ -181,7 +182,7 @@ To set up a mirror from GitLab to AWS CodeCommit: not confuse it with the IAM user ID or AWS keys of this user. 1. Copy or download special Git HTTPS user ID and password. -1. In the AWS CodeCommit console, create a new repository to mirror from your GitLab repo. +1. In the AWS CodeCommit console, create a new repository to mirror from your GitLab repository. 1. Open your new repository and click **Clone URL > Clone HTTPS** (not **Clone HTTPS (GRC)**). 1. In GitLab, open the repository to be push-mirrored. 1. Click **Settings > Repository** and expand **Mirroring repositories**. @@ -192,7 +193,7 @@ To set up a mirror from GitLab to AWS CodeCommit: ``` Replace `<your_aws_git_userid>` with the AWS **special HTTPS Git user ID** from the IAM Git - credentials created earlier. Replace `<your_codecommit_repo>` with the name of your repo in CodeCommit. + credentials created earlier. Replace `<your_codecommit_repo>` with the name of your repository in CodeCommit. 1. For **Mirror direction**, select **Push**. 1. For **Authentication method**, select **Password** and fill in the **Password** field with the special IAM Git clone user ID **password** created earlier in AWS. diff --git a/doc/user/project/repository/x509_signed_commits/index.md b/doc/user/project/repository/x509_signed_commits/index.md index 972a46ee7a3..9b420d84f50 100644 --- a/doc/user/project/repository/x509_signed_commits/index.md +++ b/doc/user/project/repository/x509_signed_commits/index.md @@ -26,7 +26,7 @@ For a commit or tag to be *verified* by GitLab: [Omnibus install custom public certificates](https://docs.gitlab.com/omnibus/settings/ssl.html#install-custom-public-certificates). - The signing time has to be within the time range of the [certificate validity](https://www.rfc-editor.org/rfc/rfc5280.html#section-4.1.2.5) which is usually up to three years. -- The signing time is equal or later then commit time. +- The signing time is equal or later than commit time. NOTE: **Note:** Certificate revocation lists are checked on a daily basis via background worker. diff --git a/doc/user/project/requirements/img/requirement_create_v13_5.png b/doc/user/project/requirements/img/requirement_create_v13_5.png Binary files differnew file mode 100644 index 00000000000..ef1bab6e6d2 --- /dev/null +++ b/doc/user/project/requirements/img/requirement_create_v13_5.png diff --git a/doc/user/project/requirements/img/requirement_view_v13_5.png b/doc/user/project/requirements/img/requirement_view_v13_5.png Binary files differnew file mode 100644 index 00000000000..7fcb24a5e3b --- /dev/null +++ b/doc/user/project/requirements/img/requirement_view_v13_5.png diff --git a/doc/user/project/requirements/img/requirements_list_v13_1.png b/doc/user/project/requirements/img/requirements_list_v13_1.png Binary files differdeleted file mode 100644 index 0ebda571928..00000000000 --- a/doc/user/project/requirements/img/requirements_list_v13_1.png +++ /dev/null diff --git a/doc/user/project/requirements/img/requirements_list_v13_5.png b/doc/user/project/requirements/img/requirements_list_v13_5.png Binary files differnew file mode 100644 index 00000000000..19516e5e66e --- /dev/null +++ b/doc/user/project/requirements/img/requirements_list_v13_5.png diff --git a/doc/user/project/requirements/index.md b/doc/user/project/requirements/index.md index 9d7d3914905..f533f8807d2 100644 --- a/doc/user/project/requirements/index.md +++ b/doc/user/project/requirements/index.md @@ -7,7 +7,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Requirements Management **(ULTIMATE)** -> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/2703) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.10. +> - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/2703) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.10. +> - The ability to add and edit a requirement's long description [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/224622) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.5. With requirements, you can set criteria to check your products against. They can be based on users, stakeholders, system, software, or anything else you find important to capture. @@ -22,7 +23,7 @@ When a feature is no longer necessary, you can [archive the related requirement] <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> For an overview, see [GitLab 12.10 Introduces Requirements Management](https://www.youtube.com/watch?v=uSS7oUNSEoU). - + ## Create a requirement @@ -32,29 +33,43 @@ can create a new requirement. To create a requirement: 1. From your project page, go to **{requirements}** **Requirements**. -1. Click **New requirement**. -1. Enter a descriptive title and click **Create requirement**. +1. Select **New requirement**. +1. Enter a title and description and select **Create requirement**. -You will see the newly created requirement on the top of the list, as the requirements -list is sorted by creation date in descending order. + + +You can see the newly created requirement on the top of the list, with the requirements +list being sorted by creation date, in descending order. + +## View a requirement + +You can view a requirement from the list by selecting it. + + + +To edit a requirement while viewing it, select the **Edit** icon (**{pencil}**) +next to the requirement title. ## Edit a requirement +> The ability to mark a requirement as Satisfied [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/218607) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.5. + You can edit a requirement (if you have the necessary privileges) from the requirements list page. To edit a requirement: -1. From the requirements list, click **Edit** (**{pencil}**). -1. Update the title in text input field. -1. Click **Save changes**. +1. From the requirements list, select the **Edit** icon (**{pencil}**). +1. Update the title and description in text input field. You can also mark a + requirement as satisfied in the edit form by using the check box **Satisfied**. +1. Select **Save changes**. ## Archive a requirement You can archive an open requirement (if you have the necessary privileges) while you're in the **Open** tab. -To archive a requirement, click **Archive** (**{archive}**). +To archive a requirement, select **Archive** (**{archive}**). As soon as a requirement is archived, it no longer appears in the **Open** tab. @@ -64,7 +79,7 @@ You can view the list of archived requirements in the **Archived** tab.  -To reopen an archived requirement, click **Reopen**. +To reopen an archived requirement, select **Reopen**. As soon as a requirement is reopened, it no longer appears in the **Archived** tab. @@ -80,7 +95,7 @@ You can search for a requirement from the requirements list page based on the fo To search for a requirement: 1. In a project, go to **{requirements}** **Requirements > List**. -1. Click the **Search or filter results** field. A dropdown menu appears. +1. Select the **Search or filter results** field. A dropdown menu appears. 1. Select the requirement author from the dropdown or enter plain text to search by requirement title. 1. Press <kbd>Enter</kbd> on your keyboard to filter the list. @@ -97,7 +112,7 @@ You can also sort the requirements list by: GitLab supports [requirements test reports](../../../ci/pipelines/job_artifacts.md#artifactsreportsrequirements) now. You can add a job to your CI pipeline that, when triggered, marks all existing -requirements as Satisfied. +requirements as Satisfied (you may manually satisfy a requirement in the edit form [edit a requirement](#edit-a-requirement)). ### Add the manual job to CI diff --git a/doc/user/project/service_desk.md b/doc/user/project/service_desk.md index e9b07f54b91..0a1b81a6359 100644 --- a/doc/user/project/service_desk.md +++ b/doc/user/project/service_desk.md @@ -10,8 +10,6 @@ info: To determine the technical writer assigned to the Stage/Group associated w > - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/214839) to [GitLab Starter](https://about.gitlab.com/pricing/) in 13.0. > - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/215364) to [GitLab Core](https://about.gitlab.com/pricing/) in 13.2. -## Overview - Service Desk is a module that allows your team to connect directly with any external party through email right inside of GitLab; no external tools required. An ongoing conversation right where your software is built ensures that user feedback ends @@ -129,7 +127,7 @@ in the email, `%{ISSUE_PATH}` placeholder which will be replaced by You can customize the email display name. Emails sent from Service Desk will have this name in the `From` header. The default display name is `GitLab Support Bot`. -### Using custom email address +### Using custom email address **(CORE ONLY)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/2201) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.0. diff --git a/doc/user/project/settings/index.md b/doc/user/project/settings/index.md index 395d4bf30c5..8fdcf58b5aa 100644 --- a/doc/user/project/settings/index.md +++ b/doc/user/project/settings/index.md @@ -8,7 +8,7 @@ type: reference, index, howto # Project settings NOTE: **Note:** -Only project Maintainers and Admin users have the [permissions](../../permissions.md#project-members-permissions) +Only project maintainers and administrators have the [permissions](../../permissions.md#project-members-permissions) to access a project settings. You can adjust your [project](../index.md) settings by navigating @@ -145,7 +145,7 @@ Here you can run housekeeping, archive, rename, transfer, [remove a fork relatio Archiving a project makes it read-only for all users and indicates that it's no longer actively maintained. Projects that have been archived can also be -unarchived. Only project Owners and Admin users have the +unarchived. Only project owners and administrators have the [permissions](../../permissions.md#project-members-permissions) to archive a project. When a project is archived, the repository, packages, issues, merge requests, and all @@ -162,12 +162,12 @@ To archive a project: #### Unarchiving a project Unarchiving a project removes the read-only restriction on a project, and makes it -available in project listings. Only project Owners and Admin users have the +available in project listings. Only project owners and administrators have the [permissions](../../permissions.md#project-members-permissions) to unarchive a project. To find an archived project: -1. Sign in to GitLab as a user with project Owner or Admin permissions. +1. Sign in to GitLab as a user with project owner or administrator permissions. 1. If you: - Have the project's URL, open the project's page in your browser. - Don't have the project's URL: @@ -186,7 +186,7 @@ Next, to unarchive the project: #### Renaming a repository NOTE: **Note:** -Only project Maintainers and Admin users have the [permissions](../../permissions.md#project-members-permissions) to rename a +Only project maintainers and administrators have the [permissions](../../permissions.md#project-members-permissions) to rename a repository. Not to be confused with a project's name where it can also be changed from the [general project settings](#general-project-settings). @@ -207,7 +207,7 @@ old URL won't be able to push or pull. Read more about what happens with the #### Transferring an existing project into another namespace NOTE: **Note:** -Only project Owners and Admin users have the [permissions](../../permissions.md#project-members-permissions) +Only project owners and administrators have the [permissions](../../permissions.md#project-members-permissions) to transfer a project. You can transfer an existing project into a [group](../../group/index.md) if: @@ -229,13 +229,13 @@ read what happens with the [redirects from the old project to the new one](../index.md#redirects-when-changing-repository-paths). NOTE: **Note:** -GitLab administrators can use the admin interface to move any project to any +GitLab administrators can use the administration interface to move any project to any namespace if needed. #### Delete a project NOTE: **Note:** -Only project owners and admins have [permissions](../../permissions.md#project-members-permissions) to delete a project. +Only project owners and administrators have [permissions](../../permissions.md#project-members-permissions) to delete a project. To delete a project: @@ -247,7 +247,7 @@ This action: - Deletes a project including all associated resources (issues, merge requests etc). - From [GitLab 13.2](https://gitlab.com/gitlab-org/gitlab/-/issues/220382) on [Premium or Silver](https://about.gitlab.com/pricing/) or higher tiers, -group admins can [configure](../../group/index.md#enabling-delayed-project-removal) projects within a group +group administrators can [configure](../../group/index.md#enabling-delayed-project-removal) projects within a group to be deleted after a delayed period. When enabled, actual deletion happens after number of days specified in [instance settings](../../admin_area/settings/visibility_and_access_controls.md#default-deletion-delay). @@ -270,7 +270,7 @@ To restore a project marked for deletion: Forking is a great way to [contribute to a project](../repository/forking_workflow.md) of which you're not a member. If you want to use the fork for yourself and don't need to send -[merge requests](../merge_requests.md) to the upstream project, +[merge requests](../merge_requests/index.md) to the upstream project, you can safely remove the fork relationship. CAUTION: **Caution:** diff --git a/doc/user/project/settings/project_access_tokens.md b/doc/user/project/settings/project_access_tokens.md index 57cb610a2e9..b6ce21ebea6 100644 --- a/doc/user/project/settings/project_access_tokens.md +++ b/doc/user/project/settings/project_access_tokens.md @@ -1,21 +1,22 @@ --- -stage: Create -group: Source Code +stage: Manage +group: Access info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" type: reference, howto --- -# Project access tokens **(CORE ONLY)** +# Project access tokens + +NOTE: **Note:** +Project access tokens are supported for self-managed instances on Core and above. They are also supported on GitLab.com Bronze and above. > - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/2587) in GitLab 13.0. > - It was [deployed](https://gitlab.com/groups/gitlab-org/-/epics/2587) behind a feature flag, disabled by default. > - [Became enabled by default](https://gitlab.com/gitlab-org/gitlab/-/issues/218722) in GitLab 13.3. -> - It's disabled on GitLab.com. -> - It can be enabled or disabled by project. +> - [Became available on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/issues/235765) in 13.5. > - It's recommended for production use. -> - For GitLab self-managed instances, GitLab administrators can [disable it](#enable-or-disable-project-access-tokens). -Project access tokens are scoped to a project and can be used to authenticate with the [GitLab API](../../../api/README.md#personalproject-access-tokens). You can also use project access tokens with Git to authenticate over HTTP or SSH. +Project access tokens are scoped to a project and can be used to authenticate with the [GitLab API](../../../api/README.md#personalproject-access-tokens). You can also use project access tokens with Git to authenticate over HTTP. Project access tokens expire on the date you define, at midnight UTC. @@ -48,12 +49,12 @@ API calls made with a project access token are associated with the corresponding These users will appear in **Members** but can not be modified. Furthermore, the bot user can not be added to any other project. -When the project access token is [revoked](#revoking-a-project-access-token) the bot user will be deleted and all -records will be moved to a system-wide user with the username "Ghost User". For more information, -see [Associated Records](../../profile/account/delete_account.md#associated-records). +- The username is set to `project_{project_id}_bot` for the first access token, such as `project_123_bot`. +- The username is set to `project_{project_id}_bot{bot_count}` for further access tokens, such as `project_123_bot1`. + +When the project access token is [revoked](#revoking-a-project-access-token) the bot user is then deleted and all records are moved to a system-wide user with the username "Ghost User". For more information, see [Associated Records](../../profile/account/delete_account.md#associated-records). -Project bot users are a [GitLab-created service account](../../../subscriptions/self_managed/index.md#choose-the-number-of-users), but count as a licensed seat. -These users will not count against your licensed seat in the future when [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/223695) is resolved. +Project bot users are [GitLab-created service accounts](../../../subscriptions/self_managed/index.md#choose-the-number-of-users) and do not count as licensed seats. ## Revoking a project access token @@ -74,33 +75,3 @@ the following table. | `write_registry` | Allows write-access (push) to [container registry](../../packages/container_registry/index.md). | | `read_repository` | Allows read-only access (pull) to the repository. | | `write_repository` | Allows read-write access (pull, push) to the repository. | - -### Enable or disable project access tokens - -Project access tokens are deployed behind a feature flag that is **enabled by default**. -[GitLab administrators with access to the GitLab Rails console](../../../administration/feature_flags.md) -can disable it for your instance, globally or by project. - -To disable it globally: - -```ruby -Feature.disable(:resource_access_token) -``` - -To disable it for a specific project: - -```ruby -Feature.disable(:resource_access_token, project) -``` - -To enable it globally: - -```ruby -Feature.enable(:resource_access_token) -``` - -To enable it for a specific project: - -```ruby -Feature.enable(:resource_access_token, project) -``` diff --git a/doc/user/project/static_site_editor/img/front_matter_ui_v13_4.png b/doc/user/project/static_site_editor/img/front_matter_ui_v13_4.png Binary files differnew file mode 100644 index 00000000000..89864858ed3 --- /dev/null +++ b/doc/user/project/static_site_editor/img/front_matter_ui_v13_4.png diff --git a/doc/user/project/static_site_editor/index.md b/doc/user/project/static_site_editor/index.md index ce14cefba92..8a2f62ec7a2 100644 --- a/doc/user/project/static_site_editor/index.md +++ b/doc/user/project/static_site_editor/index.md @@ -6,50 +6,43 @@ type: reference, how-to description: "The static site editor enables users to edit content on static websites without prior knowledge of the underlying templating language, site architecture or Git commands." --- -# Static Site Editor +# Static Site Editor **(CORE)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/28758) in GitLab 12.10. > - WYSIWYG editor [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/214559) in GitLab 13.0. -> - Support for adding images through the WYSIWYG editor [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/216640) in GitLab 13.1. -> - Markdown front matter hidden on the WYSIWYG editor [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/216834) in GitLab 13.1. -> - Support for `*.md.erb` files [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/223171) in GitLab 13.2. > - Non-Markdown content blocks uneditable on the WYSIWYG mode [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/216836) in GitLab 13.3. -DANGER: **Danger:** -In GitLab 13.0, we [introduced breaking changes](https://gitlab.com/gitlab-org/gitlab/-/issues/213282) -to the URL structure of the Static Site Editor. Follow the instructions in this -[snippet](https://gitlab.com/gitlab-org/project-templates/static-site-editor-middleman/snippets/1976539) -to update your project with the latest changes. - -Static Site Editor enables users to edit content on static websites without +Static Site Editor (SSE) enables users to edit content on static websites without prior knowledge of the underlying templating language, site architecture, or Git commands. A contributor to your project can quickly edit a Markdown page and submit the changes for review. ## Use cases -The Static Site Editors allows collaborators to submit changes to static site +The Static Site Editor allows collaborators to submit changes to static site files seamlessly. For example: -- Non-technical collaborators can easily edit a page directly from the browser; they don't need to know Git and the details of your project to be able to contribute. +- Non-technical collaborators can easily edit a page directly from the browser; + they don't need to know Git and the details of your project to be able to contribute. - Recently hired team members can quickly edit content. -- Temporary collaborators can jump from project to project and quickly edit pages instead of having to clone or fork every single project they need to submit changes to. +- Temporary collaborators can jump from project to project and quickly edit pages instead + of having to clone or fork every single project they need to submit changes to. ## Requirements - In order use the Static Site Editor feature, your project needs to be -pre-configured with the [Static Site Editor Middleman template](https://gitlab.com/gitlab-org/project-templates/static-site-editor-middleman). -- The editor needs to be logged into GitLab and needs to be a member of the -project (with Developer or higher permission levels). + pre-configured with the [Static Site Editor Middleman template](https://gitlab.com/gitlab-org/project-templates/static-site-editor-middleman). +- You need to be logged into GitLab and be a member of the + project (with Developer or higher permission levels). ## How it works -The Static Site Editor is in an early stage of development and only works for +The Static Site Editor is in an early stage of development and only supports Middleman sites for now. You have to use a specific site template to start using it. The project template is configured to deploy a [Middleman](https://middlemanapp.com/) static website with [GitLab Pages](../pages/index.md). -Once your website is up and running, you'll see a button **Edit this page** on +Once your website is up and running, an **Edit this page** button displays on the bottom-left corner of its pages:  @@ -65,44 +58,128 @@ creates a new branch, commits their changes, and opens a merge request. The editor lands directly on the merge request, and then they can assign it to a colleague for review. -## Getting started +## Set up your project First, set up the project. Once done, you can use the Static Site Editor to -easily edit your content. - -### Set up your project - -1. To get started, create a new project from the -[Static Site Editor - Middleman](https://gitlab.com/gitlab-org/project-templates/static-site-editor-middleman) -template. You can either [fork it](../repository/forking_workflow.md#creating-a-fork) -or [create a new project from a template](../../../gitlab-basics/create-project.md#built-in-templates). -1. Edit the `data/config.yml` file adding your project's path. -1. Editing the file triggers a CI/CD pipeline to deploy your project with GitLab Pages. +easily [edit your content](#edit-content). + +1. To get started, create a new project from the [Static Site Editor - Middleman](https://gitlab.com/gitlab-org/project-templates/static-site-editor-middleman) + template. You can either [fork it](../repository/forking_workflow.md#creating-a-fork) + or [create a new project from a template](../../../gitlab-basics/create-project.md#built-in-templates). +1. Edit the [`data/config.yml`](#configuration-files) configuration file + to replace `<username>` and `<project-name>` with the proper values for + your project's path. This triggers a CI/CD pipeline to deploy your project + with GitLab Pages. 1. When the pipeline finishes, from your project's left-side menu, go to **Settings > Pages** to find the URL of your new website. 1. Visit your website and look at the bottom-left corner of the screen to see the new **Edit this page** button. -Anyone satisfying the [requirements](#requirements) will be able to edit the +Anyone satisfying the [requirements](#requirements) can edit the content of the pages without prior knowledge of Git or of your site's codebase. -NOTE: **Note:** -From GitLab 13.1 onwards, the YAML front matter of Markdown files is hidden on the -WYSIWYG editor to avoid unintended changes. To edit it, use the Markdown editing mode, the regular -GitLab file editor, or the Web IDE. +## Edit content + +> - Support for modifying the default merge request title and description [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/216861) in GitLab 13.5. + +After setting up your project, you can start editing content directly from the Static Site Editor. + +To edit a file: + +1. Visit the page you want to edit. +1. Click the **Edit this page** button. +1. The file is opened in the Static Site Editor in **WYSIWYG** mode. If you + wish to edit the raw Markdown instead, you can toggle the **Markdown** mode + in the bottom-right corner. +1. When you're done, click **Submit changes...**. +1. (Optional) Adjust the default title and description of the merge request that will be submitted with your changes. +1. Click **Submit changes**. +1. A new merge request is automatically created and you can assign a colleague for review. + +### Text + +> Support for `*.md.erb` files [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/223171) in GitLab 13.2. + +The Static Site Editors supports Markdown files (`.md`, `.md.erb`) for editing text. + +### Images + +> - Support for adding images through the WYSIWYG editor [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/216640) in GitLab 13.1. + +You can add image files on the WYSIWYG mode by clicking the image icon (**{doc-image}**). +From there, link to a URL, add optional [ALT text](https://moz.com/learn/seo/alt-text), +and you're done. The link can reference images already hosted in your project, an asset hosted +externally on a content delivery network, or any other external URL. The editor renders thumbnail previews +so you can verify the correct image is included and there aren't any references to missing images. +default directory (`source/images/`). + +### Videos -### Use the Static Site Editor to edit your content +> - Support for embedding YouTube videos through the WYSIWYG editor [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/216642) in GitLab 13.5. -For instance, suppose you are a recently hired technical writer at a large -company and a new feature has been added to the company product. +You can embed YouTube videos on the WYSIWYG mode by clicking the video icon (**{live-preview}**). +The following URL/ID formats are supported: -1. You are assigned the task of updating the documentation. -1. You visit a page and see content that needs to be edited. -1. Click the **Edit this page** button on the production site. -1. The file is opened in the Static Site Editor in **WYSIWYG** mode. If you wish to edit the raw Markdown - instead, you can toggle the **Markdown** mode in the bottom-right corner. -1. You edit the file right there and click **Submit changes**. -1. A new merge request is automatically created and you assign it to your colleague for review. +- YouTube watch URL (e.g. `https://www.youtube.com/watch?v=0t1DgySidms`) +- YouTube embed URL (e.g. `https://www.youtube.com/embed/0t1DgySidms`) +- YouTube video ID (e.g. `0t1DgySidms`) + +### Front matter + +> - Markdown front matter hidden on the WYSIWYG editor [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/216834) in GitLab 13.1. +> - Ability to edit page front matter [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/235921) in GitLab 13.5. + +Front matter is a flexible and convenient way to define page-specific variables in data files +intended to be parsed by a static site generator. It is commonly used for setting a page's +title, layout template, or author, but can be used to pass any kind of metadata to the +generator as the page renders out to HTML. Included at the very top of each data file, the +front matter is often formatted as YAML or JSON and requires consistent and accurate syntax. + +To edit the front matter from the Static Site Editor you can use the GitLab's regular file editor, +the Web IDE, or easily update the data directly from the WYSIWYG editor: + +1. Click the **Page settings** button on the bottom-right to reveal a web form with the data you + have on the page's front matter. The form is populated with the current data: + +  + +1. Update the values as you wish and close the panel. +1. When you're done, click **Submit changes...**. +1. Describe your changes (add a commit message). +1. Click **Submit changes**. +1. Click **View merge request** and GitLab will take you there. + +Note that support for adding new attributes to the page's front matter from the form is not supported +yet. You can do so by editing the file locally, through the GitLab regular file editor, or through the Web IDE. Once added, the form will load the new fields. + +## Configuration files + +The Static Site Editor uses Middleman's configuration file, `data/config.yml` +to customize the behavior of the project itself and to control the **Edit this +page** button, rendered through the file [`layout.erb`](https://gitlab.com/gitlab-org/project-templates/static-site-editor-middleman/-/blob/master/source/layouts/layout.erb). + +To [configure the project template to your own project](#set-up-your-project), +you must replace the `<username>` and `<project-name>` in the `data/config.yml` +file with the proper values for your project's path. + +[Other Static Site Generators](#using-other-static-site-generators) used with +the Static Site Editor may use different configuration files or approaches. + +## Using Other Static Site Generators + +Although Middleman is the only Static Site Generator currently officially supported +by the Static Site Editor, you can configure your project's build and deployment +to use a different Static Site Generator. In this case, use the Middleman layout +as an example, and follow a similar approach to properly render an **Edit this page** +button in your Static Site Generator's layout. + +## Upgrade from GitLab 12.10 to 13.0 + +In GitLab 13.0, we [introduced breaking changes](https://gitlab.com/gitlab-org/gitlab/-/issues/213282) +to the URL structure of the Static Site Editor. Follow the instructions in this +[snippet](https://gitlab.com/gitlab-org/project-templates/static-site-editor-middleman/snippets/1976539) +to update your project with the 13.0 changes. ## Limitations -- The Static Site Editor still cannot be quickly added to existing Middleman sites. Follow this [epic](https://gitlab.com/groups/gitlab-org/-/epics/2784) for updates. +- The Static Site Editor still cannot be quickly added to existing Middleman sites. + Follow this [epic](https://gitlab.com/groups/gitlab-org/-/epics/2784) for updates. diff --git a/doc/user/project/web_ide/index.md b/doc/user/project/web_ide/index.md index 821b42af049..4da9b5f88ff 100644 --- a/doc/user/project/web_ide/index.md +++ b/doc/user/project/web_ide/index.md @@ -32,7 +32,7 @@ file path fragments to start seeing results. ## Syntax highlighting As expected from an IDE, syntax highlighting for many languages within -the Web IDE will make your direct editing even easier. +the Web IDE makes your direct editing even easier. The Web IDE currently provides: @@ -79,7 +79,7 @@ which apply to the entire Web IDE screen. > - Support for validation based on custom schemas [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/226982) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.4. The Web IDE provides validation support for certain JSON and YAML files using schemas -based on the [JSON Schema Store](https://www.schemastore.org/json/). +based on the [JSON Schema Store](https://www.schemastore.org/json/). ### Predefined schemas @@ -143,7 +143,7 @@ The Web IDE supports configuration of certain editor settings by using [`.editorconfig` files](https://editorconfig.org/). When opening a file, the Web IDE looks for a file named `.editorconfig` in the current directory and all parent directories. If a configuration file is found and has settings -that match the file's path, these settings will be enforced on the opened file. +that match the file's path, these settings are enforced on the opened file. The Web IDE currently supports the following `.editorconfig` settings: @@ -166,7 +166,7 @@ review the list of changed files. Once you have finalized your changes, you can add a commit message, commit the changes and directly create a merge request. In case you don't have write -access to the selected branch, you will see a warning, but still be able to create +access to the selected branch, you see a warning, but can still create a new branch and start a merge request. To discard a change in a particular file, click the **Discard changes** button on that @@ -201,8 +201,7 @@ left. > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/19318) in [GitLab Core](https://about.gitlab.com/pricing/) 11.0. To switch between your authored and assigned merge requests, click the -dropdown in the top of the sidebar to open a list of merge requests. You will -need to commit or discard all your changes before switching to a different merge +dropdown in the top of the sidebar to open a list of merge requests. You need to commit or discard all your changes before switching to a different merge request. ## Switching branches @@ -211,7 +210,7 @@ request. To switch between branches of the current project repository, click the dropdown in the top of the sidebar to open a list of branches. -You will need to commit or discard all your changes before switching to a +You need to commit or discard all your changes before switching to a different branch. ## Markdown editing @@ -226,7 +225,7 @@ supports [GitLab Flavored Markdown](../../markdown.md#gitlab-flavored-markdown-g You can also upload any local images by pasting them directly in the Markdown file. The image is uploaded to the same directory and is named `image.png` by default. If another file already exists with the same name, a numeric suffix is automatically -added to the file name. +added to the filename. ## Live Preview @@ -249,7 +248,7 @@ The Live Preview feature needs to be enabled in the GitLab instances admin settings. Live Preview is enabled for all projects on GitLab.com - + Once you have done that, you can preview projects with a `package.json` file and a `main` entry point inside the Web IDE. An example `package.json` is shown @@ -292,7 +291,7 @@ to work: [enabled](../../../administration/integration/terminal.md#enabling-and-disabling-terminal-support). **(ULTIMATE ONLY)** If you have the terminal open and the job has finished with its tasks, the -terminal will block the job from finishing for the duration configured in +terminal blocks the job from finishing for the duration configured in [`[session_server].session_timeout`](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-session_server-section) until you close the terminal window. @@ -308,15 +307,15 @@ In order to enable the Web IDE terminals you need to create the file file is fairly similar to the [CI configuration file](../../../ci/yaml/README.md) syntax but with some restrictions: -- No global blocks can be defined (ie: `before_script` or `after_script`) +- No global blocks can be defined (i.e., `before_script` or `after_script`) - Only one job named `terminal` can be added to this file. - Only the keywords `image`, `services`, `tags`, `before_script`, `script`, and `variables` are allowed to be used to configure the job. - To connect to the interactive terminal, the `terminal` job must be still alive - and running, otherwise the terminal won't be able to connect to the job's session. + and running, otherwise the terminal cannot connect to the job's session. By default the `script` keyword has the value `sleep 60` to prevent the job from ending and giving the Web IDE enough time to connect. This means - that, if you override the default `script` value, you'll have to add a command + that, if you override the default `script` value, you have to add a command which would keep the job running, like `sleep`. In the code below there is an example of this configuration file: @@ -333,40 +332,39 @@ terminal: NODE_ENV: "test" ``` -Once the terminal has started, the console will be displayed and we could access +Once the terminal has started, the console is displayed and we could access the project repository files. **Important**. The terminal job is branch dependent. This means that the -configuration file used to trigger and configure the terminal will be the one in +configuration file used to trigger and configure the terminal is the one in the selected branch of the Web IDE. -If there is no configuration file in a branch, an error message will be shown. +If there is no configuration file in a branch, an error message is shown. ### Running interactive terminals in the Web IDE -If Interactive Terminals are available for the current user, the **Terminal** button -will be visible in the right sidebar of the Web IDE. Click this button to open +If Interactive Terminals are available for the current user, the **Terminal** button is visible in the right sidebar of the Web IDE. Click this button to open or close the terminal tab. -Once open, the tab will show the **Start Web Terminal** button. This button may +Once open, the tab shows the **Start Web Terminal** button. This button may be disabled if the environment is not configured correctly. If so, a status -message will describe the issue. Here are some reasons why **Start Web Terminal** +message describes the issue. Here are some reasons why **Start Web Terminal** may be disabled: - `.gitlab/.gitlab-webide.yml` does not exist or is set up incorrectly. - No active private runners are available for the project. -If active, clicking the **Start Web Terminal** button will load the terminal view +If active, clicking the **Start Web Terminal** button loads the terminal view and start connecting to the runner's terminal. At any time, the **Terminal** tab -can be closed and reopened and the state of the terminal will not be affected. +can be closed and reopened and the state of the terminal is not affected. When the terminal is started and is successfully connected to the runner, then the -runner's shell prompt will appear in the terminal. From here, you can enter -commands that will be executed within the runner's environment. This is similar +runner's shell prompt appears in the terminal. From here, you can enter +commands executed within the runner's environment. This is similar to running commands in a local terminal or through SSH. While the terminal is running, it can be stopped by clicking **Stop Terminal**. -This will disconnect the terminal and stop the runner's terminal job. From here, +This disconnects the terminal and stops the runner's terminal job. From here, click **Restart Terminal** to start a new terminal session. ### File syncing to web terminal @@ -408,14 +406,14 @@ terminal: more information. - `$CI_PROJECT_DIR` is a [predefined environment variable](../../../ci/variables/predefined_variables.md) - for GitLab Runner. This is where your project's repository will be. + for GitLab Runners. This is where your project's repository resides. Once you have configured the web terminal for file syncing, then when the web -terminal is started, a **Terminal** status will be visible in the status bar. +terminal is started, a **Terminal** status is visible in the status bar.  -Changes made to your files via the Web IDE will sync to the running terminal +Changes made to your files via the Web IDE sync to the running terminal when: - <kbd>Ctrl</kbd> + <kbd>S</kbd> (or <kbd>Cmd</kbd> + <kbd>S</kbd> on Mac) @@ -425,9 +423,12 @@ when: ### Limitations -Interactive Terminals is in a beta phase and will continue to be improved upon in upcoming -releases. In the meantime, please note that the user is limited to having only one -active terminal at a time. +The Web IDE has a few limitations: + +- Interactive Terminals is in a beta phase and continues to be improved in upcoming releases. In the meantime, please note that the user is limited to having only one + active terminal at a time. + +- LFS files can be rendered and displayed but they cannot be updated and committed using the Web IDE. If an LFS file is modified and pushed to the repository, the LFS pointer in the repository will be overwritten with the modified LFS file content. ### Troubleshooting diff --git a/doc/user/project/wiki/img/wiki_sidebar.png b/doc/user/project/wiki/img/wiki_sidebar.png Binary files differdeleted file mode 100644 index ff39c861a73..00000000000 --- a/doc/user/project/wiki/img/wiki_sidebar.png +++ /dev/null diff --git a/doc/user/project/wiki/img/wiki_sidebar_v13_5.png b/doc/user/project/wiki/img/wiki_sidebar_v13_5.png Binary files differnew file mode 100644 index 00000000000..0f445d61d71 --- /dev/null +++ b/doc/user/project/wiki/img/wiki_sidebar_v13_5.png diff --git a/doc/user/project/wiki/index.md b/doc/user/project/wiki/index.md index 40ef5e216fd..64608b9a915 100644 --- a/doc/user/project/wiki/index.md +++ b/doc/user/project/wiki/index.md @@ -5,7 +5,7 @@ info: "To determine the technical writer assigned to the Stage/Group associated type: reference, how-to --- -# Wiki +# Wiki **(CORE)** A separate system for documentation called Wiki, is built right into each GitLab project. It is enabled by default on all new projects and you can find @@ -19,6 +19,9 @@ You can create Wiki pages in the web interface or [locally using Git](#adding-and-editing-wiki-pages-locally) since every Wiki is a separate Git repository. +[Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/13195) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.5, +**group wikis** became available. Their usage is similar to project wikis, with a few [limitations](../../group/index.md#group-wikis). + ## First time creating the Home page The first time you visit a Wiki, you will be directed to create the Home page. @@ -127,10 +130,12 @@ be preceded by the slash (`/`) character. ## Viewing a list of all created wiki pages +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/17673/) in GitLab 13.5, wiki pages are displayed as a nested tree in the sidebar and pages overview. + Every wiki has a sidebar from which a short list of the created pages can be found. The list is ordered alphabetically. - + If you have many pages, not all will be listed in the sidebar. Click on **View All Pages** to see all of them. @@ -163,48 +168,13 @@ Similar to versioned diff file views, you can see the changes made in a given Wi > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/14902) in **GitLab 12.10.** > - Git events were [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/216014) in **GitLab 13.0.** -> - It's enabled on GitLab.com. -> - Git access activity creation is managed by a feature flag. -> - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enable-or-disable-wiki-events-in-git). **(CORE ONLY)** +> - [Feature flag for Git events was removed](https://gitlab.com/gitlab-org/gitlab/-/issues/258665) in **GitLab 13.5** Wiki events (creation, deletion, and updates) are tracked by GitLab and displayed on the [user profile](../../profile/index.md#user-profile), [group](../../group/index.md#view-group-activity), and [project](../index.md#project-activity) activity pages. -### Enable or disable Wiki events in Git **(CORE ONLY)** - -Tracking wiki events through Git is under development and not ready for production use. It is -deployed behind a feature flag that is **disabled by default**. -[GitLab administrators with access to the GitLab Rails console](../../../administration/feature_flags.md) -can enable it for your instance. - -To enable it: - -```ruby -Feature.enable(:wiki_events_on_git_push) -``` - -To enable for just a particular project: - -```ruby -project = Project.find_by_full_path('your-group/your-project') -Feature.enable(:wiki_events_on_git_push, project) -``` - -To disable it: - -```ruby -Feature.disable(:wiki_events_on_git_push) -``` - -To disable for just a particular project: - -```ruby -project = Project.find_by_full_path('your-group/your-project') -Feature.disable(:wiki_events_on_git_push, project) -``` - ## Adding and editing wiki pages locally Since wikis are based on Git repositories, you can clone them locally and edit diff --git a/doc/user/search/advanced_global_search.md b/doc/user/search/advanced_global_search.md index 53ec8b35631..3152229c39f 100644 --- a/doc/user/search/advanced_global_search.md +++ b/doc/user/search/advanced_global_search.md @@ -1,6 +1,6 @@ --- -stage: Create -group: Editor +stage: Enablement +group: Global Search info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" type: reference --- diff --git a/doc/user/search/advanced_search_syntax.md b/doc/user/search/advanced_search_syntax.md index 804d4c540ac..ed5ecc17a8b 100644 --- a/doc/user/search/advanced_search_syntax.md +++ b/doc/user/search/advanced_search_syntax.md @@ -1,6 +1,6 @@ --- -stage: Create -group: Editor +stage: Enablement +group: Global Search info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" type: reference --- @@ -62,6 +62,7 @@ The Advanced Search Syntax also supports the use of filters. The available filte - filename: Filters by filename. You can use the glob (`*`) operator for fuzzy matching. - path: Filters by path. You can use the glob (`*`) operator for fuzzy matching. - extension: Filters by extension in the filename. Please write the extension without a leading dot. Exact match only. +- blob: Filters by Git `object ID`. Exact match only. To use them, simply add them to your query in the format `<filter_name>:<value>` without any spaces between the colon (`:`) and the value. @@ -72,17 +73,19 @@ Examples: - Finding a file named `found_blob_spec.rb` with the text `CHANGELOG` inside of it: [`CHANGELOG filename:found_blob_spec.rb](https://gitlab.com/search?utf8=%E2%9C%93&snippets=&scope=blobs&repository_ref=&search=CHANGELOG+filename%3Afound_blob_spec.rb&group_id=9970&project_id=278964) - Finding the text `EpicLinks` inside files with the `.rb` extension: [`EpicLinks extension:rb`](https://gitlab.com/search?utf8=%E2%9C%93&snippets=&scope=blobs&repository_ref=&search=EpicLinks+extension%3Arb&group_id=9970&project_id=278964) - Finding the text `Sidekiq` in a file, when that file is in a path that includes `elastic`: [`Sidekiq path:elastic`](https://gitlab.com/search?utf8=%E2%9C%93&snippets=&scope=blobs&repository_ref=&search=Sidekiq+path%3Aelastic&group_id=9970&project_id=278964) +- Finding the files represented by the Git object ID `998707b421c89bd9a3063333f9f728ef3e43d101`: [`* blob:998707b421c89bd9a3063333f9f728ef3e43d101`](https://gitlab.com/search?utf8=%E2%9C%93&snippets=false&scope=blobs&repository_ref=&search=*+blob%3A998707b421c89bd9a3063333f9f728ef3e43d101&group_id=9970) - Syntax filters can be combined for complex filtering. Finding any file starting with `search` containing `eventHub` and with the `.js` extension: [`eventHub filename:search* extension:js`](https://gitlab.com/search?utf8=%E2%9C%93&snippets=&scope=blobs&repository_ref=&search=eventHub+filename%3Asearch*+extension%3Ajs&group_id=9970&project_id=278964) #### Excluding filters [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/31684) in GitLab Starter 13.3. -Filters can be inversed to **filter out** results from the result set, by prefixing the filter name with a `-` (hyphen) character, such as: +Filters can be inverted to **filter out** results from the result set, by prefixing the filter name with a `-` (hyphen) character, such as: - `-filename` - `-path` - `-extension` +- `-blob` Examples: diff --git a/doc/user/search/img/basic_search.png b/doc/user/search/img/basic_search.png Binary files differnew file mode 100644 index 00000000000..234805d5a4f --- /dev/null +++ b/doc/user/search/img/basic_search.png diff --git a/doc/user/search/img/basic_search_results.png b/doc/user/search/img/basic_search_results.png Binary files differnew file mode 100644 index 00000000000..52aa2e3fe6c --- /dev/null +++ b/doc/user/search/img/basic_search_results.png diff --git a/doc/user/search/index.md b/doc/user/search/index.md index 475a72385ac..412951f8a3b 100644 --- a/doc/user/search/index.md +++ b/doc/user/search/index.md @@ -32,7 +32,7 @@ You can also filter the results using the search and filter field, as described You'll also find shortcuts to issues and merge requests created by you or assigned to you on the search field on the top-right of your screen: - + ### Filtering issue and merge request lists @@ -129,14 +129,6 @@ characters to begin your search. For example, if you want to search for issues that have the assignee "Simone Presley", you'll need to type at least "Sim" before autocomplete gives any relevant results. -## Code search - -To search through code or other documents in a single project, you can use -the search field on the top-right of your screen while the project page is open. - - - - ## Search history You can view recent searches by clicking on the little arrow-clock icon, which is to the left of the search input. Click the search entry to run that search again. This feature is available for issues and merge requests. Searches are stored locally in your browser. @@ -155,24 +147,6 @@ Some filters can be added multiple times. These include but are not limited to a  -## Shortcut - -You'll also find a shortcut on the search field on the top-right of the project's dashboard to -quickly access issues and merge requests created or assigned to you within that project: - - - -### Autocomplete suggestions - -You can also type in this search bar to see autocomplete suggestions for: - -- Projects and groups -- Various help pages (try and type **API help**) -- Project feature pages (try and type **milestones**) -- Various settings pages (try and type **user settings**) -- Recently viewed issues (try and type some word from the title of a recently viewed issue) -- Recently viewed merge requests (try and type some word from the title of a recently merge request) - ## To-Do List Your [To-Do List](../todos.md#gitlab-to-do-list) can be searched by "to do" and "done". @@ -219,6 +193,61 @@ and **Labels**, select multiple issues to add to a list of your choice:  +## Shortcut + +You'll find a shortcut on the search field on the top-right of the project's dashboard to +quickly access issues and merge requests created or assigned to you within that project: + + + +### Autocomplete suggestions + +You can also type in this search bar to see autocomplete suggestions for: + +- Projects and groups +- Various help pages (try and type **API help**) +- Project feature pages (try and type **milestones**) +- Various settings pages (try and type **user settings**) +- Recently viewed issues (try and type some word from the title of a recently viewed issue) +- Recently viewed merge requests (try and type some word from the title of a recently viewed merge request) +- Recently viewed epics (try and type some word from the title of a recently viewed epic) + +## Basic search + +The Basic search in GitLab is a global search service that allows you to search +across the entire GitLab instance, within a group, or a single project. Basic search is +backed by the database and allows searching in: + +- Projects +- Issues +- Merge requests +- Milestones +- Users +- Epics (Group only) +- Code (Project only) +- Comments (Project only) +- Commits (Project only) +- Wiki (Project only) + +To start a search, type into the search bar on the top-right of the screen. You can always search +in all GitLab and may also see the options to search within a group or project if you are in the +group or project dashboard. + + + +Once the results are returned, you can modify the search, select a different type of data to +search, or choose a specific group or project. + + + +### Code search + +To search through code or other documents in a single project, you can use +the search field on the top-right of your screen while the project page is open. + + + + ## Advanced Search **(STARTER)** Leverage Elasticsearch for faster, more advanced code search across your entire diff --git a/doc/user/snippets.md b/doc/user/snippets.md index 15391f034a8..2d0c9d4f943 100644 --- a/doc/user/snippets.md +++ b/doc/user/snippets.md @@ -92,6 +92,40 @@ be changed to `http-a-weird-filename-me` to be included in the snippet's repository. As snippets are stored by ID, changing their filenames will not break direct or embedded links to the snippet. +### Multiple files by Snippet + +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/2829) in GitLab 13.5. + +GitLab Snippets support multiple files in one single snippet. It can be very handy +when your code snippet is composed of multiple parts or when they relate +to a certain context. For example: + +- A snippet that includes a script and its output. +- A snippet that includes HTML, CSS, and JS code. +- A snippet with a `docker-compose.yml` file and its associated `.env` file. +- A `gulpfile.js` file coupled with a `package.json` file, which together can be used to bootstrap a project and manage its dependencies. + +Snippets support between 1 and 10 files. They can be managed via Git (since they're [versioned](#versioned-snippets) +by a Git repository), through the [Snippets API](../api/snippets.md), or within the GitLab UI. + + + +To add a new file to your snippet through the GitLab UI: + +1. Go to your snippet in the GitLab UI. +1. Click **Edit** in the top right. +1. Select **Add another file**. +1. Add your content to the file in the form fields provided. +1. Click **Save changes**. + +To delete a file from your snippet through the GitLab UI: + +1. Go to your snippet in the GitLab UI. +1. Click **Edit** in the top right. +1. Select **Delete file** alongside the file name of each file +you wish to delete. +1. Click **Save changes**. + ### Cloning snippets Snippets can be cloned as a regular Git repository using SSH or HTTPS. Click the **Clone** @@ -114,16 +148,16 @@ see the documentation on [reducing repository size](../user/project/repository/r ### Limitations - Binary files are not supported. -- Creating or deleting branches is not supported. Only a default *master*. -branch is used. +- Creating or deleting branches is not supported. Only a default *master* branch is used. - Git tags are not supported in snippet repositories. -- Snippets' repositories are limited to one file. Attempting to push more -than one file will result in an error. +- Snippets' repositories are limited to 10 files. Attempting to push more +than 10 files will result in an error. - Revisions are not *yet* visible to the user on the GitLab UI, but it's planned to be added in future iterations. See the [revisions tab issue](https://gitlab.com/gitlab-org/gitlab/-/issues/39271) for updates. - The [maximum size for a snippet](../administration/snippets/index.md#snippets-content-size-limit) is 50 MB, by default. +- Git LFS is not supported. ## Discover snippets @@ -131,8 +165,8 @@ There are two main ways of how you can discover snippets in GitLab. For exploring all snippets that are visible to you, you can go to the Snippets dashboard of your GitLab instance via the top navigation. For GitLab.com you can -find it [here](https://gitlab.com/dashboard/snippets). This navigates you to an -overview that shows snippets you created and allows you to explore all snippets. +navigate to an [overview]((https://gitlab.com/dashboard/snippets)) that shows snippets +you created and allows you to explore all snippets. If you want to discover snippets that belong to a specific project, you can navigate to the Snippets page via the left side navigation on the project page. diff --git a/doc/user/todos.md b/doc/user/todos.md index 1fca3c0ab64..c48d2adfa45 100644 --- a/doc/user/todos.md +++ b/doc/user/todos.md @@ -15,7 +15,7 @@ spend your time. This can include taking an action, or keeping track of things do your work, being able to get started quickly is important. Your *To-Do List* offers a chronological list of items waiting for your input -(known as *to-dos*) in a single dashboard. +(known as *to do items*) in a single dashboard. The To-Do List supports tracking [actions](#what-triggers-a-to-do) related to the following: @@ -26,18 +26,18 @@ the following:  -You can access your To-Do List by clicking the **{task-done}** To-Do List icon -next to the search bar in the top navigation. If the to-do item count is: +You can access your To-Do List by clicking the To-Do List icon (**{task-done}**) +next to the search bar in the top navigation. If the to do item count is: -- *Less than 100*, the number in blue is the number of to-do items. +- *Less than 100*, the number in blue is the number of to do items. - *100 or more*, the number displays as 99+. The exact number displays in the To-Do List.  -## What triggers a to-do +## What triggers a to do -A to-do item appears on your To-Do List when: +A to do item appears on your To-Do List when: - An issue or merge request is assigned to you. - You're `@mentioned` in the description or comment of an issue or merge request @@ -51,29 +51,29 @@ A to-do item appears on your To-Do List when: - You're the author. - You're the user that set the merge request to automatically merge after a pipeline succeeds. -- [Since GitLab 13.2](https://gitlab.com/gitlab-org/gitlab/-/issues/12136), a +- [In GitLab 13.2](https://gitlab.com/gitlab-org/gitlab/-/issues/12136) and later, a merge request is removed from a [merge train](../ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md), and you're the user that added it. **(PREMIUM)** When several trigger actions occur for the same user on the same object (for -example, an issue), GitLab displays only the first action as a single to-do +example, an issue), GitLab displays only the first action as a single to do item. -To-do triggers aren't affected by [GitLab notification email settings](profile/notifications.md). +To do item triggers aren't affected by [GitLab notification email settings](profile/notifications.md). NOTE: **Note:** -When a user no longer has access to a resource related to a to-do (such as an -issue, merge request, project, or group), for security reasons GitLab deletes -any related to-do items within the next hour. Deletion is delayed to prevent -data loss, in the case where a user's access is accidentally revoked. +When a user no longer has access to a resource related to a to do item (such as +an issue, merge request, project, or group), for security reasons GitLab +deletes any related to do items within the next hour. Deletion is delayed to +prevent data loss, in the case where a user's access is accidentally revoked. -### Directly addressing a to-do +### Directly addressing a to do > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/7926) in GitLab 9.0. -If you're mentioned at the start of a line, the to-do you receive will be listed -as *directly addressed*. For example, in the following comment: +If you're mentioned at the start of a line, the to do item you receive will be +listed as *directly addressed*. For example, in the following comment: ```markdown @alice What do you think? cc: @bob @@ -87,23 +87,27 @@ as *directly addressed*. For example, in the following comment: @erin @frank thank you! ``` -The people receiving directly addressed to-do items are `@alice`, `@erin`, and -`@frank`. Directly addressed to-do items only differ from mentions in their type +The people receiving directly addressed to do items are `@alice`, `@erin`, and +`@frank`. Directly addressed to do items only differ from mentions in their type for filtering purposes; otherwise, they appear as normal. -### Manually creating a to-do +### Manually creating a to do -You can add an issue or merge request (or epic **(ULTIMATE)**) to your -To-Do List by clicking its **Add a To Do** button. +You can also add the following to your To-Do List by clicking the **Add a to do** button on an: + +- [Issue](project/issues/index.md) +- [Merge Request](project/merge_requests/index.md) +- [Epic](group/epics/index.md) **(ULTIMATE)** +- [Design](project/issues/design_management.md)  -## Marking a to-do as done +## Marking a to do as done Any action to an issue or merge request (or epic **(ULTIMATE)**) will mark its -corresponding to-do as done. +corresponding to do item as done. -Actions that dismiss to-do items include: +Actions that dismiss to do items include: - Changing the assignee - Changing the milestone @@ -111,27 +115,28 @@ Actions that dismiss to-do items include: - Commenting on the issue Your To-Do List is personal, and items are only marked as done if you take -action. If you close the issue or merge request, your to-do is marked as done. +action. If you close the issue or merge request, your to do item is marked as +done. To prevent other users from closing issues without you being notified, if someone else closes, merges, or takes action on an issue or merge request (or -epic **(ULTIMATE)**), your to-do will remain pending. +epic **(ULTIMATE)**), your to do item remains pending. -There's just one to-do for each of these, so mentioning a user many times in an -issue will only trigger one to-do item. +There's just one to do item for each of these, so mentioning a user many times +in an issue only triggers one to do item. -If no action is needed, you can manually mark the to-do as done by clicking its -corresponding **Done** button to have GitLab remove the item from your -To-Do List. +If no action is needed, you can manually mark the to do item as done by +clicking its corresponding **Done** button to have GitLab remove the item from +your To-Do List. - + -You can also mark a to-do as done by clicking the **Mark as done** button in the -sidebar of an issue or merge request (or epic **(ULTIMATE)**). +You can also mark a to do item as done by clicking the **Mark as done** button +in the sidebar of an issue or merge request (or epic **(ULTIMATE)**).  -You can mark all your to-do items as done at once by clicking the +You can mark all your to do items as done at once by clicking the **Mark all as done** button. ## Filtering your To-Do List @@ -142,9 +147,9 @@ You can use the following types of filters with your To-Do List: | ------- | ---------------------------------------------------------------- | | Project | Filter by project. | | Group | Filter by group. | -| Author | Filter by the author that triggered the To Do. | +| Author | Filter by the author that triggered the to do. | | Type | Filter by issue, merge request, design, or epic. **(ULTIMATE)** | -| Action | Filter by the action that triggered the To Do. | +| Action | Filter by the action that triggered the to do. | You can also filter by more than one of these at the same time. The previously described [triggering actions](#what-triggers-a-to-do) include: diff --git a/doc/user/upgrade_email_bypass.md b/doc/user/upgrade_email_bypass.md index 027f7337228..35fddc4a1d4 100644 --- a/doc/user/upgrade_email_bypass.md +++ b/doc/user/upgrade_email_bypass.md @@ -82,7 +82,7 @@ As an administrator, you may also confirm a user in the [Admin Area](admin_area/ ## What do I do if I am an administrator and I am locked out? If you are an administrator and cannot otherwise verify your email address, sign in to your GitLab -instance with a [Rails console session](../administration/troubleshooting/navigating_gitlab_via_rails_console.md#starting-a-rails-console-session). +instance with a [Rails console session](../administration/operations/rails_console.md#starting-a-rails-console-session). Once connected, run the following commands to confirm your administrator account: ```ruby @@ -94,7 +94,7 @@ admin.save! ## How do I force-confirm all users on my self-managed instance? If you are an administrator and would like to force-confirm all users on your system, sign in to your GitLab -instance with a [Rails console session](../administration/troubleshooting/navigating_gitlab_via_rails_console.md#starting-a-rails-console-session). +instance with a [Rails console session](../administration/operations/rails_console.md#starting-a-rails-console-session). Once connected, run the following commands to confirm all user accounts: ```ruby diff --git a/doc/web_hooks/web_hooks.md b/doc/web_hooks/web_hooks.md deleted file mode 100644 index fffb6a5d86d..00000000000 --- a/doc/web_hooks/web_hooks.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../user/project/integrations/webhooks.md' ---- - -This document was moved to [project/integrations/webhooks](../user/project/integrations/webhooks.md). diff --git a/doc/workflow/README.md b/doc/workflow/README.md deleted file mode 100644 index ce129f0663b..00000000000 --- a/doc/workflow/README.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../README.md' ---- - -This document was moved to [another location](../README.md). diff --git a/doc/workflow/add-user/add-user.md b/doc/workflow/add-user/add-user.md deleted file mode 100644 index f1ec771dd9a..00000000000 --- a/doc/workflow/add-user/add-user.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../../user/project/members/index.md' ---- - -This document was moved to [../../user/project/members/index.md](../../user/project/members/index.md) diff --git a/doc/workflow/authorization_for_merge_requests.md b/doc/workflow/authorization_for_merge_requests.md deleted file mode 100644 index 8e43d340613..00000000000 --- a/doc/workflow/authorization_for_merge_requests.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../user/project/merge_requests/authorization_for_merge_requests.md' ---- - -This document was moved to [user/project/merge_requests/authorization_for_merge_requests](../user/project/merge_requests/authorization_for_merge_requests.md) diff --git a/doc/workflow/award_emoji.md b/doc/workflow/award_emoji.md deleted file mode 100644 index 02db97b8dd6..00000000000 --- a/doc/workflow/award_emoji.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../user/award_emojis.md' ---- - -This document was moved to [another location](../user/award_emojis.md). diff --git a/doc/workflow/cherry_pick_changes.md b/doc/workflow/cherry_pick_changes.md deleted file mode 100644 index 29c4f854416..00000000000 --- a/doc/workflow/cherry_pick_changes.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../user/project/merge_requests/cherry_pick_changes.md' ---- - -This document was moved to [another location](../user/project/merge_requests/cherry_pick_changes.md). diff --git a/doc/workflow/ff_merge.md b/doc/workflow/ff_merge.md deleted file mode 100644 index 11e9e1bbd6b..00000000000 --- a/doc/workflow/ff_merge.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../user/project/merge_requests/fast_forward_merge.md' ---- - -This document was moved to [user/project/merge_requests/fast_forward_merge](../user/project/merge_requests/fast_forward_merge.md). diff --git a/doc/workflow/file_finder.md b/doc/workflow/file_finder.md deleted file mode 100644 index f7098c88fd1..00000000000 --- a/doc/workflow/file_finder.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../user/project/repository/file_finder.md' ---- - -This document was moved to [another location](../user/project/repository/file_finder.md). diff --git a/doc/workflow/forking_workflow.md b/doc/workflow/forking_workflow.md deleted file mode 100644 index fa617d859a5..00000000000 --- a/doc/workflow/forking_workflow.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../user/project/repository/forking_workflow.md' ---- - -This document was moved to [another location](../user/project/repository/forking_workflow.md). diff --git a/doc/workflow/git_annex.md b/doc/workflow/git_annex.md deleted file mode 100644 index e54d52ea70d..00000000000 --- a/doc/workflow/git_annex.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../administration/git_annex.md' ---- - -This document was moved to [another location](../administration/git_annex.md). diff --git a/doc/workflow/git_lfs.md b/doc/workflow/git_lfs.md deleted file mode 100644 index 69939c0efd0..00000000000 --- a/doc/workflow/git_lfs.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../topics/git/lfs/index.md' ---- - -This document was moved to [another location](../topics/git/lfs/index.md). diff --git a/doc/workflow/gitlab_flow.md b/doc/workflow/gitlab_flow.md deleted file mode 100644 index e03281c0ffc..00000000000 --- a/doc/workflow/gitlab_flow.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../topics/gitlab_flow.md' ---- - -This document was moved to [another location](../topics/gitlab_flow.md). diff --git a/doc/workflow/groups.md b/doc/workflow/groups.md deleted file mode 100644 index c7f4647baa9..00000000000 --- a/doc/workflow/groups.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../user/group/index.md' ---- - -This document was moved to [another location](../user/group/index.md). diff --git a/doc/workflow/importing/README.md b/doc/workflow/importing/README.md deleted file mode 100644 index 29da321ba46..00000000000 --- a/doc/workflow/importing/README.md +++ /dev/null @@ -1,5 +0,0 @@ ----
-redirect_to: '../../user/project/import/index.md'
----
-
-This document was moved to [another location](../../user/project/import/index.md).
diff --git a/doc/workflow/importing/import_projects_from_bitbucket.md b/doc/workflow/importing/import_projects_from_bitbucket.md deleted file mode 100644 index a42ba7d4518..00000000000 --- a/doc/workflow/importing/import_projects_from_bitbucket.md +++ /dev/null @@ -1,5 +0,0 @@ ----
-redirect_to: '../../user/project/import/bitbucket.md'
----
-
-This document was moved to [another location](../../user/project/import/bitbucket.md).
diff --git a/doc/workflow/importing/import_projects_from_fogbugz.md b/doc/workflow/importing/import_projects_from_fogbugz.md deleted file mode 100644 index f5c791dc6de..00000000000 --- a/doc/workflow/importing/import_projects_from_fogbugz.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../../user/project/import/fogbugz.md' ---- - -This document was moved to [another location](../../user/project/import/fogbugz.md). diff --git a/doc/workflow/importing/import_projects_from_gitea.md b/doc/workflow/importing/import_projects_from_gitea.md deleted file mode 100644 index df053835b44..00000000000 --- a/doc/workflow/importing/import_projects_from_gitea.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../../user/project/import/gitea.md' ---- - -This document was moved to [another location](../../user/project/import/gitea.md). diff --git a/doc/workflow/importing/import_projects_from_github.md b/doc/workflow/importing/import_projects_from_github.md deleted file mode 100644 index 6397fcc74b8..00000000000 --- a/doc/workflow/importing/import_projects_from_github.md +++ /dev/null @@ -1,5 +0,0 @@ ----
-redirect_to: '../../user/project/import/github.md'
----
-
-This document was moved to [another location](../../user/project/import/github.md).
diff --git a/doc/workflow/importing/import_projects_from_gitlab_com.md b/doc/workflow/importing/import_projects_from_gitlab_com.md deleted file mode 100644 index 135b9704df9..00000000000 --- a/doc/workflow/importing/import_projects_from_gitlab_com.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../../user/project/import/gitlab_com.md' ---- - -This document was moved to [another location](../../user/project/import/gitlab_com.md). diff --git a/doc/workflow/importing/migrating_from_svn.md b/doc/workflow/importing/migrating_from_svn.md deleted file mode 100644 index 99f13d6354c..00000000000 --- a/doc/workflow/importing/migrating_from_svn.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../../user/project/import/svn.md' ---- - -This document was moved to [another location](../../user/project/import/svn.md). diff --git a/doc/workflow/issue_weight.md b/doc/workflow/issue_weight.md deleted file mode 100644 index 94eb38356e8..00000000000 --- a/doc/workflow/issue_weight.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../user/project/issues/issue_weight.md' ---- - -This document was moved to [another location](../user/project/issues/issue_weight.md). diff --git a/doc/workflow/labels.md b/doc/workflow/labels.md deleted file mode 100644 index 3d07d411dd4..00000000000 --- a/doc/workflow/labels.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -redirect_to: '../user/project/labels.md' ---- - -# Labels - -This document was moved to [user/project/labels.md](../user/project/labels.md). diff --git a/doc/workflow/lfs/lfs_administration.md b/doc/workflow/lfs/lfs_administration.md deleted file mode 100644 index 4a784409eff..00000000000 --- a/doc/workflow/lfs/lfs_administration.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../../administration/lfs/index.md' ---- - -This document was moved to [another location](../../administration/lfs/index.md). diff --git a/doc/workflow/lfs/manage_large_binaries_with_git_lfs.md b/doc/workflow/lfs/manage_large_binaries_with_git_lfs.md deleted file mode 100644 index 4656bccf5e6..00000000000 --- a/doc/workflow/lfs/manage_large_binaries_with_git_lfs.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../../topics/git/lfs/index.md' ---- - -This document was moved to [another location](../../topics/git/lfs/index.md). diff --git a/doc/workflow/lfs/migrate_from_git_annex_to_git_lfs.md b/doc/workflow/lfs/migrate_from_git_annex_to_git_lfs.md deleted file mode 100644 index dd98a50e353..00000000000 --- a/doc/workflow/lfs/migrate_from_git_annex_to_git_lfs.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../../topics/git/lfs/migrate_from_git_annex_to_git_lfs.md' ---- - -This document was moved to [another location](../../topics/git/lfs/migrate_from_git_annex_to_git_lfs.md). diff --git a/doc/workflow/merge_request_approvals.md b/doc/workflow/merge_request_approvals.md deleted file mode 100644 index bfcd8faf236..00000000000 --- a/doc/workflow/merge_request_approvals.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../user/project/merge_requests/merge_request_approvals.md' ---- - -This document was moved to [another location](../user/project/merge_requests/merge_request_approvals.md). diff --git a/doc/workflow/merge_requests.md b/doc/workflow/merge_requests.md deleted file mode 100644 index fd9f9b81bc9..00000000000 --- a/doc/workflow/merge_requests.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../user/project/merge_requests/index.md' ---- - -This document was moved to [user/project/merge_requests/index.md](../user/project/merge_requests/index.md). diff --git a/doc/workflow/merge_when_build_succeeds.md b/doc/workflow/merge_when_build_succeeds.md deleted file mode 100644 index 41e6ff0cdd6..00000000000 --- a/doc/workflow/merge_when_build_succeeds.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../user/project/merge_requests/merge_when_pipeline_succeeds.md' ---- - -This document was moved to [merge_when_pipeline_succeeds](../user/project/merge_requests/merge_when_pipeline_succeeds.md). diff --git a/doc/workflow/milestones.md b/doc/workflow/milestones.md deleted file mode 100644 index 18dc15f7327..00000000000 --- a/doc/workflow/milestones.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../user/project/milestones/index.md' ---- - -This document was moved to [another location](../user/project/milestones/index.md). diff --git a/doc/workflow/notifications.md b/doc/workflow/notifications.md deleted file mode 100644 index 23f96360484..00000000000 --- a/doc/workflow/notifications.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../user/profile/notifications.md' ---- - -This document was moved to [another location](../user/profile/notifications.md). diff --git a/doc/workflow/project_features.md b/doc/workflow/project_features.md deleted file mode 100644 index f54afb768a1..00000000000 --- a/doc/workflow/project_features.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../user/project/index.md' ---- - -This document was moved to [../user/project/index.md](../user/project/index.md) diff --git a/doc/workflow/protected_branches.md b/doc/workflow/protected_branches.md deleted file mode 100644 index 1bcac4a2de5..00000000000 --- a/doc/workflow/protected_branches.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../user/project/protected_branches.md' ---- - -This document was moved to [another location](../user/project/protected_branches.md). diff --git a/doc/workflow/rebase_before_merge.md b/doc/workflow/rebase_before_merge.md deleted file mode 100644 index 10e768d3371..00000000000 --- a/doc/workflow/rebase_before_merge.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../user/project/merge_requests/fast_forward_merge.md' ---- - -This document was moved to [another location](../user/project/merge_requests/fast_forward_merge.md). diff --git a/doc/workflow/releases.md b/doc/workflow/releases.md deleted file mode 100644 index f3ba61f6a5c..00000000000 --- a/doc/workflow/releases.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../user/project/releases/index.md#add-release-notes-to-git-tags' ---- - -This document was moved to [another location](../user/project/releases/index.md#add-release-notes-to-git-tags). diff --git a/doc/workflow/repository_mirroring.md b/doc/workflow/repository_mirroring.md deleted file mode 100644 index dc77f4f47af..00000000000 --- a/doc/workflow/repository_mirroring.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../user/project/repository/repository_mirroring.md' ---- - -This document was moved to [another location](../user/project/repository/repository_mirroring.md). diff --git a/doc/workflow/revert_changes.md b/doc/workflow/revert_changes.md deleted file mode 100644 index 15f199af703..00000000000 --- a/doc/workflow/revert_changes.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../user/project/merge_requests/revert_changes.md' ---- - -This document was moved to [user/project/merge_requests/revert_changes](../user/project/merge_requests/revert_changes.md). diff --git a/doc/workflow/share_projects_with_other_groups.md b/doc/workflow/share_projects_with_other_groups.md deleted file mode 100644 index c39cd78f32d..00000000000 --- a/doc/workflow/share_projects_with_other_groups.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../user/project/members/share_project_with_groups.md' ---- - -This document was moved to [../user/project/members/share_project_with_groups.md](../user/project/members/share_project_with_groups.md) diff --git a/doc/workflow/share_with_group.md b/doc/workflow/share_with_group.md deleted file mode 100644 index c39cd78f32d..00000000000 --- a/doc/workflow/share_with_group.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../user/project/members/share_project_with_groups.md' ---- - -This document was moved to [../user/project/members/share_project_with_groups.md](../user/project/members/share_project_with_groups.md) diff --git a/doc/workflow/shortcuts.md b/doc/workflow/shortcuts.md deleted file mode 100644 index 4b35c61ec5e..00000000000 --- a/doc/workflow/shortcuts.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../user/shortcuts.md' ---- - -This document was moved to [another location](../user/shortcuts.md). diff --git a/doc/workflow/time_tracking.md b/doc/workflow/time_tracking.md deleted file mode 100644 index e109410e22d..00000000000 --- a/doc/workflow/time_tracking.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../user/project/time_tracking.md' ---- - -This document was moved to [another location](../user/project/time_tracking.md). diff --git a/doc/workflow/timezone.md b/doc/workflow/timezone.md deleted file mode 100644 index f1a2e1af66a..00000000000 --- a/doc/workflow/timezone.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../administration/timezone.md' ---- - -This document was moved to [another location](../administration/timezone.md). diff --git a/doc/workflow/todos.md b/doc/workflow/todos.md deleted file mode 100644 index 48c9a3faf1d..00000000000 --- a/doc/workflow/todos.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../user/todos.md' ---- - -This document was moved to [another location](../user/todos.md). diff --git a/doc/workflow/web_editor.md b/doc/workflow/web_editor.md deleted file mode 100644 index 2366372d984..00000000000 --- a/doc/workflow/web_editor.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../user/project/repository/web_editor.md' ---- - -This document was moved to [user/project/repository/web_editor](../user/project/repository/web_editor.md). diff --git a/doc/workflow/wip_merge_requests.md b/doc/workflow/wip_merge_requests.md deleted file mode 100644 index 020455dcbdc..00000000000 --- a/doc/workflow/wip_merge_requests.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../user/project/merge_requests/work_in_progress_merge_requests.md' ---- - -This document was moved to [user/project/merge_requests/work_in_progress_merge_requests](../user/project/merge_requests/work_in_progress_merge_requests.md). diff --git a/doc/workflow/workflow.md b/doc/workflow/workflow.md deleted file mode 100644 index c77d95cd326..00000000000 --- a/doc/workflow/workflow.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -redirect_to: '../gitlab-basics/feature_branch_workflow.md' ---- - -This document was moved to [another location](../gitlab-basics/feature_branch_workflow.md). |
