diff options
Diffstat (limited to 'doc')
843 files changed, 30105 insertions, 9441 deletions
diff --git a/doc/.vale/gitlab/Acronyms.yml b/doc/.vale/gitlab/Acronyms.yml index d166e71491c..c347c663bbf 100644 --- a/doc/.vale/gitlab/Acronyms.yml +++ b/doc/.vale/gitlab/Acronyms.yml @@ -68,6 +68,7 @@ exceptions: - SSH - SSL - SSO + - SVG - SVN - TCP - TIP diff --git a/doc/.vale/gitlab/AlertBoxStyle.yml b/doc/.vale/gitlab/AlertBoxStyle.yml index 831d5395fc8..8f9e444edc1 100644 --- a/doc/.vale/gitlab/AlertBoxStyle.yml +++ b/doc/.vale/gitlab/AlertBoxStyle.yml @@ -5,12 +5,12 @@ # # For a list of all options, see https://errata-ai.gitbook.io/vale/getting-started/styles extends: existence -message: 'Alert box "%s" must use the formatting detailed in the documentation style guide.' +message: 'Alert box "%s" must use the formatting in the style guide.' link: https://docs.gitlab.com/ee/development/documentation/styleguide.html#alert-boxes level: error scope: raw raw: - '((NOTE|TIP|CAUTION|DANGER): \*\*[^:]*\*\*)|' - - '((NOTE: \*\*NOTE:\*\*)|(TIP: \*\*TIP:\*\*)|(CAUTION: \*\*CAUTION:\*\*)|(DANGER: \*\*DANGER:\*\*))|' - - '((NOTE: \*\*note:\*\*)|(TIP: \*\*tip:\*\*)|(CAUTION: \*\*caution:\*\*)|(DANGER: \*\*danger:\*\*))|' - - '((NOTE|TIP|CAUTION|DANGER): \*\*.*\*\*.+)' + - '((NOTE: \*\*(NOTE|note):\*\*)|(TIP: \*\*(TIP|tip):\*\*)|(CAUTION: \*\*(CAUTION|caution):\*\*)|(DANGER: \*\*(DANGER|danger):\*\*))|' + - '((NOTE|TIP|CAUTION|DANGER): \*\*.*\*\*.+)|' + - '((\n[> ]*(\*){1,2}(NOTE|Note|note|TIP|Tip|tip|CAUTION|Caution|caution|DANGER|Danger|danger):(\*){1,2}))' diff --git a/doc/.vale/gitlab/Contractions.yml b/doc/.vale/gitlab/ContractionsDiscard.yml index dc48b876f40..698fda86b5b 100644 --- a/doc/.vale/gitlab/Contractions.yml +++ b/doc/.vale/gitlab/ContractionsDiscard.yml @@ -1,7 +1,7 @@ --- -# Suggestion: gitlab.Contractions +# Suggestion: gitlab.ContractionsDiscard # -# Checks for use of common and uncommon contractions. +# Suggests a list of agreed-upon contractions to discard. # # For a list of all options, see https://errata-ai.gitbook.io/vale/getting-started/styles extends: substitution @@ -12,18 +12,6 @@ nonword: false ignorecase: true swap: - # Common contractions are ok - it is: it's - can not: can't - cannot: can't - do not: don't - have not: haven't - that is: that's - we are: we're - would not: wouldn't - you are: you're - you have: you've - # Uncommon contractions are not ok aren't: are not couldn't: could not @@ -42,4 +30,3 @@ swap: where's: where is who's: who is why's: why is - diff --git a/doc/.vale/gitlab/ContractionsKeep.yml b/doc/.vale/gitlab/ContractionsKeep.yml new file mode 100644 index 00000000000..eeaf65e0829 --- /dev/null +++ b/doc/.vale/gitlab/ContractionsKeep.yml @@ -0,0 +1,25 @@ +--- +# Suggestion: gitlab.ContractionsKeep +# +# Suggests a list of agreed-upon contractions to keep. +# +# For a list of all options, see https://errata-ai.gitbook.io/vale/getting-started/styles +extends: substitution +message: 'Use "%s" instead of "%s", for a friendly, informal tone.' +link: https://docs.gitlab.com/ee/development/documentation/styleguide.html#language +level: suggestion +nonword: false +ignorecase: true +swap: + + # Common contractions are ok + it is: it's + can not: can't + cannot: can't + do not: don't + have not: haven't + that is: that's + we are: we're + would not: wouldn't + you are: you're + you have: you've diff --git a/doc/.vale/gitlab/FutureTense.yml b/doc/.vale/gitlab/FutureTense.yml index a53a7dd29cc..4a74ee87331 100644 --- a/doc/.vale/gitlab/FutureTense.yml +++ b/doc/.vale/gitlab/FutureTense.yml @@ -11,7 +11,7 @@ ignorecase: true level: warning link: https://docs.gitlab.com/ee/development/documentation/styleguide.html#language-to-avoid raw: - - "(going to( |\n|[[:punct:]])[a-zA-Z]*|" - - "will( |\n|[[:punct:]])[a-zA-Z]*|" - - "won't( |\n|[[:punct:]])[a-zA-Z]*|" - - "[a-zA-Z]*'ll( |\n|[[:punct:]])[a-zA-Z]*)" + - "(going to( |\n|[[:punct:]])[a-zA-Z]*|" + - "will( |\n|[[:punct:]])[a-zA-Z]*|" + - "won't( |\n|[[:punct:]])[a-zA-Z]*|" + - "[a-zA-Z]*'ll( |\n|[[:punct:]])[a-zA-Z]*)" diff --git a/doc/.vale/gitlab/OutdatedVersions.yml b/doc/.vale/gitlab/OutdatedVersions.yml index 3252481a523..1bc0bf58f90 100644 --- a/doc/.vale/gitlab/OutdatedVersions.yml +++ b/doc/.vale/gitlab/OutdatedVersions.yml @@ -3,7 +3,7 @@ # # Checks for references to versions of GitLab that are no longer supported. # -# For a list of all options, see https://errata-ai.github.io/vale/styles/ +# For a list of all options, see https://errata-ai.gitbook.io/vale/getting-started/styles extends: existence message: 'Can this reference to "%s" be refactored?' link: https://docs.gitlab.com/ee/development/documentation/styleguide.html#importance-of-referencing-gitlab-versions-and-tiers diff --git a/doc/.vale/gitlab/spelling-exceptions.txt b/doc/.vale/gitlab/spelling-exceptions.txt index 28da80557ec..1301e8c4ca1 100644 --- a/doc/.vale/gitlab/spelling-exceptions.txt +++ b/doc/.vale/gitlab/spelling-exceptions.txt @@ -191,6 +191,7 @@ http https idempotence idmapper +inclusivity Ingress initializer initializers @@ -308,6 +309,8 @@ Piwik PgBouncer plaintext Poedit +polyfill +polyfills pooler PostgreSQL precompile @@ -330,6 +333,7 @@ profiler Prometheus proxied proxies +proxyable proxying pseudonymized pseudonymizer @@ -384,6 +388,7 @@ reverify Rubix Rubocop Rubular +ruleset runbook runbooks runit @@ -464,6 +469,7 @@ TypeScript Twilio Twitter Ubuntu +unapplied unarchive unarchived unarchives diff --git a/doc/README.md b/doc/README.md index 115026dae6e..b73300accab 100644 --- a/doc/README.md +++ b/doc/README.md @@ -50,12 +50,12 @@ Have a look at some of our most popular topics: ## The entire DevOps Lifecycle GitLab is the first single application for software development, security, -and operations that enables [Concurrent DevOps](https://about.gitlab.com/concurrent-devops/), +and operations that enables [Concurrent DevOps](https://about.gitlab.com/topics/concurrent-devops/), making the software lifecycle faster and radically improving the speed of business. GitLab provides solutions for [each of the stages of the DevOps lifecycle](https://about.gitlab.com/stages-devops-lifecycle/): - + GitLab is like a top-of-the-line kitchen for making software. As the executive chef, you decide what software you want to serve. Using your recipe, GitLab handles @@ -71,10 +71,11 @@ The following sections provide links to documentation for each DevOps stage: | [Create](#create) | Source code, data creation, and management features. | | [Verify](#verify) | Testing, code quality, and continuous integration features. | | [Package](#package) | Docker container registry. | +| [Secure](#secure) | Security capability features. | | [Release](#release) | Application release and delivery features. | | [Configure](#configure) | Application and infrastructure configuration tools. | | [Monitor](#monitor) | Application monitoring and metrics features. | -| [Secure](#secure) | Security capability features. | +| [Defend](#defend) | Protection against security intrusions. | <div align="right"> <a type="button" class="btn btn-default" href="#overview"> @@ -155,7 +156,8 @@ The following documentation relates to the DevOps **Create** stage: | [File locking](user/project/file_lock.md) **(PREMIUM)** | Lock files to avoid merge conflicts. | | [GitLab Pages](user/project/pages/index.md) | Build, test, and deploy your static website with GitLab Pages. | | [Groups](user/group/index.md) and [Subgroups](user/group/subgroups/index.md) | Organize your projects in groups. | -| [Issues Analytics](user/group/issues_analytics/index.md) **(PREMIUM)** | Check how many issues were created per month. | +| [Issue Analytics](user/group/issues_analytics/index.md) **(PREMIUM)** | Check how many issues were created per month. | +| [Merge Request Analytics](user/analytics/merge_request_analytics.md) **(PREMIUM)** | Check your throughput productivity - how many merge requests were merged per month. | | [Projects](user/project/index.md), including [project access](public_access/public_access.md)<br/>and [settings](user/project/settings/index.md) | Host source code, and control your project's visibility and set configuration. | | [Search through GitLab](user/search/index.md) | Search for issues, merge requests, projects, groups, and todos. | | [Snippets](user/snippets.md) | Snippets allow you to create little bits of code. | @@ -212,7 +214,8 @@ The following documentation relates to the DevOps **Create** stage: | Create topics - Integration and Automation | Description | |:------------------------------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------| -| [GitLab API](api/README.md) | Integrate GitLab via a simple and powerful API. | +| [GitLab REST API](api/README.md) | Integrate with GitLab using our REST API. | +| [GitLab GraphQL API](api/graphql/index.md) | Integrate with GitLab using our GraphQL API. | | [GitLab Integration](integration/README.md) | Integrate with multiple third-party services with GitLab to allow external issue trackers and external authentication. | | [GitLab Webhooks](user/project/integrations/webhooks.md) | Let GitLab notify you when new code has been pushed to your project. | | [Jira Development Panel](integration/jira_development_panel.md) **(PREMIUM)** | See GitLab information in the Jira Development Panel. | @@ -264,9 +267,9 @@ The following documentation relates to the DevOps **Package** stage: |:----------------------------------------------------------------|:-------------------------------------------------------| | [Container Registry](user/packages/container_registry/index.md) | The GitLab Container Registry enables every project in GitLab to have its own space to store [Docker](https://www.docker.com/) images. | | [Dependency Proxy](user/packages/dependency_proxy/index.md) **(PREMIUM)** | The GitLab Dependency Proxy sets up a local proxy for frequently used upstream images/packages. | -| [Conan Repository](user/packages/conan_repository/index.md) **(PREMIUM)** | The GitLab Conan Repository enables every project in GitLab to have its own space to store [Conan](https://conan.io/) packages. | -| [Maven Repository](user/packages/maven_repository/index.md) **(PREMIUM)** | The GitLab Maven Repository enables every project in GitLab to have its own space to store [Maven](https://maven.apache.org/) packages. | -| [NPM Registry](user/packages/npm_registry/index.md) **(PREMIUM)** | The GitLab NPM Registry enables every project in GitLab to have its own space to store [NPM](https://www.npmjs.com/) packages. | +| [Conan Repository](user/packages/conan_repository/index.md) | The GitLab Conan Repository enables every project in GitLab to have its own space to store [Conan](https://conan.io/) packages. | +| [Maven Repository](user/packages/maven_repository/index.md) | The GitLab Maven Repository enables every project in GitLab to have its own space to store [Maven](https://maven.apache.org/) packages. | +| [NPM Registry](user/packages/npm_registry/index.md) | The GitLab NPM Registry enables every project in GitLab to have its own space to store [NPM](https://www.npmjs.com/) packages. | <div align="right"> <a type="button" class="btn btn-default" href="#overview"> @@ -274,6 +277,30 @@ The following documentation relates to the DevOps **Package** stage: </a> </div> +### Secure + +Check your application for security vulnerabilities that may lead to unauthorized access, data +leaks, or denial of service. GitLab can perform static and dynamic tests on your application's code, +looking for known flaws and reporting them in the merge request. You can then fix flaws prior to +merge. Security teams can use dashboards to get a high-level view on projects and groups, and start +remediation processes when needed. + +The following documentation relates to the DevOps **Secure** stage: + +| Secure topics | Description | +|:------------------------------------------------------------------------------------------------------|:-----------------------------------------------------------------------| +| [Compliance Dashboard](user/compliance/compliance_dashboard/index.md) **(ULTIMATE)** | View the most recent Merge Request activity in a group. | +| [Container Scanning](user/application_security/container_scanning/index.md) **(ULTIMATE)** | Use Clair to scan Docker images for known vulnerabilities. | +| [Dependency List](user/application_security/dependency_list/index.md) **(ULTIMATE)** | View your project's dependencies and their known vulnerabilities. | +| [Dependency Scanning](user/application_security/dependency_scanning/index.md) **(ULTIMATE)** | Analyze your dependencies for known vulnerabilities. | +| [Dynamic Application Security Testing (DAST)](user/application_security/dast/index.md) **(ULTIMATE)** | Analyze running web applications for known vulnerabilities. | +| [Group Security Dashboard](user/application_security/security_dashboard/index.md#group-security-dashboard) **(ULTIMATE)** | View vulnerabilities in all the projects in a group and its subgroups. | +| [Instance Security Dashboard](user/application_security/security_dashboard/index.md#instance-security-dashboard) **(ULTIMATE)** | View vulnerabilities in all the projects you're interested in. | +| [License Compliance](user/compliance/license_compliance/index.md) **(ULTIMATE)** | Search your project's dependencies for their licenses. | +| [Pipeline Security](user/application_security/security_dashboard/index.md#pipeline-security) **(ULTIMATE)** | View the security reports for your project's pipelines. | +| [Project Security Dashboard](user/application_security/security_dashboard/index.md#project-security-dashboard) **(ULTIMATE)** | View the latest security reports for your project. | +| [Static Application Security Testing (SAST)](user/application_security/sast/index.md) **(ULTIMATE)** | Analyze source code for known vulnerabilities. | + ### Release Spend less time configuring your tools, and more time creating. Whether you’re @@ -344,7 +371,7 @@ The following documentation relates to the DevOps **Monitor** stage: | [Health check](user/admin_area/monitoring/health_check.md) | GitLab provides liveness and readiness probes to indicate service health and reachability to required services. | | [Prometheus project integration](user/project/integrations/prometheus.md) | Configure the Prometheus integration per project and monitor your CI/CD environments. | | [Prometheus metrics](user/project/integrations/prometheus_library/index.md) | Let Prometheus collect metrics from various services, like Kubernetes, NGINX, NGINX Ingress controller, HAProxy, and Amazon Cloud Watch. | -| [Incident management](user/incident_management/index.md) | Use GitLab to help you better respond to incidents that may occur in your systems. | +| [Incident management](operations/incident_management/index.md) | Use GitLab to help you better respond to incidents that may occur in your systems. | <div align="right"> <a type="button" class="btn btn-default" href="#overview"> @@ -352,29 +379,22 @@ The following documentation relates to the DevOps **Monitor** stage: </a> </div> -### Secure +### Defend -Check your application for security vulnerabilities that may lead to unauthorized access, -data leaks, and denial of services. GitLab will perform static and dynamic tests on the -code of your application, looking for known flaws and report them in the merge request -so you can fix them before merging. Security teams can use dashboards to get a -high-level view on projects and groups, and start remediation processes when needed. +GitLab Defend enables organizations to proactively protect cloud-native environments by providing +context-aware technologies to reduce overall security risk. Defend is a natural extension of your +existing operation's practices and provides security visibility across the entire DevSecOps +lifecycle. This visibility empowers your organization to apply DevSecOps best practices from the +first line of code written and extends all the way through to greater monitoring and protection for +your applications that are deployed in production. -The following documentation relates to the DevOps **Secure** stage: +The following documentation relates to the DevOps **Defend** stage: -| Secure topics | Description | +| Defend topics | Description | |:------------------------------------------------------------------------------------------------------|:-----------------------------------------------------------------------| -| [Compliance Dashboard](user/compliance/compliance_dashboard/index.md) **(ULTIMATE)** | View the most recent Merge Request activity in a group. | -| [Container Scanning](user/application_security/container_scanning/index.md) **(ULTIMATE)** | Use Clair to scan Docker images for known vulnerabilities. | -| [Dependency List](user/application_security/dependency_list/index.md) **(ULTIMATE)** | View your project's dependencies and their known vulnerabilities. | -| [Dependency Scanning](user/application_security/dependency_scanning/index.md) **(ULTIMATE)** | Analyze your dependencies for known vulnerabilities. | -| [Dynamic Application Security Testing (DAST)](user/application_security/dast/index.md) **(ULTIMATE)** | Analyze running web applications for known vulnerabilities. | -| [Group Security Dashboard](user/application_security/security_dashboard/index.md#group-security-dashboard) **(ULTIMATE)** | View vulnerabilities in all the projects in a group and its subgroups. | -| [Instance Security Dashboard](user/application_security/security_dashboard/index.md#instance-security-dashboard) **(ULTIMATE)** | View vulnerabilities in all the projects you're interested in. | -| [License Compliance](user/compliance/license_compliance/index.md) **(ULTIMATE)** | Search your project's dependencies for their licenses. | -| [Pipeline Security](user/application_security/security_dashboard/index.md#pipeline-security) **(ULTIMATE)** | View the security reports for your project's pipelines. | -| [Project Security Dashboard](user/application_security/security_dashboard/index.md#project-security-dashboard) **(ULTIMATE)** | View the latest security reports for your project. | -| [Static Application Security Testing (SAST)](user/application_security/sast/index.md) **(ULTIMATE)** | Analyze source code for known vulnerabilities. | +| [Web Application Firewall with ModSecurity](user/compliance/compliance_dashboard/index.md) | Filter, monitor, and block HTTP traffic to and from a web application. | +| [Container Host Security](user/clusters/applications.md#install-falco-using-gitlab-cicd) | Detect and respond to security threats at the Kubernetes, network, and host level. | +| [Container Network Security](user/clusters/applications.md#install-cilium-using-gitlab-cicd) | Detect and block unauthorized network traffic between pods and to/from the internet.| ## New to Git and GitLab? @@ -450,7 +470,8 @@ There are many ways to integrate with GitLab, including: | Topic | Description | |:-----------------------------------------------------------|:------------------------------------------------| -| [GitLab API](api/README.md) | Integrate GitLab via a simple and powerful API. | +| [GitLab REST API](api/README.md) | Integrate with GitLab using our REST API. | +| [GitLab GraphQL API](api/graphql/index.md) | Integrate with GitLab using our GraphQL API. | | [Integrations and automation](#integration-and-automation) | All GitLab integration and automation options. | <div align="right"> diff --git a/doc/administration/audit_events.md b/doc/administration/audit_events.md index c85fb2d2e47..e7eab5a291e 100644 --- a/doc/administration/audit_events.md +++ b/doc/administration/audit_events.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Analytics +group: Compliance info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- @@ -48,24 +48,25 @@ You need Owner [permissions](../user/permissions.md) to view the group Audit Eve To view a group's audit events, navigate to **Group > Settings > Audit Events**. From there, you can see the following actions: -- Group name or path changed -- Group repository size limit changed -- Group created or deleted -- Group changed visibility -- User was added to group and with which [permissions](../user/permissions.md) -- User sign-in via [Group SAML](../user/group/saml_sso/index.md) -- Permissions changes of a user assigned to a group -- Removed user from group -- Project repository imported into group +- Group name or path changed. +- Group repository size limit changed. +- Group created or deleted. +- Group changed visibility. +- User was added to group and with which [permissions](../user/permissions.md). +- User sign-in via [Group SAML](../user/group/saml_sso/index.md). +- Permissions changes of a user assigned to a group. +- Removed user from group. +- Project repository imported into group. - [Project shared with group](../user/project/members/share_project_with_groups.md) - and with which [permissions](../user/permissions.md) -- Removal of a previously shared group with a project -- LFS enabled or disabled -- Shared runners minutes limit changed -- Membership lock enabled or disabled -- Request access enabled or disabled -- 2FA enforcement or grace period changed -- Roles allowed to create project changed + and with which [permissions](../user/permissions.md). +- Removal of a previously shared group with a project. +- LFS enabled or disabled. +- Shared runners minutes limit changed. +- Membership lock enabled or disabled. +- Request access enabled or disabled. +- 2FA enforcement or grace period changed. +- Roles allowed to create project changed. +- Group CI/CD variable added, removed, or protected status changed. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/30857) in GitLab 13.3. Group events can also be accessed via the [Group Audit Events API](../api/audit_events.md#group-audit-events-starter) diff --git a/doc/administration/audit_reports.md b/doc/administration/audit_reports.md new file mode 100644 index 00000000000..d5a08b711be --- /dev/null +++ b/doc/administration/audit_reports.md @@ -0,0 +1,31 @@ +--- +stage: Manage +group: Compliance +description: 'Learn how to create evidence artifacts typically requested by a 3rd party auditor.' +--- + +# Audit Reports + +GitLab can help owners and administrators respond to auditors by generating +comprehensive reports. These **Audit Reports** vary in scope, depending on the +need: + +## Use cases + +- Generate a report of audit events to provide to an external auditor requesting proof of certain logging capabilities. +- Provide a report of all users showing their group and project memberships for a quarterly access review so the auditor can verify compliance with an organization's access management policy. + +## APIs + +- `https://docs.gitlab.com/ee/api/audit_events.html` +- `https://docs.gitlab.com/ee/api/graphql/reference/#user` +- `https://docs.gitlab.com/ee/api/graphql/reference/#groupmember` +- `https://docs.gitlab.com/ee/api/graphql/reference/#projectmember` + +## Features + +- `https://docs.gitlab.com/ee/administration/audit_events.html` +- `https://docs.gitlab.com/ee/administration/logs.html` + +We plan on making Audit Events [downloadable as a CSV](https://gitlab.com/gitlab-org/gitlab/-/issues/1449) +in the near future. diff --git a/doc/administration/auth/ldap/index.md b/doc/administration/auth/ldap/index.md index aef6c70ff92..548e734c931 100644 --- a/doc/administration/auth/ldap/index.md +++ b/doc/administration/auth/ldap/index.md @@ -548,7 +548,7 @@ or more LDAP group links](#adding-group-links-starter-only). ### Adding group links **(STARTER ONLY)** -For information on adding group links via CNs and filters, refer to [the GitLab groups documentation](../../../user/group/index.md#manage-group-memberships-via-ldap). +For information on adding group links via CNs and filters, refer to [the GitLab groups documentation](../../../user/group/index.md#manage-group-memberships-via-ldap-starter-only). ### Administrator sync **(STARTER ONLY)** @@ -596,6 +596,8 @@ group, as opposed to the full DN. ### Global group memberships lock **(STARTER ONLY)** +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/1793) in GitLab 12.0. + "Lock memberships to LDAP synchronization" setting allows instance administrators to lock down user abilities to invite new members to a group. diff --git a/doc/administration/auth/okta.md b/doc/administration/auth/okta.md index 78b10aedb77..7a55e127733 100644 --- a/doc/administration/auth/okta.md +++ b/doc/administration/auth/okta.md @@ -156,7 +156,7 @@ You might want to try this out on an incognito browser window. ## Configuring groups ->**Note:** +NOTE: **Note:** Make sure the groups exist and are assigned to the Okta app. You can take a look of the [SAML documentation](../../integration/saml.md#saml-groups) on configuring groups. diff --git a/doc/administration/auth/smartcard.md b/doc/administration/auth/smartcard.md index 80d2efbad84..0ecf3ca090d 100644 --- a/doc/administration/auth/smartcard.md +++ b/doc/administration/auth/smartcard.md @@ -310,6 +310,10 @@ attribute. As a prerequisite, you must use an LDAP server that: 1. Save the file and [restart](../restart_gitlab.md#installations-from-source) GitLab for the changes to take effect. +## Passwords for users created via smartcard authentication + +The [Generated passwords for users created through integrated authentication](../../security/passwords_for_integrated_authentication_methods.md) guide provides an overview of how GitLab generates and sets passwords for users created via smartcard authentication. + <!-- ## Troubleshooting Include any troubleshooting steps that you can foresee. If you know beforehand what issues diff --git a/doc/administration/consul.md b/doc/administration/consul.md new file mode 100644 index 00000000000..ae22d8bd4d0 --- /dev/null +++ b/doc/administration/consul.md @@ -0,0 +1,239 @@ +--- +type: reference +--- + +# How to set up Consul **(PREMIUM ONLY)** + +A Consul cluster consists of both +[server and client agents](https://www.consul.io/docs/agent). +The servers run on their own nodes and the clients run on other nodes that in +turn communicate with the servers. + +GitLab Premium includes a bundled version of [Consul](https://www.consul.io/) +a service networking solution that you can manage by using `/etc/gitlab/gitlab.rb`. + +## Configure the Consul nodes + +NOTE: **Important:** +Before proceeding, refer to the +[available reference architectures](reference_architectures/index.md#available-reference-architectures) +to find out how many Consul server nodes you should have. + +On **each** Consul server node perform the following: + +1. Follow the instructions to [install](https://about.gitlab.com/install/) + GitLab by choosing your preferred platform, but do not supply the + `EXTERNAL_URL` value when asked. +1. Edit `/etc/gitlab/gitlab.rb`, and add the following by replacing the values + noted in the `retry_join` section. In the example below, there are three + nodes, two denoted with their IP, and one with its FQDN, you can use either + notation: + + ```ruby + # Disable all components except Consul + roles ['consul_role'] + + # Consul nodes: can be FQDN or IP, separated by a whitespace + consul['configuration'] = { + server: true, + retry_join: %w(10.10.10.1 consul1.gitlab.example.com 10.10.10.2) + } + + # Disable auto migrations + gitlab_rails['auto_migrate'] = false + ``` + +1. [Reconfigure GitLab](restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes + to take effect. +1. Run the following command to ensure Consul is both configured correctly and + to verify that all server nodes are communicating: + + ```shell + sudo /opt/gitlab/embedded/bin/consul members + ``` + + The output should be similar to: + + ```plaintext + Node Address Status Type Build Protocol DC + CONSUL_NODE_ONE XXX.XXX.XXX.YYY:8301 alive server 0.9.2 2 gitlab_consul + CONSUL_NODE_TWO XXX.XXX.XXX.YYY:8301 alive server 0.9.2 2 gitlab_consul + CONSUL_NODE_THREE XXX.XXX.XXX.YYY:8301 alive server 0.9.2 2 gitlab_consul + ``` + + If the results display any nodes with a status that isn't `alive`, or if any + of the three nodes are missing, see the [Troubleshooting section](#troubleshooting-consul). + +## Upgrade the Consul nodes + +To upgrade your Consul nodes, upgrade the GitLab package. + +Nodes should be: + +- Members of a healthy cluster prior to upgrading the Omnibus GitLab package. +- Upgraded one node at a time. + +Identify any existing health issues in the cluster by running the following command +within each node. The command will return an empty array if the cluster is healthy: + +```shell +curl http://127.0.0.1:8500/v1/health/state/critical +``` + +Consul nodes communicate using the raft protocol. If the current leader goes +offline, there needs to be a leader election. A leader node must exist to facilitate +synchronization across the cluster. If too many nodes go offline at the same time, +the cluster will lose quorum and not elect a leader due to +[broken consensus](https://www.consul.io/docs/internals/consensus.html). + +Consult the [troubleshooting section](#troubleshooting-consul) if the cluster is not +able to recover after the upgrade. The [outage recovery](#outage-recovery) may +be of particular interest. + +NOTE: **Note:** +GitLab uses Consul to store only transient data that is easily regenerated. If +the bundled Consul was not used by any process other than GitLab itself, then +[rebuilding the cluster from scratch](#recreate-from-scratch) is fine. + +## Troubleshooting Consul + +Below are some useful operations should you need to debug any issues. +You can see any error logs by running: + +```shell +sudo gitlab-ctl tail consul +``` + +### Check the cluster membership + +To determine which nodes are part of the cluster, run the following on any member in the cluster: + +```shell +sudo /opt/gitlab/embedded/bin/consul members +``` + +The output should be similar to: + +```plaintext +Node Address Status Type Build Protocol DC +consul-b XX.XX.X.Y:8301 alive server 0.9.0 2 gitlab_consul +consul-c XX.XX.X.Y:8301 alive server 0.9.0 2 gitlab_consul +consul-c XX.XX.X.Y:8301 alive server 0.9.0 2 gitlab_consul +db-a XX.XX.X.Y:8301 alive client 0.9.0 2 gitlab_consul +db-b XX.XX.X.Y:8301 alive client 0.9.0 2 gitlab_consul +``` + +Ideally all nodes will have a `Status` of `alive`. + +### Restart Consul + +If it is necessary to restart Consul, it is important to do this in +a controlled manner to maintain quorum. If quorum is lost, to recover the cluster, +you will need to follow the Consul [outage recovery](#outage-recovery) process. + +To be safe, it's recommended that you only restart Consul in one node at a time to +ensure the cluster remains intact. For larger clusters, it is possible to restart +multiple nodes at a time. See the +[Consul consensus document](https://www.consul.io/docs/internals/consensus.html#deployment-table) +for how many failures it can tolerate. This will be the number of simultaneous +restarts it can sustain. + +To restart Consul: + +```shell +sudo gitlab-ctl restart consul +``` + +### Consul nodes unable to communicate + +By default, Consul will attempt to +[bind](https://www.consul.io/docs/agent/options.html#_bind) to `0.0.0.0`, but +it will advertise the first private IP address on the node for other Consul nodes +to communicate with it. If the other nodes cannot communicate with a node on +this address, then the cluster will have a failed status. + +If you are running into this issue, you will see messages like the following in `gitlab-ctl tail consul` output: + +```plaintext +2017-09-25_19:53:39.90821 2017/09/25 19:53:39 [WARN] raft: no known peers, aborting election +2017-09-25_19:53:41.74356 2017/09/25 19:53:41 [ERR] agent: failed to sync remote state: No cluster leader +``` + +To fix this: + +1. Pick an address on each node that all of the other nodes can reach this node through. +1. Update your `/etc/gitlab/gitlab.rb` + + ```ruby + consul['configuration'] = { + ... + bind_addr: 'IP ADDRESS' + } + ``` + +1. Reconfigure GitLab; + + ```shell + gitlab-ctl reconfigure + ``` + +If you still see the errors, you may have to +[erase the Consul database and reinitialize](#recreate-from-scratch) on the affected node. + +### Consul does not start - multiple private IPs + +In case that a node has multiple private IPs, Consul will be confused as to +which of the private addresses to advertise, and then immediately exit on start. + +You will see messages like the following in `gitlab-ctl tail consul` output: + +```plaintext +2017-11-09_17:41:45.52876 ==> Starting Consul agent... +2017-11-09_17:41:45.53057 ==> Error creating agent: Failed to get advertise address: Multiple private IPs found. Please configure one. +``` + +To fix this: + +1. Pick an address on the node that all of the other nodes can reach this node through. +1. Update your `/etc/gitlab/gitlab.rb` + + ```ruby + consul['configuration'] = { + ... + bind_addr: 'IP ADDRESS' + } + ``` + +1. Reconfigure GitLab; + + ```shell + gitlab-ctl reconfigure + ``` + +### Outage recovery + +If you lost enough Consul nodes in the cluster to break quorum, then the cluster +is considered failed, and it will not function without manual intervention. +In that case, you can either recreate the nodes from scratch or attempt a +recover. + +#### Recreate from scratch + +By default, GitLab does not store anything in the Consul node that cannot be +recreated. To erase the Consul database and reinitialize: + +```shell +sudo gitlab-ctl stop consul +sudo rm -rf /var/opt/gitlab/consul/data +sudo gitlab-ctl start consul +``` + +After this, the node should start back up, and the rest of the server agents rejoin. +Shortly after that, the client agents should rejoin as well. + +#### Recover a failed node + +If you have taken advantage of Consul to store other data and want to restore +the failed node, follow the +[Consul guide](https://learn.hashicorp.com/tutorials/consul/recovery-outage) +to recover a failed cluster. diff --git a/doc/administration/environment_variables.md b/doc/administration/environment_variables.md index ecf9464eb91..ebc3848017f 100644 --- a/doc/administration/environment_variables.md +++ b/doc/administration/environment_variables.md @@ -33,6 +33,7 @@ Variable | Type | Description `GITLAB_UNICORN_MEMORY_MIN` | integer | The minimum memory threshold (in bytes) for the Unicorn worker killer `GITLAB_UNICORN_MEMORY_MAX` | integer | The maximum memory threshold (in bytes) for the Unicorn worker killer `GITLAB_SHARED_RUNNERS_REGISTRATION_TOKEN` | string | Sets the initial registration token used for GitLab Runners +`UNSTRUCTURED_RAILS_LOG` | string | Enables the unstructured log in addition to JSON logs (defaults to `true`) ## Complete database variables diff --git a/doc/administration/feature_flags.md b/doc/administration/feature_flags.md index 678ab6c5d7b..7cc7692975a 100644 --- a/doc/administration/feature_flags.md +++ b/doc/administration/feature_flags.md @@ -103,10 +103,10 @@ Some feature flags can be enabled or disabled on a per project basis: Feature.enable(:<feature flag>, Project.find(<project id>)) ``` -For example, to enable the [`:junit_pipeline_view`](../ci/junit_test_reports.md#enabling-the-junit-test-reports-feature-core-only) feature flag for project `1234`: +For example, to enable the [`:product_analytics`](../operations/product_analytics.md#enable-or-disable-product-analytics) feature flag for project `1234`: ```ruby -Feature.enable(:junit_pipeline_view, Project.find(1234)) +Feature.enable(:product_analytics, Project.find(1234)) ``` `Feature.enable` and `Feature.disable` always return `nil`, this is not an indication that the command failed: diff --git a/doc/administration/file_hooks.md b/doc/administration/file_hooks.md index 7903da675fd..c0b31769e7f 100644 --- a/doc/administration/file_hooks.md +++ b/doc/administration/file_hooks.md @@ -1,3 +1,10 @@ +--- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference +--- + # File hooks > - Introduced in GitLab 10.6. diff --git a/doc/administration/geo/disaster_recovery/background_verification.md b/doc/administration/geo/disaster_recovery/background_verification.md index 88218d24b2f..89a93e45b1d 100644 --- a/doc/administration/geo/disaster_recovery/background_verification.md +++ b/doc/administration/geo/disaster_recovery/background_verification.md @@ -58,14 +58,14 @@ Feature.enable('geo_repository_verification') ## Repository verification -Navigate to the **{admin}** **Admin Area >** **{location-dot}** **Geo** dashboard on the **primary** node and expand +Navigate to the **Admin Area > Geo** dashboard on the **primary** node and expand the **Verification information** tab for that node to view automatic checksumming status for repositories and wikis. Successes are shown in green, pending work in gray, and failures in red.  -Navigate to the **{admin}** **Admin Area >** **{location-dot}** **Geo** dashboard on the **secondary** node and expand +Navigate to the **Admin Area > Geo** dashboard on the **secondary** node and expand the **Verification information** tab for that node to view automatic verification status for repositories and wikis. As with checksumming, successes are shown in green, pending work in gray, and failures in red. @@ -92,7 +92,7 @@ data. The default and recommended re-verification interval is 7 days, though an interval as short as 1 day can be set. Shorter intervals reduce risk but increase load and vice versa. -Navigate to the **{admin}** **Admin Area >** **{location-dot}** **Geo** dashboard on the **primary** node, and +Navigate to the **Admin Area > Geo** dashboard on the **primary** node, and click the **Edit** button for the **primary** node to customize the minimum re-verification interval: @@ -141,7 +141,7 @@ sudo gitlab-rake geo:verification:wiki:reset If the **primary** and **secondary** nodes have a checksum verification mismatch, the cause may not be apparent. To find the cause of a checksum mismatch: -1. Navigate to the **{admin}** **Admin Area >** **{overview}** **Overview > Projects** dashboard on the **primary** node, find the +1. Navigate to the **Admin Area > Overview > Projects** dashboard on the **primary** node, find the project that you want to check the checksum differences and click on the **Edit** button:  diff --git a/doc/administration/geo/disaster_recovery/planned_failover.md b/doc/administration/geo/disaster_recovery/planned_failover.md index ef0e67434d0..a0cf263a762 100644 --- a/doc/administration/geo/disaster_recovery/planned_failover.md +++ b/doc/administration/geo/disaster_recovery/planned_failover.md @@ -110,7 +110,7 @@ The maintenance window won't end until Geo replication and verification is completely finished. To keep the window as short as possible, you should ensure these processes are close to 100% as possible during active use. -Navigate to the **{admin}** **Admin Area >** **{location-dot}** **Geo** dashboard on the **secondary** node to +Navigate to the **Admin Area > Geo** dashboard on the **secondary** node to review status. Replicated objects (shown in green) should be close to 100%, and there should be no failures (shown in red). If a large proportion of objects aren't yet replicated (shown in gray), consider giving the node more @@ -135,8 +135,8 @@ This [content was moved to another location](background_verification.md). ### Notify users of scheduled maintenance -On the **primary** node, navigate to **{admin}** **Admin Area >** **{bullhorn}** **Messages**, add a broadcast -message. You can check under **{admin}** **Admin Area >** **{location-dot}** **Geo** to estimate how long it +On the **primary** node, navigate to **Admin Area > Messages**, add a broadcast +message. You can check under **Admin Area > Geo** to estimate how long it will take to finish syncing. An example message would be: > A scheduled maintenance will take place at XX:XX UTC. We expect it to take @@ -181,7 +181,7 @@ access to the **primary** node during the maintenance window. connection. 1. Disable non-Geo periodic background jobs on the **primary** node by navigating - to **{admin}** **Admin Area >** **{monitor}** **Monitoring > Background Jobs > Cron**, pressing `Disable All`, + to **Admin Area > Monitoring > Background Jobs > Cron**, pressing `Disable All`, and then pressing `Enable` for the `geo_sidekiq_cron_config_worker` cron job. This job will re-enable several other cron jobs that are essential for planned failover to complete successfully. @@ -190,11 +190,11 @@ access to the **primary** node during the maintenance window. 1. If you are manually replicating any data not managed by Geo, trigger the final replication process now. -1. On the **primary** node, navigate to **{admin}** **Admin Area >** **{monitor}** **Monitoring > Background Jobs > Queues** +1. On the **primary** node, navigate to **Admin Area > Monitoring > Background Jobs > Queues** and wait for all queues except those with `geo` in the name to drop to 0. These queues contain work that has been submitted by your users; failing over before it is completed will cause the work to be lost. -1. On the **primary** node, navigate to **{admin}** **Admin Area >** **{location-dot}** **Geo** and wait for the +1. On the **primary** node, navigate to **Admin Area > Geo** and wait for the following conditions to be true of the **secondary** node you are failing over to: - All replication meters to each 100% replicated, 0% failures. @@ -202,7 +202,7 @@ access to the **primary** node during the maintenance window. - Database replication lag is 0ms. - The Geo log cursor is up to date (0 events behind). -1. On the **secondary** node, navigate to **{admin}** **Admin Area >** **{monitor}** **Monitoring > Background Jobs > Queues** +1. On the **secondary** node, navigate to **Admin Area > Monitoring > Background Jobs > Queues** and wait for all the `geo` queues to drop to 0 queued and 0 running jobs. 1. On the **secondary** node, use [these instructions](../../raketasks/check.md) to verify the integrity of CI artifacts, LFS objects, and uploads in file diff --git a/doc/administration/geo/replication/configuration.md b/doc/administration/geo/replication/configuration.md index 3d08ed81700..74fa8e3b8f2 100644 --- a/doc/administration/geo/replication/configuration.md +++ b/doc/administration/geo/replication/configuration.md @@ -191,17 +191,16 @@ keys must be manually replicated to the **secondary** node. gitlab-ctl reconfigure ``` -1. Visit the **primary** node's **{admin}** **Admin Area >** **{location-dot}** **Geo** +1. Visit the **primary** node's **Admin Area > Geo** (`/admin/geo/nodes`) in your browser. 1. Click the **New node** button. -  +  1. Fill in **Name** with the `gitlab_rails['geo_node_name']` in `/etc/gitlab/gitlab.rb`. These values must always match *exactly*, character for character. 1. Fill in **URL** with the `external_url` in `/etc/gitlab/gitlab.rb`. These values must always match, but it doesn't matter if one ends with a `/` and the other doesn't. -1. **Do NOT** check the **This is a primary node** checkbox. 1. Optionally, choose which groups or storage shards should be replicated by the **secondary** node. Leave blank to replicate all. Read more in [selective synchronization](#selective-synchronization). @@ -238,7 +237,7 @@ You can login to the **secondary** node with the same credentials as used for th Using Hashed Storage significantly improves Geo replication. Project and group renames no longer require synchronization between nodes. -1. Visit the **primary** node's **{admin}** **Admin Area >** **{settings}** **Settings > Repository** +1. Visit the **primary** node's **Admin Area > Settings > Repository** (`/admin/application_settings/repository`) in your browser. 1. In the **Repository storage** section, check **Use hashed storage paths for newly created and renamed projects**. @@ -255,7 +254,7 @@ on the **secondary** node. ### Step 6. Enable Git access over HTTP/HTTPS Geo synchronizes repositories over HTTP/HTTPS, and therefore requires this clone -method to be enabled. Navigate to **{admin}** **Admin Area >** **{settings}** **Settings** +method to be enabled. Navigate to **Admin Area > Settings** (`/admin/application_settings/general`) on the **primary** node, and set `Enabled Git access protocols` to `Both SSH and HTTP(S)` or `Only HTTP(S)`. @@ -264,7 +263,7 @@ method to be enabled. Navigate to **{admin}** **Admin Area >** **{settings}** ** Your **secondary** node is now configured! You can login to the **secondary** node with the same credentials you used for the -**primary** node. Visit the **secondary** node's **{admin}** **Admin Area >** **{location-dot}** **Geo** +**primary** node. Visit the **secondary** node's **Admin Area > Geo** (`/admin/geo/nodes`) in your browser to check if it's correctly identified as a **secondary** Geo node and if Geo is enabled. diff --git a/doc/administration/geo/replication/database.md b/doc/administration/geo/replication/database.md index 02f51e79907..0bc37ce6438 100644 --- a/doc/administration/geo/replication/database.md +++ b/doc/administration/geo/replication/database.md @@ -37,8 +37,7 @@ recover. See below for more details. The following guide assumes that: - You are using Omnibus and therefore you are using PostgreSQL 11 or later - which includes the [`pg_basebackup` tool](https://www.postgresql.org/docs/11/app-pgbasebackup.html) and improved - [Foreign Data Wrapper](https://www.postgresql.org/docs/11/postgres-fdw.html) support. + which includes the [`pg_basebackup` tool](https://www.postgresql.org/docs/11/app-pgbasebackup.html). - You have a **primary** node already set up (the GitLab server you are replicating from), running Omnibus' PostgreSQL (or equivalent version), and you have a new **secondary** server set up with the same versions of the OS, @@ -167,6 +166,11 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o corresponding to the given address. See [the PostgreSQL documentation](https://www.postgresql.org/docs/11/runtime-config-connection.html) for more details. + NOTE: **Note:** + If you need to use `0.0.0.0` or `*` as the listen_address, you will also need to add + `127.0.0.1/32` to the `postgresql['md5_auth_cidr_addresses']` setting, to allow Rails to connect through + `127.0.0.1`. For more information, see [omnibus-5258](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5258). + Depending on your network configuration, the suggested addresses may not be correct. If your **primary** node and **secondary** nodes connect over a local area network, or a virtual network connecting availability zones like @@ -341,10 +345,10 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o Ensure that the contents of `~gitlab-psql/data/server.crt` on the **primary** node match the contents of `~gitlab-psql/.postgresql/root.crt` on the **secondary** node. -1. Configure PostgreSQL to enable FDW support: +1. Configure PostgreSQL: This step is similar to how we configured the **primary** instance. - We need to enable this, to enable FDW support, even if using a single node. + We need to enable this, even if using a single node. Edit `/etc/gitlab/gitlab.rb` and add the following, replacing the IP addresses with addresses appropriate to your network configuration: @@ -369,11 +373,6 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o ## postgresql['sql_user_password'] = '<md5_hash_of_your_password>' gitlab_rails['db_password'] = '<your_password_here>' - - ## - ## Enable FDW support for the Geo Tracking Database (improves performance) - ## - geo_secondary['db_fdw'] = true ``` For external PostgreSQL instances, see [additional instructions](external_database.md). @@ -385,15 +384,12 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o gitlab-ctl reconfigure ``` -1. Restart PostgreSQL for the IP change to take effect and reconfigure again: +1. Restart PostgreSQL for the IP change to take effect: ```shell gitlab-ctl restart postgresql - gitlab-ctl reconfigure ``` - This last reconfigure will provision the FDW configuration and enable it. - ### Step 3. Initiate the replication process Below we provide a script that connects the database on the **secondary** node to @@ -468,48 +464,6 @@ high-availability configuration with a cluster of nodes supporting a Geo **primary** node and another cluster of nodes supporting a Geo **secondary** node. For more information, see [High Availability with Omnibus GitLab](../../postgresql/replication_and_failover.md). -For a Geo **secondary** node to work properly with PgBouncer in front of the database, -it will need a separate read-only user to make [PostgreSQL FDW queries](https://www.postgresql.org/docs/11/postgres-fdw.html) -work: - -1. On the **primary** Geo database, enter the PostgreSQL on the console as an - admin user. If you are using an Omnibus-managed database, log onto the **primary** - node that is running the PostgreSQL database (the default Omnibus database name is `gitlabhq_production`): - - ```shell - sudo \ - -u gitlab-psql /opt/gitlab/embedded/bin/psql \ - -h /var/opt/gitlab/postgresql gitlabhq_production - ``` - -1. Then create the read-only user: - - ```sql - -- NOTE: Use the password defined earlier - CREATE USER gitlab_geo_fdw WITH password 'mypassword'; - GRANT CONNECT ON DATABASE gitlabhq_production to gitlab_geo_fdw; - GRANT USAGE ON SCHEMA public TO gitlab_geo_fdw; - GRANT SELECT ON ALL TABLES IN SCHEMA public TO gitlab_geo_fdw; - GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO gitlab_geo_fdw; - - -- Tables created by "gitlab" should be made read-only for "gitlab_geo_fdw" - -- automatically. - ALTER DEFAULT PRIVILEGES FOR USER gitlab IN SCHEMA public GRANT SELECT ON TABLES TO gitlab_geo_fdw; - ALTER DEFAULT PRIVILEGES FOR USER gitlab IN SCHEMA public GRANT SELECT ON SEQUENCES TO gitlab_geo_fdw; - ``` - -1. On the **secondary** nodes, change `/etc/gitlab/gitlab.rb`: - - ```ruby - geo_postgresql['fdw_external_user'] = 'gitlab_geo_fdw' - ``` - -1. Save the file and reconfigure GitLab for the changes to be applied: - - ```shell - gitlab-ctl reconfigure - ``` - ## Troubleshooting Read the [troubleshooting document](troubleshooting.md). diff --git a/doc/administration/geo/replication/datatypes.md b/doc/administration/geo/replication/datatypes.md index 5636ff79189..b8d01e80371 100644 --- a/doc/administration/geo/replication/datatypes.md +++ b/doc/administration/geo/replication/datatypes.md @@ -5,7 +5,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w type: howto --- -# Geo data types support +# Geo data types support **(PREMIUM ONLY)** A Geo data type is a specific class of data that is required by one or more GitLab features to store relevant information. @@ -126,6 +126,33 @@ these epics/issues: - [Unreplicated Data Types](https://gitlab.com/groups/gitlab-org/-/epics/893) - [Verify all replicated data](https://gitlab.com/groups/gitlab-org/-/epics/1430) +### Replicated data types behind a feature flag + +The replication for some data types is behind a corresponding feature flag: + +> - They're deployed behind a feature flag, enabled by default. +> - They're enabled on GitLab.com. +> - They can't be enabled or disabled per-project. +> - They are recommended for production use. +> - For GitLab self-managed instances, GitLab administrators can opt to [disable them](#enable-or-disable-replication-for-some-data-types-core-only). **(CORE ONLY)** + +#### Enable or disable replication (for some data types) **(CORE ONLY)** + +Replication for some data types are released behind feature flags that are **enabled by default**. +[GitLab administrators with access to the GitLab Rails console](../../feature_flags.md) can opt to disable it for your instance. You can find feature flag names of each of those data types in the notes column of the table below. + +To disable, such as for package file replication: + +```ruby +Feature.disable(:geo_package_file_replication) +``` + +To enable, such as for package file replication: + +```ruby +Feature.enable(:geo_package_file_replication) +``` + DANGER: **Danger:** Features not on this list, or with **No** in the **Replicated** column, are not replicated on the **secondary** node. Failing over without manually @@ -151,12 +178,12 @@ successfully, you must replicate their data using some other means. | [Elasticsearch integration](../../../integration/elasticsearch.md) | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/1186) | No | | | [GitLab Pages](../../pages/index.md) | [No](https://gitlab.com/groups/gitlab-org/-/epics/589) | No | | | [Container Registry](../../packages/container_registry.md) | **Yes** (12.3) | No | | -| [NPM Registry](../../../user/packages/npm_registry/index.md) | **Yes** (13.2) | No | | -| [Maven Repository](../../../user/packages/maven_repository/index.md) | **Yes** (13.2) | No | | -| [Conan Repository](../../../user/packages/conan_repository/index.md) | **Yes** (13.2) | No | | -| [NuGet Repository](../../../user/packages/nuget_repository/index.md) | **Yes** (13.2) | No | | -| [PyPi Repository](../../../user/packages/pypi_repository/index.md) | **Yes** (13.2) | No | | -| [Composer Repository](../../../user/packages/composer_repository/index.md) | **Yes** (13.2) | No | | +| [NPM Registry](../../../user/packages/npm_registry/index.md) | **Yes** (13.2) | No | Behind feature flag `geo_package_file_replication`, enabled by default | | +| [Maven Repository](../../../user/packages/maven_repository/index.md) | **Yes** (13.2) | No | Behind feature flag `geo_package_file_replication`, enabled by default | | +| [Conan Repository](../../../user/packages/conan_repository/index.md) | **Yes** (13.2) | No | Behind feature flag `geo_package_file_replication`, enabled by default | | +| [NuGet Repository](../../../user/packages/nuget_repository/index.md) | **Yes** (13.2) | No | Behind feature flag `geo_package_file_replication`, enabled by default | | +| [PyPi Repository](../../../user/packages/pypi_repository/index.md) | **Yes** (13.2) | No | Behind feature flag `geo_package_file_replication`, enabled by default | | +| [Composer Repository](../../../user/packages/composer_repository/index.md) | **Yes** (13.2) | No | Behind feature flag `geo_package_file_replication`, enabled by default | | | [External merge request diffs](../../merge_request_diffs.md) | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/33817) | No | | | [Terraform State](../../terraform_state.md) | [No](https://gitlab.com/groups/gitlab-org/-/epics/3112)(*3*) | No | | | [Vulnerability Export](../../../user/application_security/security_dashboard/#export-vulnerabilities) | [No](https://gitlab.com/groups/gitlab-org/-/epics/3111)(*3*) | No | | | diff --git a/doc/administration/geo/replication/disable_geo.md b/doc/administration/geo/replication/disable_geo.md index f53b4c1b796..aed8e5fc3bc 100644 --- a/doc/administration/geo/replication/disable_geo.md +++ b/doc/administration/geo/replication/disable_geo.md @@ -33,7 +33,7 @@ in order to do that. ## Remove the primary node from the UI -1. Go to **{admin}** **Admin Area >** **{location-dot}** **Geo** (`/admin/geo/nodes`). +1. Go to **Admin Area > Geo** (`/admin/geo/nodes`). 1. Click the **Remove** button for the **primary** node. 1. Confirm by clicking **Remove** when the prompt appears. diff --git a/doc/administration/geo/replication/docker_registry.md b/doc/administration/geo/replication/docker_registry.md index c34732cba67..5403edc69da 100644 --- a/doc/administration/geo/replication/docker_registry.md +++ b/doc/administration/geo/replication/docker_registry.md @@ -122,7 +122,7 @@ generate a short-lived JWT that is pull-only-capable to access the ### Verify replication -To verify Container Registry replication is working, go to **{admin}** **Admin Area >** **{location-dot}** **Geo** +To verify Container Registry replication is working, go to **Admin Area > Geo** (`/admin/geo/nodes`) on the **secondary** node. The initial replication, or "backfill", will probably still be in progress. You can monitor the synchronization process on each Geo node from the **primary** node's **Geo Nodes** dashboard in your browser. diff --git a/doc/administration/geo/replication/external_database.md b/doc/administration/geo/replication/external_database.md index e85a82fa414..d860a3dd368 100644 --- a/doc/administration/geo/replication/external_database.md +++ b/doc/administration/geo/replication/external_database.md @@ -183,9 +183,6 @@ to grant additional roles to your tracking database user (by default, this is - Amazon RDS requires the [`rds_superuser`](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Appendix.PostgreSQL.CommonDBATasks.html#Appendix.PostgreSQL.CommonDBATasks.Roles) role. - Azure Database for PostgreSQL requires the [`azure_pg_admin`](https://docs.microsoft.com/en-us/azure/postgresql/howto-create-users#how-to-create-additional-admin-users-in-azure-database-for-postgresql) role. -The tracking database requires an [FDW](https://www.postgresql.org/docs/11/postgres-fdw.html) -connection with the **secondary** replica database for improved performance. - If you have an external database ready to be used as the tracking database, follow the instructions below to use it: @@ -224,7 +221,6 @@ the tracking database on port 5432. geo_secondary['db_host'] = '<tracking_database_host>' geo_secondary['db_port'] = <tracking_database_port> # change to the correct port - geo_secondary['db_fdw'] = true # enable FDW geo_postgresql['enable'] = false # don't use internal managed instance ``` @@ -236,48 +232,3 @@ the tracking database on port 5432. gitlab-rake geo:db:create gitlab-rake geo:db:migrate ``` - -1. Configure the [PostgreSQL FDW](https://www.postgresql.org/docs/11/postgres-fdw.html) - connection and credentials: - - Save the script below in a file, ex. `/tmp/geo_fdw.sh` and modify the connection - parameters to match your environment. Execute it to set up the FDW connection. - - ```shell - #!/bin/bash - - # Secondary Database connection params: - DB_HOST="<public_ip_or_vpc_private_ip>" - DB_NAME="gitlabhq_production" - DB_USER="gitlab" - DB_PASS="<your_password_here>" - DB_PORT="5432" - - # Tracking Database connection params: - GEO_DB_HOST="<public_ip_or_vpc_private_ip>" - GEO_DB_NAME="gitlabhq_geo_production" - GEO_DB_USER="gitlab_geo" - GEO_DB_PORT="5432" - - query_exec () { - gitlab-psql -h $GEO_DB_HOST -U $GEO_DB_USER -d $GEO_DB_NAME -p $GEO_DB_PORT -c "${1}" - } - - query_exec "CREATE EXTENSION postgres_fdw;" - query_exec "CREATE SERVER gitlab_secondary FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '${DB_HOST}', dbname '${DB_NAME}', port '${DB_PORT}');" - query_exec "CREATE USER MAPPING FOR ${GEO_DB_USER} SERVER gitlab_secondary OPTIONS (user '${DB_USER}', password '${DB_PASS}');" - query_exec "CREATE SCHEMA gitlab_secondary;" - query_exec "GRANT USAGE ON FOREIGN SERVER gitlab_secondary TO ${GEO_DB_USER};" - ``` - - NOTE: **Note:** - The script template above uses `gitlab-psql` as it's intended to be executed from the Geo machine, - but you can change it to `psql` and run it from any machine that has access to the database. We also recommend using - `psql` for AWS RDS. - -1. Save the file and [restart GitLab](../../restart_gitlab.md#omnibus-gitlab-restart) -1. Populate the FDW tables: - - ```shell - gitlab-rake geo:db:refresh_foreign_tables - ``` diff --git a/doc/administration/geo/replication/geo_validation_tests.md b/doc/administration/geo/replication/geo_validation_tests.md index 0255e5c9883..323f6f367b1 100644 --- a/doc/administration/geo/replication/geo_validation_tests.md +++ b/doc/administration/geo/replication/geo_validation_tests.md @@ -5,7 +5,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w type: howto --- -# Geo validation tests +# Geo validation tests **(PREMIUM ONLY)** The Geo team performs manual testing and validation on common deployment configurations to ensure that Geo works when upgrading between minor GitLab versions and major PostgreSQL database versions. @@ -27,6 +27,13 @@ The following are GitLab upgrade validation tests we performed. - [Investigate why `reconfigure` and `hup` cause downtime on multi-node Geo deployments](https://gitlab.com/gitlab-org/gitlab/-/issues/228898) - [Geo multi-node deployment upgrade: investigate order when upgrading non-deploy nodes](https://gitlab.com/gitlab-org/gitlab/-/issues/228954) +[Switch from repmgr to Patroni on a Geo primary site](https://gitlab.com/gitlab-org/gitlab/-/issues/224652): + +- Description: Tested switching from repmgr to Patroni on a multi-node Geo primary site. Used [the orchestrator tool](https://gitlab.com/gitlab-org/gitlab-orchestrator) to deploy a Geo installation with 3 database nodes managed by repmgr. With this approach, we were also able to address a related issue for [verifying a Geo installation with Patroni and PostgreSQL 11](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5113). +- Outcome: Partial success. We enabled Patroni on the primary site and set up database replication on the secondary site. However, we found that Patroni would delete the secondary site's replication slot whenever Patroni was restarted. Another issue is that when Patroni elects a new leader in the cluster, the secondary site will fail to automatically follow the new leader. Until these issues are resolved, we cannot officially support and recommend Patroni for Geo installations. +- Follow up issues/actions: + - [Investigate permanent replication slot for Patroni with Geo single node secondary](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5528) + ### June 2020 [Upgrade Geo multi-node installation](https://gitlab.com/gitlab-org/gitlab/-/issues/223284): @@ -107,6 +114,19 @@ The following are GitLab upgrade validation tests we performed. The following are PostgreSQL upgrade validation tests we performed. +### August 2020 + +[Verify Geo installation with PostgreSQL 12](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5453): + +- Description: Prior to PostgreSQL 12 becoming available as an opt-in version in GitLab 13.3, + we tested fresh installations of GitLab 13.3 with PostgreSQL 12 enabled and Geo installed. +- Outcome: Setting up a Geo secondary required manual intervention because the `recovery.conf` file + is no longer supported in PostgreSQL 12. We do not recommend deploying Geo with PostgreSQL 12 until + the appropriate changes have been made to Omnibus and verified. +- Follow up issues: + - [Update `replicate-geo-database` to support PostgreSQL 12](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5575) + - [Remove PostgreSQL 12 check in `replicate-geo-database` for 14.0](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5576) + ### April 2020 [PostgreSQL 11 upgrade procedure for Geo installations](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/4975): diff --git a/doc/administration/geo/replication/img/adding_a_secondary_node.png b/doc/administration/geo/replication/img/adding_a_secondary_node.png Binary files differdeleted file mode 100644 index e33b690da18..00000000000 --- a/doc/administration/geo/replication/img/adding_a_secondary_node.png +++ /dev/null diff --git a/doc/administration/geo/replication/img/adding_a_secondary_node_v13_3.png b/doc/administration/geo/replication/img/adding_a_secondary_node_v13_3.png Binary files differnew file mode 100644 index 00000000000..e43ace99666 --- /dev/null +++ b/doc/administration/geo/replication/img/adding_a_secondary_node_v13_3.png diff --git a/doc/administration/geo/replication/index.md b/doc/administration/geo/replication/index.md index 51c831abaf3..f1cc9f0df8b 100644 --- a/doc/administration/geo/replication/index.md +++ b/doc/administration/geo/replication/index.md @@ -117,7 +117,7 @@ The following are required to run Geo: The following operating systems are known to ship with a current version of OpenSSH: - [CentOS](https://www.centos.org) 7.4+ - [Ubuntu](https://ubuntu.com) 16.04+ -- PostgreSQL 11+ with [FDW](https://www.postgresql.org/docs/11/postgres-fdw.html) support and [Streaming Replication](https://wiki.postgresql.org/wiki/Streaming_Replication) +- PostgreSQL 11+ with [Streaming Replication](https://wiki.postgresql.org/wiki/Streaming_Replication) - Git 2.9+ - All nodes must run the same GitLab version. @@ -166,7 +166,6 @@ The tracking database instance is used as metadata to control what needs to be u - Fetch changes from a repository that has recently been updated. Because the replicated database instance is read-only, we need this additional database instance for each **secondary** node. -The tracking database requires the `postgres_fdw` extension. ### Geo Log Cursor diff --git a/doc/administration/geo/replication/multiple_servers.md b/doc/administration/geo/replication/multiple_servers.md index d8f04e07fb0..31d1ea2cd2b 100644 --- a/doc/administration/geo/replication/multiple_servers.md +++ b/doc/administration/geo/replication/multiple_servers.md @@ -137,7 +137,7 @@ documentation: synchronized from the **primary** node. NOTE: **Note:** -[NFS](../../high_availability/nfs.md) can be used in place of Gitaly but is not +[NFS](../../nfs.md) can be used in place of Gitaly but is not recommended. ### Step 2: Configure the main read-only replica PostgreSQL database on the **secondary** node @@ -196,9 +196,27 @@ the **primary** database. Use the following as a guide. geo_postgresql['enable'] = false ## - ## Disable `geo_logcursor` service so Rails doesn't get configured here + ## Disable all other services that aren't needed. Note that we had to enable + ## geo_secondary_role to cause some configuration changes to postgresql, but + ## the role enables single-node services by default. ## + alertmanager['enable'] = false + consul['enable'] = false geo_logcursor['enable'] = false + gitaly['enable'] = false + gitlab_exporter['enable'] = false + gitlab_workhorse['enable'] = false + nginx['enable'] = false + node_exporter['enable'] = false + pgbouncer_exporter['enable'] = false + prometheus['enable'] = false + redis['enable'] = false + redis_exporter['enable'] = false + repmgr['enable'] = false + sidekiq['enable'] = false + sidekiq_cluster['enable'] = false + puma['enable'] = false + unicorn['enable'] = false ``` After making these changes, [reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure) so the changes take effect. @@ -242,10 +260,8 @@ Configure the tracking database. geo_postgresql['sql_user_password'] = '<tracking_database_password_md5_hash>' ## - ## Configure FDW connection to the replica database + ## Configure PostgreSQL connection to the replica database ## - geo_secondary['db_fdw'] = true - geo_postgresql['fdw_external_password'] = '<replica_database_password_plaintext>' geo_postgresql['md5_auth_cidr_addresses'] = ['<replica_database_ip>/32'] gitlab_rails['db_host'] = '<replica_database_ip>' @@ -253,11 +269,11 @@ Configure the tracking database. gitlab_rails['auto_migrate'] = false ## - ## Disable all other services that aren't needed, since we don't have a role - ## that does this. + ## Ensure unnecessary services are disabled ## alertmanager['enable'] = false consul['enable'] = false + geo_logcursor['enable'] = false gitaly['enable'] = false gitlab_exporter['enable'] = false gitlab_workhorse['enable'] = false @@ -270,7 +286,9 @@ Configure the tracking database. redis_exporter['enable'] = false repmgr['enable'] = false sidekiq['enable'] = false + sidekiq_cluster['enable'] = false puma['enable'] = false + unicorn['enable'] = false ``` After making these changes, [reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure) so the changes take effect. @@ -284,9 +302,9 @@ In the architecture overview, there are two machines running the GitLab application services. These services are enabled selectively in the configuration. -Configure the application servers following -[Configuring GitLab for multiple nodes](../../high_availability/gitlab.md), then make the -following modifications: +Configure the GitLab Rails application servers following the relevant steps +outlined in the [reference architectures](../../reference_architectures/index.md), +then make the following modifications: 1. Edit `/etc/gitlab/gitlab.rb` on each application server in the **secondary** cluster, and add the following: @@ -373,7 +391,7 @@ application servers. In this topology, a load balancer is required at each geographic location to route traffic to the application servers. -See [Load Balancer for GitLab with multiple nodes](../../high_availability/load_balancer.md) for +See [Load Balancer for GitLab with multiple nodes](../../load_balancer.md) for more information. ### Step 6: Configure the backend application servers on the **secondary** node @@ -417,6 +435,7 @@ application servers above, with some changes to run only the `sidekiq` service: redis_exporter['enable'] = false repmgr['enable'] = false puma['enable'] = false + unicorn['enable'] = false ## ## The unique identifier for the Geo node. diff --git a/doc/administration/geo/replication/object_storage.md b/doc/administration/geo/replication/object_storage.md index 3cc0ade414e..159e2524198 100644 --- a/doc/administration/geo/replication/object_storage.md +++ b/doc/administration/geo/replication/object_storage.md @@ -33,7 +33,7 @@ whether they are stored on the local filesystem or in object storage. To enable GitLab replication, you must: -1. Go to **{admin}** **Admin Area >** **{location-dot}** **Geo**. +1. Go to **Admin Area > Geo**. 1. Press **Edit** on the **secondary** node. 1. Enable the **Allow this secondary node to replicate content on Object Storage** checkbox. diff --git a/doc/administration/geo/replication/remove_geo_node.md b/doc/administration/geo/replication/remove_geo_node.md index 539132776b3..2b01231241c 100644 --- a/doc/administration/geo/replication/remove_geo_node.md +++ b/doc/administration/geo/replication/remove_geo_node.md @@ -9,7 +9,7 @@ type: howto **Secondary** nodes can be removed from the Geo cluster using the Geo admin page of the **primary** node. To remove a **secondary** node: -1. Navigate to **{admin}** **Admin Area >** **{location-dot}** **Geo** (`/admin/geo/nodes`). +1. Navigate to **Admin Area > Geo** (`/admin/geo/nodes`). 1. Click the **Remove** button for the **secondary** node you want to remove. 1. Confirm by clicking **Remove** when the prompt appears. diff --git a/doc/administration/geo/replication/troubleshooting.md b/doc/administration/geo/replication/troubleshooting.md index c2f8aa35d2d..b8172322c10 100644 --- a/doc/administration/geo/replication/troubleshooting.md +++ b/doc/administration/geo/replication/troubleshooting.md @@ -14,7 +14,6 @@ Here is a list of steps you should take to attempt to fix problem: - Perform [basic troubleshooting](#basic-troubleshooting). - Fix any [replication errors](#fixing-replication-errors). -- Fix any [Foreign Data Wrapper](#fixing-foreign-data-wrapper-errors) errors. - Fix any [common](#fixing-common-errors) errors. ## Basic troubleshooting @@ -26,7 +25,7 @@ Before attempting more advanced troubleshooting: ### Check the health of the **secondary** node -Visit the **primary** node's **{admin}** **Admin Area >** **{location-dot}** **Geo** (`/admin/geo/nodes`) in +Visit the **primary** node's **Admin Area > Geo** (`/admin/geo/nodes`) in your browser. We perform the following health checks on each **secondary** node to help identify if something is wrong: @@ -44,6 +43,8 @@ For information on how to resolve common errors reported from the UI, see If the UI is not working, or you are unable to log in, you can run the Geo health check manually to get this information as well as a few more details. +#### Health check Rake task + This Rake task can be run on an app node in the **primary** or **secondary** Geo nodes: @@ -62,8 +63,6 @@ This machine's Geo node name matches a database record ... yes, found a secondar GitLab Geo secondary database is correctly configured ... yes Database replication enabled? ... yes Database replication working? ... yes -GitLab Geo tracking database is configured to use Foreign Data Wrapper? ... yes -GitLab Geo tracking database Foreign Data Wrapper schema is up-to-date? ... yes GitLab Geo HTTP(S) connectivity ... * Can connect to the primary node ... yes HTTP/HTTPS repository cloning is enabled ... yes @@ -77,6 +76,8 @@ All projects are in hashed storage? ... yes Checking Geo ... Finished ``` +#### Sync status Rake task + Current sync information can be found manually by running this Rake task on any **secondary** app node: @@ -128,8 +129,7 @@ Geo finds the current machine's Geo node name in `/etc/gitlab/gitlab.rb` by: - Using the `gitlab_rails['geo_node_name']` setting. - If that is not defined, using the `external_url` setting. -This name is used to look up the node with the same **Name** in -**{admin}** **Admin Area >** **{location-dot}** **Geo**. +This name is used to look up the node with the same **Name** in **Admin Area > Geo**. To check if the current machine has a node name that matches a node in the database, run the check task: @@ -205,7 +205,7 @@ sudo gitlab-rake gitlab:geo:check - Verify the correct password is set for `gitlab_rails['db_password']` that was used when creating the hash in `postgresql['sql_user_password']` by running `gitlab-ctl pg-password-md5 gitlab` and entering the password. -1. Check returns not a secondary node +1. Check returns `not a secondary node` ```plaintext Checking Geo ... @@ -252,12 +252,12 @@ sudo gitlab-rake gitlab:geo:check When performing a PostgreSQL major version (9 > 10) update this is expected. Follow: - [initiate-the-replication-process](database.md#step-3-initiate-the-replication-process) - - [Geo database has an outdated FDW remote schema](troubleshooting.md#geo-database-has-an-outdated-fdw-remote-schema-error) ## Fixing replication errors The following sections outline troubleshooting steps for fixing replication -errors. +errors (indicated by `Database replication working? ... no` in the +[`geo:check` output](#health-check-rake-task). ### Message: `ERROR: replication slots can only be used if max_replication_slots > 0`? @@ -390,13 +390,13 @@ to respect the CIDR format (i.e. `1.2.3.4/32`). GitLab places a timeout on all repository clones, including project imports and Geo synchronization operations. If a fresh `git clone` of a repository -on the **primary** takes more than a few minutes, you may be affected by this. +on the **primary** takes more than the default three hours, you may be affected by this. To increase the timeout, add the following line to `/etc/gitlab/gitlab.rb` on the **secondary** node: ```ruby -gitlab_rails['gitlab_shell_git_timeout'] = 10800 +gitlab_rails['gitlab_shell_git_timeout'] = 14400 ``` Then reconfigure GitLab: @@ -405,7 +405,7 @@ Then reconfigure GitLab: sudo gitlab-ctl reconfigure ``` -This will increase the timeout to three hours (10800 seconds). Choose a time +This will increase the timeout to four hours (14400 seconds). Choose a time long enough to accommodate a full clone of your largest repositories. ### New LFS objects are never replicated @@ -504,11 +504,63 @@ to start again from scratch, there are a few steps that can help you: gitlab-ctl start ``` -1. Refresh Foreign Data Wrapper tables +## Fixing errors during a PostgreSQL upgrade or downgrade - ```shell - gitlab-rake geo:db:refresh_foreign_tables - ``` +### Message: `ERROR: psql: FATAL: role "gitlab-consul" does not exist` + +When +[upgrading PostgreSQL on a Geo instance](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance), you might encounter the +following error: + +```plaintext +$ sudo gitlab-ctl pg-upgrade --target-version=11 +Checking for an omnibus managed postgresql: OK +Checking if postgresql['version'] is set: OK +Checking if we already upgraded: NOT OK +Checking for a newer version of PostgreSQL to install +Upgrading PostgreSQL to 11.7 +Checking if PostgreSQL bin files are symlinked to the expected location: OK +Waiting 30 seconds to ensure tasks complete before PostgreSQL upgrade. +See https://docs.gitlab.com/omnibus/settings/database.html#upgrade-packaged-postgresql-server for details +If you do not want to upgrade the PostgreSQL server at this time, enter Ctrl-C and see the documentation for details + +Please hit Ctrl-C now if you want to cancel the operation. +..............................Detected an HA cluster. +Error running command: /opt/gitlab/embedded/bin/psql -qt -d gitlab_repmgr -h /var/opt/gitlab/postgresql -p 5432 -c "SELECT name FROM repmgr_gitlab_cluster.repl_nodes WHERE type='master' AND active != 'f'" -U gitlab-consul +ERROR: psql: FATAL: role "gitlab-consul" does not exist +Traceback (most recent call last): + 10: from /opt/gitlab/embedded/bin/omnibus-ctl:23:in `<main>' + 9: from /opt/gitlab/embedded/bin/omnibus-ctl:23:in `load' + 8: from /opt/gitlab/embedded/lib/ruby/gems/2.6.0/gems/omnibus-ctl-0.6.0/bin/omnibus-ctl:31:in `<top (required)>' + 7: from /opt/gitlab/embedded/lib/ruby/gems/2.6.0/gems/omnibus-ctl-0.6.0/lib/omnibus-ctl.rb:746:in `run' + 6: from /opt/gitlab/embedded/lib/ruby/gems/2.6.0/gems/omnibus-ctl-0.6.0/lib/omnibus-ctl.rb:204:in `block in add_command_under_category' + 5: from /opt/gitlab/embedded/service/omnibus-ctl/pg-upgrade.rb:171:in `block in load_file' + 4: from /opt/gitlab/embedded/service/omnibus-ctl-ee/lib/repmgr.rb:248:in `is_master?' + 3: from /opt/gitlab/embedded/service/omnibus-ctl-ee/lib/repmgr.rb:100:in `execute_psql' + 2: from /opt/gitlab/embedded/service/omnibus-ctl-ee/lib/repmgr.rb:113:in `cmd' + 1: from /opt/gitlab/embedded/lib/ruby/gems/2.6.0/gems/mixlib-shellout-3.0.9/lib/mixlib/shellout.rb:287:in `error!' +/opt/gitlab/embedded/lib/ruby/gems/2.6.0/gems/mixlib-shellout-3.0.9/lib/mixlib/shellout.rb:300:in `invalid!': Expected process to exit with [0], but received '2' (Mixlib::ShellOut::ShellCommandFailed) +---- Begin output of /opt/gitlab/embedded/bin/psql -qt -d gitlab_repmgr -h /var/opt/gitlab/postgresql -p 5432 -c "SELECT name FROM repmgr_gitlab_cluster.repl_nodes WHERE type='master' AND active != 'f'" -U gitlab-consul ---- +STDOUT: +STDERR: psql: FATAL: role "gitlab-consul" does not exist +---- End output of /opt/gitlab/embedded/bin/psql -qt -d gitlab_repmgr -h /var/opt/gitlab/postgresql -p 5432 -c "SELECT name FROM repmgr_gitlab_cluster.repl_nodes WHERE type='master' AND active != 'f'" -U gitlab-consul ---- +Ran /opt/gitlab/embedded/bin/psql -qt -d gitlab_repmgr -h /var/opt/gitlab/postgresql -p 5432 -c "SELECT name FROM repmgr_gitlab_cluster.repl_nodes WHERE type='master' AND active != 'f'" -U gitlab-consul returned 2 +``` + +If you are upgrading the PostgreSQL read-replica of a Geo secondary node, and +you are not using `consul` or `repmgr`, you may need to disable `consul` and/or +`repmgr` services in `gitlab.rb`: + +```ruby +consul['enable'] = false +repmgr['enable'] = false +``` + +Then reconfigure GitLab: + +```shell +sudo gitlab-ctl reconfigure +``` ## Fixing errors during a failover or when promoting a secondary to a primary node @@ -551,6 +603,35 @@ or `gitlab-ctl promote-to-primary-node`, either: bug](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/22021) was fixed. +If the above does not work, another possible reason is that you have paused replication +from the original primary node before attempting to promote this node. + +To double check this, you can do the following: + +- Get the current secondary node's ID using: + + ```shell + sudo gitlab-rails runner 'puts GeoNode.current_node.id' + ``` + +- Double check that the node is active through the database by running the following + on the secondary node, replacing `ID_FROM_ABOVE`: + + ```shell + sudo gitlab-rails dbconsole + + SELECT enabled FROM geo_nodes WHERE id = ID_FROM_ABOVE; + ``` + +- If the above returned `f` it means that the replication was paused. + You can re-enable it through an `UPDATE` statement in the database: + + ```shell + sudo gitlab-rails dbconsole + + UPDATE geo_nodes SET enabled = 't' WHERE id = ID_FROM_ABOVE; + ``` + ### Message: ``NoMethodError: undefined method `secondary?' for nil:NilClass`` When [promoting a **secondary** node](../disaster_recovery/index.md#step-3-promoting-a-secondary-node), @@ -593,231 +674,6 @@ sudo /opt/gitlab/embedded/bin/gitlab-pg-ctl promote GitLab 12.9 and later are [unaffected by this error](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5147). -## Fixing Foreign Data Wrapper errors - -This section documents ways to fix potential Foreign Data Wrapper errors. - -### "Foreign Data Wrapper (FDW) is not configured" error - -When setting up Geo, you might see this warning in the `gitlab-rake -gitlab:geo:check` output: - -```plaintext -GitLab Geo tracking database Foreign Data Wrapper schema is up-to-date? ... foreign data wrapper is not configured -``` - -There are a few key points to remember: - -1. The FDW settings are configured on the Geo **tracking** database. -1. The configured foreign server enables a login to the Geo - **secondary**, read-only database. - -By default, the Geo secondary and tracking database are running on the -same host on different ports. That is, 5432 and 5431 respectively. - -#### Checking configuration - -NOTE: **Note:** -The following steps are for Omnibus installs only. Using Geo with source-based installs was **deprecated** in GitLab 11.5. - -To check the configuration: - -1. SSH into an app node in the **secondary**: - - ```shell - sudo -i - ``` - - Note: An app node is any machine running at least one of the following services: - - - `puma` - - `unicorn` - - `sidekiq` - - `geo-logcursor` - -1. Enter the database console: - - If the tracking database is running on the same node: - - ```shell - gitlab-geo-psql - ``` - - Or, if the tracking database is running on a different node, you must specify - the user and host when entering the database console: - - ```shell - gitlab-geo-psql -U gitlab_geo -h <IP of tracking database> - ``` - - You will be prompted for the password of the `gitlab_geo` user. You can find - it in plaintext in `/etc/gitlab/gitlab.rb` at: - - ```ruby - geo_secondary['db_password'] = '<geo_tracking_db_password>' - ``` - - This password is normally set on the tracking database during - [Step 3: Configure the tracking database on the secondary node](multiple_servers.md#step-3-configure-the-tracking-database-on-the-secondary-node), - and it is set on the app nodes during - [Step 4: Configure the frontend application servers on the secondary node](multiple_servers.md#step-4-configure-the-frontend-application-servers-on-the-secondary-node). - -1. Check whether any tables are present with the following statement: - - ```sql - SELECT * from information_schema.foreign_tables; - ``` - - If everything is working, you should see something like this: - - ```plaintext - gitlabhq_geo_production=# SELECT * from information_schema.foreign_tables; - foreign_table_catalog | foreign_table_schema | foreign_table_name | foreign_server_catalog | foreign_server_name - -------------------------+----------------------+-------------------------------------------------+-------------------------+--------------------- - gitlabhq_geo_production | gitlab_secondary | abuse_reports | gitlabhq_geo_production | gitlab_secondary - gitlabhq_geo_production | gitlab_secondary | appearances | gitlabhq_geo_production | gitlab_secondary - gitlabhq_geo_production | gitlab_secondary | application_setting_terms | gitlabhq_geo_production | gitlab_secondary - gitlabhq_geo_production | gitlab_secondary | application_settings | gitlabhq_geo_production | gitlab_secondary - <snip> - ``` - - However, if the query returns with `0 rows`, then continue onto the next steps. - -1. Check that the foreign server mapping is correct via `\des+`. The - results should look something like this: - - ```plaintext - gitlabhq_geo_production=# \des+ - List of foreign servers - -[ RECORD 1 ]--------+------------------------------------------------------------ - Name | gitlab_secondary - Owner | gitlab-psql - Foreign-data wrapper | postgres_fdw - Access privileges | "gitlab-psql"=U/"gitlab-psql" + - | gitlab_geo=U/"gitlab-psql" - Type | - Version | - FDW Options | (host '0.0.0.0', port '5432', dbname 'gitlabhq_production') - Description | - ``` - - NOTE: **Note:** - Pay particular attention to the host and port under - FDW options. That configuration should point to the Geo secondary - database. - - If you need to experiment with changing the host or password, the - following queries demonstrate how: - - ```sql - ALTER SERVER gitlab_secondary OPTIONS (SET host '<my_new_host>'); - ALTER SERVER gitlab_secondary OPTIONS (SET port 5432); - ``` - - If you change the host and/or port, you will also have to adjust the - following settings in `/etc/gitlab/gitlab.rb` and run `gitlab-ctl - reconfigure`: - - - `gitlab_rails['db_host']` - - `gitlab_rails['db_port']` - -1. Check that the user mapping is configured properly via `\deu+`: - - ```plaintext - gitlabhq_geo_production=# \deu+ - List of user mappings - Server | User name | FDW Options - ------------------+------------+-------------------------------------------------------------------------------- - gitlab_secondary | gitlab_geo | ("user" 'gitlab', password 'YOUR-PASSWORD-HERE') - (1 row) - ``` - - Make sure the password is correct. You can test that logins work by running `psql`: - - ```shell - # Connect to the tracking database as the `gitlab_geo` user - sudo \ - -u git /opt/gitlab/embedded/bin/psql \ - -h /var/opt/gitlab/geo-postgresql \ - -p 5431 \ - -U gitlab_geo \ - -W \ - -d gitlabhq_geo_production - ``` - - If you need to correct the password, the following query shows how: - - ```sql - ALTER USER MAPPING FOR gitlab_geo SERVER gitlab_secondary OPTIONS (SET password '<my_new_password>'); - ``` - - If you change the user or password, you will also have to adjust the - following settings in `/etc/gitlab/gitlab.rb` and run `gitlab-ctl - reconfigure`: - - - `gitlab_rails['db_username']` - - `gitlab_rails['db_password']` - - If you are using [PgBouncer in front of the secondary - database](database.md#pgbouncer-support-optional), be sure to update - the following settings: - - - `geo_postgresql['fdw_external_user']` - - `geo_postgresql['fdw_external_password']` - -#### Manual reload of FDW schema - -If you're still unable to get FDW working, you may want to try a manual -reload of the FDW schema. To manually reload the FDW schema: - -1. On the node running the Geo tracking database, enter the PostgreSQL console via - the `gitlab_geo` user: - - ```shell - sudo \ - -u git /opt/gitlab/embedded/bin/psql \ - -h /var/opt/gitlab/geo-postgresql \ - -p 5431 \ - -U gitlab_geo \ - -W \ - -d gitlabhq_geo_production - ``` - - Be sure to adjust the port and hostname for your configuration. You - may be asked to enter a password. - -1. Reload the schema via: - - ```sql - DROP SCHEMA IF EXISTS gitlab_secondary CASCADE; - CREATE SCHEMA gitlab_secondary; - GRANT USAGE ON FOREIGN SERVER gitlab_secondary TO gitlab_geo; - IMPORT FOREIGN SCHEMA public FROM SERVER gitlab_secondary INTO gitlab_secondary; - ``` - -1. Test that queries work: - - ```sql - SELECT * from information_schema.foreign_tables; - SELECT * FROM gitlab_secondary.projects limit 1; - ``` - -### "Geo database has an outdated FDW remote schema" error - -GitLab can error with a `Geo database has an outdated FDW remote schema` message. - -For example: - -```plaintext -Geo database has an outdated FDW remote schema. It contains 229 of 236 expected tables. Please refer to Geo Troubleshooting. -``` - -To resolve this, run the following command on the **secondary**: - -```shell -sudo gitlab-rake geo:db:refresh_foreign_tables -``` - ## Expired artifacts If you notice for some reason there are more artifacts on the Geo @@ -835,7 +691,7 @@ If you are able to log in to the **primary** node, but you receive this error when attempting to log into a **secondary**, you should check that the Geo node's URL matches its external URL. -1. On the primary, visit **{admin}** **Admin Area >** **{location-dot}** **Geo**. +1. On the primary, visit **Admin Area > Geo**. 1. Find the affected **secondary** and click **Edit**. 1. Ensure the **URL** field matches the value found in `/etc/gitlab/gitlab.rb` in `external_url "https://gitlab.example.com"` on the frontend server(s) of @@ -896,13 +752,6 @@ If you are using Omnibus GitLab installation, something might have failed during - Run `sudo gitlab-ctl reconfigure`. - Manually trigger the database migration by running: `sudo gitlab-rake geo:db:migrate` as root on the **secondary** node. -### Geo database is not configured to use Foreign Data Wrapper - -This error means the Geo Tracking Database doesn't have the FDW server and credentials -configured. - -See ["Foreign Data Wrapper (FDW) is not configured" error?](#foreign-data-wrapper-fdw-is-not-configured-error). - ### GitLab indicates that more than 100% of repositories were synced This can be caused by orphaned records in the project registry. You can clear them diff --git a/doc/administration/geo/replication/tuning.md b/doc/administration/geo/replication/tuning.md index 63a8f81eecb..fe0b189863c 100644 --- a/doc/administration/geo/replication/tuning.md +++ b/doc/administration/geo/replication/tuning.md @@ -7,18 +7,25 @@ type: howto # Tuning Geo **(PREMIUM ONLY)** -## Changing the sync capacity values +## Changing the sync/verification capacity values -In the Geo admin page at **{admin}** **Admin Area >** **{location-dot}** **Geo** (`/admin/geo/nodes`), +In the Geo admin page at **Admin Area > Geo** (`/admin/geo/nodes`), there are several variables that can be tuned to improve performance of Geo: -- Repository sync capacity. -- File sync capacity. +- Repository sync capacity +- File sync capacity +- Container repositories sync capacity +- Verification capacity -Increasing these values will increase the number of jobs that are scheduled. +Increasing capacity values will increase the number of jobs that are scheduled. However, this may not lead to more downloads in parallel unless the number of available Sidekiq threads is also increased. For example, if repository sync capacity is increased from 25 to 50, you may also want to increase the number of Sidekiq threads from 25 to 50. See the [Sidekiq concurrency documentation](../../operations/extra_sidekiq_processes.md#number-of-threads) for more details. + +## Repository re-verification + +See +[Automatic background verification](../disaster_recovery/background_verification.md). diff --git a/doc/administration/geo/replication/version_specific_updates.md b/doc/administration/geo/replication/version_specific_updates.md index 777de715a8c..900d09bdd34 100644 --- a/doc/administration/geo/replication/version_specific_updates.md +++ b/doc/administration/geo/replication/version_specific_updates.md @@ -11,6 +11,35 @@ Check this document if it includes instructions for the version you are updating These steps go together with the [general steps](updating_the_geo_nodes.md#general-update-steps) for updating Geo nodes. +## Updating to GitLab 13.3 + +In GitLab 13.3, Geo removed the PostgreSQL [Foreign Data Wrapper](https://www.postgresql.org/docs/11/postgres-fdw.html) dependency for the tracking database. + +The FDW server, user, and the extension will be removed during the upgrade process on each **secondary** node. The GitLab settings related to the FDW in the `/etc/gitlab/gitlab.rb` have been deprecated and can be safely removed. + +There are some scenarios like using an external PostgreSQL instance for the tracking database where the FDW settings must be removed manually. Enter the PostgreSQL console of that instance and remove them: + +```shell +DROP SERVER gitlab_secondary CASCADE; +DROP EXTENSION IF EXISTS postgres_fdw; +``` + +## Updating to GitLab 13.0 + +Upgrading to GitLab 13.0 requires GitLab 12.10 to already be using PostgreSQL +version 11. Please see +[the Omnibus documentation](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance) +for the recommended procedure. + +## Updating to GitLab 12.10 + +GitLab 12.10 does not attempt to automatically update the embedded PostgreSQL +server when using Geo, because the PostgreSQL upgrade requires downtime on +secondaries while reinitializing streaming replication. It must be upgraded +manually. Please see +[the Omnibus documentation](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance) +for the recommended procedure. + ## Updating to GitLab 12.9 CAUTION: **Warning:** @@ -18,6 +47,32 @@ GitLab 12.9.0 through GitLab 12.9.3 are affected by [a bug that stops repository verification](https://gitlab.com/gitlab-org/gitlab/-/issues/213523). The issue is fixed in GitLab 12.9.4. Please upgrade to GitLab 12.9.4 or later. +By default, GitLab 12.9 will attempt to automatically update the embedded +PostgreSQL server to 10.12 from 9.6, which requires downtime on secondaries +while reinitializing streaming replication. Please see +[the Omnibus documentation](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance) +for the recommended procedure. + +This can be temporarily disabled by running the following before updating: + +```shell +sudo touch /etc/gitlab/disable-postgresql-upgrade +``` + +## Updating to GitLab 12.8 + +By default, GitLab 12.8 will attempt to automatically update the embedded +PostgreSQL server to 10.12 from 9.6, which requires downtime on secondaries +while reinitializing streaming replication. Please see +[the Omnibus documentation](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance) +for the recommended procedure. + +This can be temporarily disabled by running the following before updating: + +```shell +sudo touch /etc/gitlab/disable-postgresql-upgrade +``` + ## Updating to GitLab 12.7 DANGER: **Danger:** @@ -28,8 +83,92 @@ bug](https://gitlab.com/gitlab-org/gitlab/-/issues/199672) that causes Geo fix](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/24021) was shipped in 12.7.5. +By default, GitLab 12.7 will attempt to automatically update the embedded +PostgreSQL server to 10.9 from 9.6, which requires downtime on secondaries while +reinitializing streaming replication. Please see +[the Omnibus documentation](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance) +for the recommended procedure. + +This can be temporarily disabled by running the following before updating: + +```shell +sudo touch /etc/gitlab/disable-postgresql-upgrade +``` + +## Updating to GitLab 12.6 + +By default, GitLab 12.6 will attempt to automatically update the embedded +PostgreSQL server to 10.9 from 9.6, which requires downtime on secondaries while +reinitializing streaming replication. Please see +[the Omnibus documentation](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance) +for the recommended procedure. + +This can be temporarily disabled by running the following before updating: + +```shell +sudo touch /etc/gitlab/disable-postgresql-upgrade +``` + +## Updating to GitLab 12.5 + +By default, GitLab 12.5 will attempt to automatically update the embedded +PostgreSQL server to 10.9 from 9.6, which requires downtime on secondaries while +reinitializing streaming replication. Please see +[the Omnibus documentation](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance) +for the recommended procedure. + +This can be temporarily disabled by running the following before updating: + +```shell +sudo touch /etc/gitlab/disable-postgresql-upgrade +``` + +## Updating to GitLab 12.4 + +By default, GitLab 12.4 will attempt to automatically update the embedded +PostgreSQL server to 10.9 from 9.6, which requires downtime on secondaries while +reinitializing streaming replication. Please see +[the Omnibus documentation](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance) +for the recommended procedure. + +This can be temporarily disabled by running the following before updating: + +```shell +sudo touch /etc/gitlab/disable-postgresql-upgrade +``` + +## Updating to GitLab 12.3 + +DANGER: **Danger:** +If the existing PostgreSQL server version is 9.6.x, it is recommended to +upgrade to GitLab 12.4 or newer. By default, GitLab 12.3 will attempt to +automatically update the embedded PostgreSQL server to 10.9 from 9.6. In +certain circumstances, it will fail. Please see +[the Omnibus documentation](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance) +for more information. + +Additionally, if the PostgreSQL upgrade does not fail, a successful upgrade +requires downtime on secondaries while reinitializing streaming replication. +Please see +[the Omnibus documentation](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance) +for the recommended procedure. + ## Updating to GitLab 12.2 +DANGER: **Danger:** +If the existing PostgreSQL server version is 9.6.x, it is recommended to +upgrade to GitLab 12.4 or newer. By default, GitLab 12.2 will attempt to +automatically update the embedded PostgreSQL server to 10.9 from 9.6. In +certain circumstances, it will fail. Please see +[the Omnibus documentation](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance) +for more information. + +Additionally, if the PostgreSQL upgrade does not fail, a successful upgrade +requires downtime on secondaries while reinitializing streaming replication. +Please see +[the Omnibus documentation](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance) +for the recommended procedure. + GitLab 12.2 includes the following minor PostgreSQL updates: - To version `9.6.14` if you run PostgreSQL 9.6. @@ -48,17 +187,20 @@ The restart avoids a version mismatch when PostgreSQL tries to load the FDW exte ## Updating to GitLab 12.1 -By default, GitLab 12.1 will attempt to automatically update the -embedded PostgreSQL server to 10.7 from 9.6. Please see -[the omnibus documentation](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance) +DANGER: **Danger:** +If the existing PostgreSQL server version is 9.6.x, it is recommended to +upgrade to GitLab 12.4 or newer. By default, GitLab 12.1 will attempt to +automatically update the embedded PostgreSQL server to 10.9 from 9.6. In +certain circumstances, it will fail. Please see +[the Omnibus documentation](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance) +for more information. + +Additionally, if the PostgreSQL upgrade does not fail, a successful upgrade +requires downtime on secondaries while reinitializing streaming replication. +Please see +[the Omnibus documentation](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance) for the recommended procedure. -This can be temporarily disabled by running the following before updating: - -```shell -sudo touch /etc/gitlab/disable-postgresql-upgrade -``` - ## Updating to GitLab 12.0 CAUTION: **Warning:** @@ -214,7 +356,7 @@ Replicating over SSH has been deprecated, and support for this option will be removed in a future release. To switch to HTTP/HTTPS replication, log into the **primary** node as an admin and visit -**{admin}** **Admin Area >** **{location-dot}** **Geo** (`/admin/geo/nodes`). For each **secondary** node listed, +**Admin Area > Geo** (`/admin/geo/nodes`). For each **secondary** node listed, press the "Edit" button, change the "Repository cloning" setting from "SSH (deprecated)" to "HTTP/HTTPS", and press "Save changes". This should take effect immediately. diff --git a/doc/administration/git_annex.md b/doc/administration/git_annex.md index ed6218ede91..59c1e621e46 100644 --- a/doc/administration/git_annex.md +++ b/doc/administration/git_annex.md @@ -1,4 +1,8 @@ --- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference, howto disqus_identifier: 'https://docs.gitlab.com/ee/workflow/git_annex.html' --- @@ -90,8 +94,8 @@ one is located in `config.yml` of GitLab Shell. ## Using GitLab git-annex -> **Note:** -> Your Git remotes must be using the SSH protocol, not HTTP(S). +NOTE: **Note:** +Your Git remotes must be using the SSH protocol, not HTTP(S). Here is an example workflow of uploading a very large file and then checking it into your Git repository: diff --git a/doc/administration/git_protocol.md b/doc/administration/git_protocol.md index e1600d972bd..841e636bd0a 100644 --- a/doc/administration/git_protocol.md +++ b/doc/administration/git_protocol.md @@ -1,4 +1,8 @@ --- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference description: "Set and configure Git protocol v2" --- diff --git a/doc/administration/gitaly/img/cluster_example_v13_3.png b/doc/administration/gitaly/img/cluster_example_v13_3.png Binary files differnew file mode 100644 index 00000000000..efca79ecef9 --- /dev/null +++ b/doc/administration/gitaly/img/cluster_example_v13_3.png diff --git a/doc/administration/gitaly/img/shard_example_v13_3.png b/doc/administration/gitaly/img/shard_example_v13_3.png Binary files differnew file mode 100644 index 00000000000..47f582cfa78 --- /dev/null +++ b/doc/administration/gitaly/img/shard_example_v13_3.png diff --git a/doc/administration/gitaly/index.md b/doc/administration/gitaly/index.md index 057d0559c14..9558488c89e 100644 --- a/doc/administration/gitaly/index.md +++ b/doc/administration/gitaly/index.md @@ -376,7 +376,7 @@ This can be risky because anything that prevents your Gitaly clients from reachi servers will cause all Gitaly requests to fail. For example, any sort of network, firewall, or name resolution problems. -Additionally, you must [disable Rugged](../high_availability/nfs.md#improving-nfs-performance-with-gitlab) +Additionally, you must [disable Rugged](../nfs.md#improving-nfs-performance-with-gitlab) if previously enabled manually. Gitaly makes the following assumptions: diff --git a/doc/administration/gitaly/praefect.md b/doc/administration/gitaly/praefect.md index 1f97cd304f9..2e36a754c79 100644 --- a/doc/administration/gitaly/praefect.md +++ b/doc/administration/gitaly/praefect.md @@ -59,6 +59,38 @@ Follow the [HA Gitaly epic](https://gitlab.com/groups/gitlab-org/-/epics/1489) for improvements including [horizontally distributing reads](https://gitlab.com/groups/gitlab-org/-/epics/2013). +## Cluster or shard + +Gitaly supports multiple models of scaling: + +- Clustering using Gitaly Cluster, where each repository is stored on multiple Gitaly nodes in the + cluster. Read requests are distributed between repository replicas and write requests are + broadcast to repository replicas. +- Sharding using [repository storage paths](../repository_storage_paths.md), where each repository + is stored on the assigned Gitaly node. All requests are routed to this node. + +| Cluster | Shard | +|---|---| +|  |  | + +Generally, Gitaly Cluster can replace sharded configurations, at the expense of additional storage +needed to store each repository on multiple Gitaly nodes. The benefit of using Gitaly Cluster over +sharding is: + +- Improved fault tolerance, because each Gitaly node has a copy of every repository. +- Improved resource utilization, reducing the need for over-provisioning for shard-specific peak + loads, because read loads are distributed across replicas. +- Manual rebalancing for performance is not required, because read loads are distributed across + replicas. +- Simpler management, because all Gitaly nodes are identical. + +Under some workloads, CPU and memory requirements may require a large fleet of Gitaly nodes and it +can be uneconomical to have one to one replication factor. + +A hybrid approach can be used in these instances, where each shard is configured as a smaller +cluster. [Variable replication factor](https://gitlab.com/groups/gitlab-org/-/epics/3372) is planned +to provide greater flexibility for extremely large GitLab instances. + ## Requirements for configuring a Gitaly Cluster The minimum recommended configuration for a Gitaly Cluster requires: @@ -142,6 +174,16 @@ database on the same PostgreSQL server if using [Geo](../geo/replication/index.md). The replication state is internal to each instance of GitLab and should not be replicated. +These instructions help set up a single PostgreSQL database, which creates a single point of +failure. For greater fault tolerance, the following options are available: + +- For non-Geo installations, use one of the fault-tolerant + [PostgreSQL setups](../postgresql/index.md). +- For Geo instances, either: + - Set up a separate [PostgreSQL instance](https://www.postgresql.org/docs/11/high-availability.html). + - Use a cloud-managed PostgreSQL service. AWS + [Relational Database Service](https://aws.amazon.com/rds/)) is recommended. + To complete this section you will need: - 1 Praefect node @@ -188,6 +230,49 @@ node, using `psql` which is installed by Omnibus GitLab. The database used by Praefect is now configured. +#### PgBouncer + +To reduce PostgreSQL resource consumption, you should set up and configure +[PgBouncer](https://www.pgbouncer.org/) in front of the PostgreSQL instance. To do +this, replace value of the `POSTGRESQL_SERVER_ADDRESS` with corresponding IP or host +address of the PgBouncer instance. + +This documentation doesn't provide PgBouncer installation instructions, +you can: + +- Find instructions on the [official website](https://www.pgbouncer.org/install.html). +- Use a [Docker image](https://hub.docker.com/r/edoburu/pgbouncer/). + +In addition to base PgBouncer configuration options, set the following values: + +- The [Praefect PostgreSQL database](#postgresql) in the `[databases]` section: + + ```ini + [databases] + * = host=POSTGRESQL_SERVER_ADDRESS port=5432 auth_user=praefect + ``` + +- [`pool_mode`](https://www.pgbouncer.org/config.html#pool_mode) + and [`ignore_startup_parameters`](https://www.pgbouncer.org/config.html#ignore_startup_parameters) + in the `[pgbouncer]` section: + + ```ini + [pgbouncer] + pool_mode = transaction + ignore_startup_parameters = extra_float_digits + ``` + +The `praefect` user and its password should be included in the file (default is +`userlist.txt`) used by PgBouncer if the [`auth_file`](https://www.pgbouncer.org/config.html#auth_file) +configuration option is set. + +NOTE: **Note:** +By default PgBouncer uses port `6432` to accept incoming +connections. You can change it by setting the [`listen_port`](https://www.pgbouncer.org/config.html#listen_port) +configuration option. We recommend setting it to the default port value (`5432`) used by +PostgreSQL instances. Otherwise you should change the configuration parameter +`praefect['database_port']` for each Praefect instance to the correct value. + ### Praefect To complete this section you will need: @@ -212,6 +297,7 @@ application server, or a Gitaly node. postgresql['enable'] = false redis['enable'] = false nginx['enable'] = false + alertmanager['enable'] = false prometheus['enable'] = false grafana['enable'] = false puma['enable'] = false @@ -657,6 +743,13 @@ internal traffic from the GitLab application to the Praefect nodes. The specifics on which load balancer to use or the exact configuration is beyond the scope of the GitLab documentation. +NOTE: **Note:** +The load balancer must be configured to accept traffic from the Gitaly nodes in +addition to the GitLab nodes. Some requests handled by +[`gitaly-ruby`](index.md#gitaly-ruby) sidecar processes call into the main Gitaly +process. `gitaly-ruby` uses the Gitaly address set in the GitLab server's +`git_data_dirs` setting to make this connection. + We hope that if you’re managing HA systems like GitLab, you have a load balancer of choice already. Some examples include [HAProxy](https://www.haproxy.org/) (open-source), [Google Internal Load Balancer](https://cloud.google.com/load-balancing/docs/internal/), @@ -847,19 +940,14 @@ cluster. ## Distributed reads -> Introduced in GitLab 13.1 in [beta](https://about.gitlab.com/handbook/product/#alpha-beta-ga) with feature flag `gitaly_distributed_reads` set to disabled. - Praefect supports distribution of read operations across Gitaly nodes that are configured for the virtual node. -To allow for [performance testing](https://gitlab.com/gitlab-org/quality/performance/-/issues/231), -distributed reads are currently in -[beta](https://about.gitlab.com/handbook/product/#alpha-beta-ga) and disabled by -default. To enable distributed reads, the `gitaly_distributed_reads` -[feature flag](../feature_flags.md) must be enabled in a Ruby console: +The feature is enabled by default. To disable distributed reads, the `gitaly_distributed_reads` +[feature flag](../feature_flags.md) must be disabled in a Ruby console: ```ruby -Feature.enable(:gitaly_distributed_reads) +Feature.disable(:gitaly_distributed_reads) ``` If enabled, all RPCs marked with `ACCESSOR` option like @@ -884,32 +972,51 @@ They reflect configuration defined for this instance of Praefect. ## Strong consistency -> Introduced in GitLab 13.1 in [alpha](https://about.gitlab.com/handbook/product/#alpha-beta-ga), disabled by default. +> - Introduced in GitLab 13.1 in [alpha](https://about.gitlab.com/handbook/product/gitlab-the-product/#alpha-beta-ga), disabled by default. +> - Entered [beta](https://about.gitlab.com/handbook/product/gitlab-the-product/#alpha-beta-ga) in GitLab 13.2, disabled by default. +> - From GitLab 13.3, disabled unless primary-wins reference transactions strategy is disabled. Praefect guarantees eventual consistency by replicating all writes to secondary nodes after the write to the primary Gitaly node has happened. Praefect can instead provide strong consistency by creating a transaction and writing changes to all Gitaly nodes at once. Strong consistency is currently in -[alpha](https://about.gitlab.com/handbook/product/#alpha-beta-ga) and not enabled by +[alpha](https://about.gitlab.com/handbook/product/gitlab-the-product/#alpha-beta-ga) and not enabled by default. If enabled, transactions are only available for a subset of RPCs. For more information, see the [strong consistency epic](https://gitlab.com/groups/gitlab-org/-/epics/1189). To enable strong consistency: -- In GitLab 13.2 and later, enable the `:gitaly_reference_transactions` feature flag. +- In GitLab 13.3 and later, reference transactions are enabled by default with + a primary-wins strategy. This strategy causes all transactions to succeed for + the primary and thus does not ensure strong consistency. To enable strong + consistency, disable the `:gitaly_reference_transactions_primary_wins` + feature flag. +- In GitLab 13.2, enable the `:gitaly_reference_transactions` feature flag. - In GitLab 13.1, enable the `:gitaly_reference_transactions` and `:gitaly_hooks_rpc` feature flags. -Enabling feature flags requires [access to the Rails console](../feature_flags.md#start-the-gitlab-rails-console). +Changing feature flags requires [access to the Rails console](../feature_flags.md#start-the-gitlab-rails-console). In the Rails console, enable or disable the flags as required. For example: ```ruby Feature.enable(:gitaly_reference_transactions) +Feature.disable(:gitaly_reference_transactions_primary_wins) ``` -To monitor strong consistency, use the `gitaly_praefect_transactions_total` and -`gitaly_praefect_transactions_delay_seconds` Prometheus counter metrics. +To monitor strong consistency, you can use the following Prometheus metrics: + +- `gitaly_praefect_transactions_total`: Number of transactions created and + voted on. +- `gitaly_praefect_subtransactions_per_transaction_total`: Number of times + nodes cast a vote for a single transaction. This can happen multiple times if + multiple references are getting updated in a single transaction. +- `gitaly_praefect_voters_per_transaction_total`: Number of Gitaly nodes taking + part in a transaction. +- `gitaly_praefect_transactions_delay_seconds`: Server-side delay introduced by + waiting for the transaction to be committed. +- `gitaly_hook_transaction_voting_delay_seconds`: Client-side delay introduced + by waiting for the transaction to be committed. ## Automatic failover and leader election @@ -940,76 +1047,167 @@ strategy in the future. ## Primary Node Failure -Praefect recovers from a failing primary Gitaly node by promoting a healthy secondary as the new primary. To minimize data loss, Praefect elects the secondary with the least unreplicated writes from the primary. There can still be some unreplicated writes, leading to data loss. +Gitaly Cluster recovers from a failing primary Gitaly node by promoting a healthy secondary as the +new primary. + +To minimize data loss, Gitaly Cluster: + +- Switches repositories that are outdated on the new primary to [read-only mode](#read-only-mode). +- Elects the secondary with the least unreplicated writes from the primary to be the new primary. + Because there can still be some unreplicated writes, [data loss can occur](#check-for-data-loss). + +### Read-only mode + +> - Introduced in GitLab 13.0 as [generally available](https://about.gitlab.com/handbook/product/gitlab-the-product/#generally-available-ga). +> - Between GitLab 13.0 and GitLab 13.2, read-only mode applied to the whole virtual storage and occurred whenever failover occurred. +> - [In GitLab 13.3 and later](https://gitlab.com/gitlab-org/gitaly/-/issues/2862), read-only mode applies on a per-repository basis and only occurs if a new primary is out of date. + +When Gitaly Cluster switches to a new primary, repositories enter read-only mode if they are out of +date. This can happen after failing over to an outdated secondary. Read-only mode eases data +recovery efforts by preventing writes that may conflict with the unreplicated writes on other nodes. + +To enable writes again, an administrator can: + +1. [Check](#check-for-data-loss) for data loss. +1. Attempt to [recover](#recover-missing-data) missing data. +1. Either [enable writes](#enable-writes-or-accept-data-loss) in the virtual storage or + [accept data loss](#enable-writes-or-accept-data-loss) if necessary, depending on the version of + GitLab. -Praefect switches a virtual storage in to read-only mode after a failover event. This eases data recovery efforts by preventing new, possibly conflicting writes to the newly elected primary. This allows the administrator to attempt recovering the lost data before allowing new writes. +### Check for data loss -If you prefer write availability over consistency, this behavior can be turned off by setting `praefect['failover_read_only_after_failover'] = false` in `/etc/gitlab/gitlab.rb` and [reconfiguring Praefect](../restart_gitlab.md#omnibus-gitlab-reconfigure). +The Praefect `dataloss` sub-command identifies replicas that are likely to be outdated. This is +useful for identifying potential data loss after a failover. The following parameters are +available: -### Checking for data loss +- `-virtual-storage` that specifies which virtual storage to check. The default behavior is to + display outdated replicas of read-only repositories as they generally require administrator + action. +- In GitLab 13.3 and later, `-partially-replicated` that specifies whether to display a list of + [outdated replicas of writable repositories](#outdated-replicas-of-writable-repositories). -The Praefect `dataloss` sub-command helps identify lost writes by checking for uncompleted replication jobs. This is useful for identifying possible data loss cases after a failover. This command must be executed on a Praefect node. +NOTE: **Note:** +`dataloss` is still in beta and the output format is subject to change. + +To check for outdated replicas of read-only repositories, run: ```shell sudo /opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml dataloss [-virtual-storage <virtual-storage>] ``` -If the virtual storage is not specified, every configured virtual storage is checked for data loss. +Every configured virtual storage is checked if none is specified: ```shell sudo /opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml dataloss ``` +The number of potentially unapplied changes to repositories is listed for each replica. Listed +repositories might have the latest changes but it is not guaranteed. Only outdated replicas of +read-only repositories are listed by default. For example: + ```shell Virtual storage: default - Current read-only primary: gitaly-2 - Previous write-enabled primary: gitaly-1 - Nodes with data loss from failing over from gitaly-1: - @hashed/2c/62/2c624232cdd221771294dfbb310aca000a0df6ac8b66b696d90ef06fdefb64a3.git: gitaly-0 - @hashed/4b/22/4b227777d4dd1fc61c6f884f48641d02b4d121d3fd328cb08b5531fcacdabf8a.git: gitaly-0, gitaly-2 + Primary: gitaly-3 + Outdated repositories: + @hashed/2c/62/2c624232cdd221771294dfbb310aca000a0df6ac8b66b696d90ef06fdefb64a3.git (read-only): + gitaly-2 is behind by 1 change or less + gitaly-3 is behind by 2 changes or less ``` -Currently `dataloss` only considers a repository up to date if it has been directly replicated to from the previous write-enabled primary. While reconciling from an up to date secondary can recover the data, this is not visible in the data loss report. This is due for improvement via [Gitaly#2866](https://gitlab.com/gitlab-org/gitaly/-/issues/2866). +A confirmation is printed out when every repository is writable. For example: -NOTE: **Note:** -`dataloss` is still in beta and the output format is subject to change. +```shell +Virtual storage: default + Primary: gitaly-1 + All repositories are writable! +``` -### Checking repository checksums +#### Outdated replicas of writable repositories -To check a project's repository checksums across on all Gitaly nodes, run the -[replicas Rake task](../raketasks/praefect.md#replica-checksums) on the main GitLab node. +> [Introduced](https://gitlab.com/gitlab-org/gitaly/-/issues/3019) in GitLab 13.3. -### Recovering lost writes +To also list information for outdated replicas of writable repositories, use the +`-partially-replicated` parameter. -The Praefect `reconcile` sub-command can be used to recover lost writes from the -previous primary once it is back online. This is only possible when the virtual storage -is still in read-only mode. +A repository is writable if the primary has the latest changes. Secondaries might be temporarily +outdated while they are waiting to replicate the latest changes. ```shell -sudo /opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml reconcile -virtual <virtual-storage> -reference <previous-primary> -target <current-primary> -f +sudo /opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml dataloss [-virtual-storage <virtual-storage>] [-partially-replicated] ``` -Refer to [Backend Node Recovery](#backend-node-recovery) section for more details on -the `reconcile` sub-command. +Example output: -### Enabling Writes +```shell +Virtual storage: default + Primary: gitaly-3 + Outdated repositories: + @hashed/2c/62/2c624232cdd221771294dfbb310aca000a0df6ac8b66b696d90ef06fdefb64a3.git (read-only): + gitaly-2 is behind by 1 change or less + gitaly-3 is behind by 2 changes or less + @hashed/4b/22/4b227777d4dd1fc61c6f884f48641d02b4d121d3fd328cb08b5531fcacdabf8a.git (writable): + gitaly-2 is behind by 1 change or less +``` -Any data recovery attempts should have been made before enabling writes to eliminate -any chance of conflicting writes. Virtual storage can be re-enabled for writes by using -the Praefect `enable-writes` sub-command. +With the `-partially-replicated` flag set, a confirmation is printed out if every replica is fully up to date. +For example: ```shell -sudo /opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml enable-writes -virtual-storage <virtual-storage> +Virtual storage: default + Primary: gitaly-1 + All repositories are up to date! ``` -## Backend Node Recovery +### Check repository checksums + +To check a project's repository checksums across on all Gitaly nodes, run the +[replicas Rake task](../raketasks/praefect.md#replica-checksums) on the main GitLab node. + +### Recover missing data + +The Praefect `reconcile` sub-command can be used to recover unreplicated changes from another replica. +The source must be on a later version than the target storage. + +```shell +sudo /opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml reconcile -virtual <virtual-storage> -reference <up-to-date-storage> -target <outdated-storage> -f +``` + +Refer to [Gitaly node recovery](#gitaly-node-recovery) section for more details on the `reconcile` sub-command. + +### Enable writes or accept data loss + +Praefect provides the following subcommands to re-enable writes: + +- In GitLab 13.2 and earlier, `enable-writes` to re-enable virtual storage for writes after data + recovery attempts. + + ```shell + sudo /opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml enable-writes -virtual-storage <virtual-storage> + ``` + +- [In GitLab 13.3](https://gitlab.com/gitlab-org/gitaly/-/merge_requests/2415) and later, + `accept-dataloss` to accept data loss and re-enable writes for repositories after data recovery + attempts have failed. Accepting data loss causes current version of the repository on the + authoritative storage to be considered latest. Other storages are brought up to date with the + authoritative storage by scheduling replication jobs. + + ```shell + sudo /opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml accept-dataloss -virtual-storage <virtual-storage> -repository <relative-path> -authoritative-storage <storage-name> + ``` + +CAUTION: **Caution:** +`accept-dataloss` causes permanent data loss by overwriting other versions of the repository. Data +[recovery efforts](#recover-missing-data) must be performed before using it. + +## Gitaly node recovery + +When a secondary Gitaly node fails and is no longer able to replicate changes, it starts +to drift from the primary Gitaly node. If the failed Gitaly node eventually recovers, +it needs to be reconciled with the primary Gitaly node. The primary Gitaly node is considered +the single source of truth for the state of a shard. -When a Praefect backend node fails and is no longer able to -replicate changes, the backend node will start to drift from the primary. If -that node eventually recovers, it will need to be reconciled with the current -primary. The primary node is considered the single source of truth for the -state of a shard. The Praefect `reconcile` sub-command allows for the manual -reconciliation between a backend node and the current primary. +The Praefect `reconcile` sub-command allows for the manual reconciliation between a secondary +Gitaly node and the current primary Gitaly node. Run the following command on the Praefect server after all placeholders (`<virtual-storage>` and `<target-storage>`) have been replaced: @@ -1018,8 +1216,8 @@ Run the following command on the Praefect server after all placeholders sudo /opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml reconcile -virtual <virtual-storage> -target <target-storage> ``` -- Replace the placeholder `<virtual-storage>` with the virtual storage containing the backend node storage to be checked. -- Replace the placeholder `<target-storage>` with the backend storage name. +- Replace the placeholder `<virtual-storage>` with the virtual storage containing the Gitaly node storage to be checked. +- Replace the placeholder `<target-storage>` with the Gitaly storage name. The command will return a list of repositories that were found to be inconsistent against the current primary. Each of these inconsistencies will @@ -1030,11 +1228,11 @@ also be logged with an accompanying replication job ID. If your GitLab instance already has repositories, these won't be migrated automatically. -Repositories may be moved from one storage location using the [Repository -API](../../api/projects.html#edit-project): +Repositories may be moved from one storage location using the [Project repository storage moves API](../../api/project_repository_storage_moves.md): ```shell -curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" --data "repository_storage=praefect" https://example.gitlab.com/api/v4/projects/123 +curl --request POST --header "Private-Token: <your_access_token>" --header "Content-Type: application/json" \ +--data '{"destination_storage_name":"praefect"}' "https://gitlab.example.com/api/v4/projects/123/repository_storage_moves" ``` ## Debugging Praefect diff --git a/doc/administration/gitaly/reference.md b/doc/administration/gitaly/reference.md index 0429149ec2d..0c211c220d7 100644 --- a/doc/administration/gitaly/reference.md +++ b/doc/administration/gitaly/reference.md @@ -233,7 +233,7 @@ The following values configure logging in Gitaly under the `[logging]` section. | `format` | string | no | Log format: `text` or `json`. Default: `text`. | | `level` | string | no | Log level: `debug`, `info`, `warn`, `error`, `fatal`, or `panic`. Default: `info`. | | `sentry_dsn` | string | no | Sentry DSN for exception monitoring. | -| `sentry_environment` | string | no | [Sentry Environment](https://docs.sentry.io/enriching-error-data/environments/) for exception monitoring. | +| `sentry_environment` | string | no | [Sentry Environment](https://docs.sentry.io/product/sentry-basics/environments/) for exception monitoring. | | `ruby_sentry_dsn` | string | no | Sentry DSN for `gitaly-ruby` exception monitoring. | While the main Gitaly application logs go to `stdout`, there are some extra log diff --git a/doc/administration/high_availability/alpha_database.md b/doc/administration/high_availability/alpha_database.md index 7afd739f44c..99c28e5c7a6 100644 --- a/doc/administration/high_availability/alpha_database.md +++ b/doc/administration/high_availability/alpha_database.md @@ -2,4 +2,4 @@ redirect_to: 'database.md' --- -This document was moved to [another location](database.md). +This document was moved to [another location](../postgresql/index.md). diff --git a/doc/administration/high_availability/consul.md b/doc/administration/high_availability/consul.md index 978ba08c4fa..362d6ee8ba7 100644 --- a/doc/administration/high_availability/consul.md +++ b/doc/administration/high_availability/consul.md @@ -1,198 +1,5 @@ --- -type: reference +redirect_to: ../consul.md --- -# Working with the bundled Consul service **(PREMIUM ONLY)** - -As part of its High Availability stack, GitLab Premium includes a bundled version of [Consul](https://www.consul.io/) that can be managed through `/etc/gitlab/gitlab.rb`. Consul is a service networking solution. When it comes to [GitLab Architecture](../../development/architecture.md), Consul utilization is supported for configuring: - -1. [Monitoring in Scaled and Highly Available environments](monitoring_node.md) -1. [PostgreSQL High Availability with Omnibus](../postgresql/replication_and_failover.md) - -A Consul cluster consists of multiple server agents, as well as client agents that run on other nodes which need to talk to the Consul cluster. - -## Prerequisites - -First, make sure to [download/install](https://about.gitlab.com/install/) -Omnibus GitLab **on each node**. - -Choose an installation method, then make sure you complete steps: - -1. Install and configure the necessary dependencies. -1. Add the GitLab package repository and install the package. - -When installing the GitLab package, do not supply `EXTERNAL_URL` value. - -## Configuring the Consul nodes - -On each Consul node perform the following: - -1. Make sure you collect [`CONSUL_SERVER_NODES`](../postgresql/replication_and_failover.md#consul-information), which are the IP addresses or DNS records of the Consul server nodes, for the next step, before executing the next step. - -1. Edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: - - ```ruby - # Disable all components except Consul - roles ['consul_role'] - - # START user configuration - # Replace placeholders: - # - # Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z - # with the addresses gathered for CONSUL_SERVER_NODES - consul['configuration'] = { - server: true, - retry_join: %w(Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z) - } - - # Disable auto migrations - gitlab_rails['auto_migrate'] = false - # - # END user configuration - ``` - - > `consul_role` was introduced with GitLab 10.3 - -1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes - to take effect. - -### Consul checkpoint - -Before moving on, make sure Consul is configured correctly. Run the following -command to verify all server nodes are communicating: - -```shell -/opt/gitlab/embedded/bin/consul members -``` - -The output should be similar to: - -```plaintext -Node Address Status Type Build Protocol DC -CONSUL_NODE_ONE XXX.XXX.XXX.YYY:8301 alive server 0.9.2 2 gitlab_consul -CONSUL_NODE_TWO XXX.XXX.XXX.YYY:8301 alive server 0.9.2 2 gitlab_consul -CONSUL_NODE_THREE XXX.XXX.XXX.YYY:8301 alive server 0.9.2 2 gitlab_consul -``` - -If any of the nodes isn't `alive` or if any of the three nodes are missing, -check the [Troubleshooting section](#troubleshooting) before proceeding. - -## Operations - -### Checking cluster membership - -To see which nodes are part of the cluster, run the following on any member in the cluster - -```shell -$ /opt/gitlab/embedded/bin/consul members -Node Address Status Type Build Protocol DC -consul-b XX.XX.X.Y:8301 alive server 0.9.0 2 gitlab_consul -consul-c XX.XX.X.Y:8301 alive server 0.9.0 2 gitlab_consul -consul-c XX.XX.X.Y:8301 alive server 0.9.0 2 gitlab_consul -db-a XX.XX.X.Y:8301 alive client 0.9.0 2 gitlab_consul -db-b XX.XX.X.Y:8301 alive client 0.9.0 2 gitlab_consul -``` - -Ideally all nodes will have a `Status` of `alive`. - -### Restarting the server cluster - -NOTE: **Note:** -This section only applies to server agents. It is safe to restart client agents whenever needed. - -If it is necessary to restart the server cluster, it is important to do this in a controlled fashion in order to maintain quorum. If quorum is lost, you will need to follow the Consul [outage recovery](#outage-recovery) process to recover the cluster. - -To be safe, we recommend you only restart one server agent at a time to ensure the cluster remains intact. - -For larger clusters, it is possible to restart multiple agents at a time. See the [Consul consensus document](https://www.consul.io/docs/internals/consensus.html#deployment-table) for how many failures it can tolerate. This will be the number of simultaneous restarts it can sustain. - -## Upgrades for bundled Consul - -Nodes running GitLab-bundled Consul should be: - -- Members of a healthy cluster prior to upgrading the Omnibus GitLab package. -- Upgraded one node at a time. - -NOTE: **Note:** -Running `curl http://127.0.0.1:8500/v1/health/state/critical` from any Consul node will identify existing health issues in the cluster. The command will return an empty array if the cluster is healthy. - -Consul clusters communicate using the raft protocol. If the current leader goes offline, there needs to be a leader election. A leader node must exist to facilitate synchronization across the cluster. If too many nodes go offline at the same time, the cluster will lose quorum and not elect a leader due to [broken consensus](https://www.consul.io/docs/internals/consensus.html). - -Consult the [troubleshooting section](#troubleshooting) if the cluster is not able to recover after the upgrade. The [outage recovery](#outage-recovery) may be of particular interest. - -NOTE: **Note:** -GitLab only uses Consul to store transient data that is easily regenerated. If the bundled Consul was not used by any process other than GitLab itself, then [rebuilding the cluster from scratch](#recreate-from-scratch) is fine. - -## Troubleshooting - -### Consul server agents unable to communicate - -By default, the server agents will attempt to [bind](https://www.consul.io/docs/agent/options.html#_bind) to '0.0.0.0', but they will advertise the first private IP address on the node for other agents to communicate with them. If the other nodes cannot communicate with a node on this address, then the cluster will have a failed status. - -You will see messages like the following in `gitlab-ctl tail consul` output if you are running into this issue: - -```plaintext -2017-09-25_19:53:39.90821 2017/09/25 19:53:39 [WARN] raft: no known peers, aborting election -2017-09-25_19:53:41.74356 2017/09/25 19:53:41 [ERR] agent: failed to sync remote state: No cluster leader -``` - -To fix this: - -1. Pick an address on each node that all of the other nodes can reach this node through. -1. Update your `/etc/gitlab/gitlab.rb` - - ```ruby - consul['configuration'] = { - ... - bind_addr: 'IP ADDRESS' - } - ``` - -1. Run `gitlab-ctl reconfigure` - -If you still see the errors, you may have to [erase the Consul database and reinitialize](#recreate-from-scratch) on the affected node. - -### Consul agents do not start - Multiple private IPs - -In the case that a node has multiple private IPs the agent be confused as to which of the private addresses to advertise, and then immediately exit on start. - -You will see messages like the following in `gitlab-ctl tail consul` output if you are running into this issue: - -```plaintext -2017-11-09_17:41:45.52876 ==> Starting Consul agent... -2017-11-09_17:41:45.53057 ==> Error creating agent: Failed to get advertise address: Multiple private IPs found. Please configure one. -``` - -To fix this: - -1. Pick an address on the node that all of the other nodes can reach this node through. -1. Update your `/etc/gitlab/gitlab.rb` - - ```ruby - consul['configuration'] = { - ... - bind_addr: 'IP ADDRESS' - } - ``` - -1. Run `gitlab-ctl reconfigure` - -### Outage recovery - -If you lost enough server agents in the cluster to break quorum, then the cluster is considered failed, and it will not function without manual intervention. - -#### Recreate from scratch - -By default, GitLab does not store anything in the Consul cluster that cannot be recreated. To erase the Consul database and reinitialize - -```shell -gitlab-ctl stop consul -rm -rf /var/opt/gitlab/consul/data -gitlab-ctl start consul -``` - -After this, the cluster should start back up, and the server agents rejoin. Shortly after that, the client agents should rejoin as well. - -#### Recover a failed cluster - -If you have taken advantage of Consul to store other data, and want to restore the failed cluster, please follow the [Consul guide](https://learn.hashicorp.com/consul/day-2-operations/outage) to recover a failed cluster. +This document was moved to [another location](../consul.md). diff --git a/doc/administration/high_availability/gitaly.md b/doc/administration/high_availability/gitaly.md index 230f5baf33a..a1e8fe3b488 100644 --- a/doc/administration/high_availability/gitaly.md +++ b/doc/administration/high_availability/gitaly.md @@ -1,60 +1,5 @@ --- -stage: Create -group: Gitaly -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers -type: reference +redirect_to: ../gitaly/index.md --- -# Configuring Gitaly for Scaled and High Availability - -A [Gitaly Cluster](../gitaly/praefect.md) can be used to increase the fault -tolerance of Gitaly in high availability configurations. - -This document is relevant for [scalable and highly available setups](../reference_architectures/index.md). - -## Running Gitaly on its own server - -See [Run Gitaly on its own server](../gitaly/index.md#run-gitaly-on-its-own-server) -in Gitaly documentation. - -Continue configuration of other components by going back to the -[reference architecture](../reference_architectures/index.md#configure-gitlab-to-scale) page. - -## Enable Monitoring - -> [Introduced](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/3786) in GitLab 12.0. - -1. Make sure to collect [`CONSUL_SERVER_NODES`](../postgresql/replication_and_failover.md#consul-information), which are the IP addresses or DNS records of the Consul server nodes, for the next step. Note they are presented as `Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z` - -1. Create/edit `/etc/gitlab/gitlab.rb` and add the following configuration: - - ```ruby - # Enable service discovery for Prometheus - consul['enable'] = true - consul['monitoring_service_discovery'] = true - - # Replace placeholders - # Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z - # with the addresses of the Consul server nodes - consul['configuration'] = { - retry_join: %w(Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z), - } - - # Set the network addresses that the exporters will listen on - node_exporter['listen_address'] = '0.0.0.0:9100' - gitaly['prometheus_listen_addr'] = "0.0.0.0:9236" - ``` - -1. Run `sudo gitlab-ctl reconfigure` to compile the configuration. - -<!-- ## Troubleshooting - -Include any troubleshooting steps that you can foresee. If you know beforehand what issues -one might have when setting this up, or when something is changed, or on upgrading, it's -important to describe those, too. Think of things that may go wrong and include them here. -This is important to minimize requests for support, and to avoid doc comments with -questions that you know someone might ask. - -Each scenario can be a third-level heading, e.g. `### Getting error message X`. -If you have none to add when creating a doc, leave this section in place -but commented out to help encourage others to add to it in the future. --> +This document was moved to [another location](../gitaly/index.md). diff --git a/doc/administration/high_availability/gitlab.md b/doc/administration/high_availability/gitlab.md index dc8c997bab5..748373c8941 100644 --- a/doc/administration/high_availability/gitlab.md +++ b/doc/administration/high_availability/gitlab.md @@ -1,215 +1,5 @@ --- -type: reference +redirect_to: ../reference_architectures/index.md --- -# Configuring GitLab application (Rails) - -This section describes how to configure the GitLab application (Rails) component. - -NOTE: **Note:** -There is some additional configuration near the bottom for -additional GitLab application servers. It's important to read and understand -these additional steps before proceeding with GitLab installation. - -NOTE: **Note:** -[Cloud Object Storage service](object_storage.md) with [Gitaly](gitaly.md) -is recommended over [NFS](nfs.md) wherever possible for improved performance. - -1. If necessary, install the NFS client utility packages using the following - commands: - - ```shell - # Ubuntu/Debian - apt-get install nfs-common - - # CentOS/Red Hat - yum install nfs-utils nfs-utils-lib - ``` - -1. Specify the necessary NFS exports in `/etc/fstab`. - The exact contents of `/etc/fstab` will depend on how you chose - to configure your NFS server. See [NFS documentation](nfs.md#nfs-client-mount-options) - for examples and the various options. - -1. Create the shared directories. These may be different depending on your NFS - mount locations. - - ```shell - mkdir -p /var/opt/gitlab/.ssh /var/opt/gitlab/gitlab-rails/uploads /var/opt/gitlab/gitlab-rails/shared /var/opt/gitlab/gitlab-ci/builds /var/opt/gitlab/git-data - ``` - -1. Download/install Omnibus GitLab using **steps 1 and 2** from - [GitLab downloads](https://about.gitlab.com/install/). Do not complete other - steps on the download page. -1. Create/edit `/etc/gitlab/gitlab.rb` and use the following configuration. - Be sure to change the `external_url` to match your eventual GitLab front-end - URL. Depending your the NFS configuration, you may need to change some GitLab - data locations. See [NFS documentation](nfs.md) for `/etc/gitlab/gitlab.rb` - configuration values for various scenarios. The example below assumes you've - added NFS mounts in the default data locations. Additionally the UID and GIDs - given are just examples and you should configure with your preferred values. - - ```ruby - external_url 'https://gitlab.example.com' - - # Prevent GitLab from starting if NFS data mounts are not available - high_availability['mountpoint'] = '/var/opt/gitlab/git-data' - - # Disable components that will not be on the GitLab application server - roles ['application_role'] - nginx['enable'] = true - - # PostgreSQL connection details - gitlab_rails['db_adapter'] = 'postgresql' - gitlab_rails['db_encoding'] = 'unicode' - gitlab_rails['db_host'] = '10.1.0.5' # IP/hostname of database server - gitlab_rails['db_password'] = 'DB password' - - # Redis connection details - gitlab_rails['redis_port'] = '6379' - gitlab_rails['redis_host'] = '10.1.0.6' # IP/hostname of Redis server - gitlab_rails['redis_password'] = 'Redis Password' - - # Ensure UIDs and GIDs match between servers for permissions via NFS - user['uid'] = 9000 - user['gid'] = 9000 - web_server['uid'] = 9001 - web_server['gid'] = 9001 - registry['uid'] = 9002 - registry['gid'] = 9002 - ``` - -1. [Enable monitoring](#enable-monitoring) - - NOTE: **Note:** - To maintain uniformity of links across HA clusters, the `external_url` - on the first application server as well as the additional application - servers should point to the external URL that users will use to access GitLab. - In a typical HA setup, this will be the URL of the load balancer which will - route traffic to all GitLab application servers in the HA cluster. - - NOTE: **Note:** - When you specify `https` in the `external_url`, as in the example - above, GitLab assumes you have SSL certificates in `/etc/gitlab/ssl/`. If - certificates are not present, NGINX will fail to start. See - [NGINX documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https) - for more information. - - NOTE: **Note:** - It is best to set the `uid` and `gid`s prior to the initial reconfigure - of GitLab. Omnibus will not recursively `chown` directories if set after the initial reconfigure. - -## First GitLab application server - -On the first application server, run: - -```shell -sudo gitlab-ctl reconfigure -``` - -This should compile the configuration and initialize the database. Do -not run this on additional application servers until the next step. - -## Extra configuration for additional GitLab application servers - -Additional GitLab servers (servers configured **after** the first GitLab server) -need some extra configuration. - -1. Configure shared secrets. These values can be obtained from the primary - GitLab server in `/etc/gitlab/gitlab-secrets.json`. Copy this file to the - secondary servers **prior to** running the first `reconfigure` in the steps - above. - - ```ruby - gitlab_shell['secret_token'] = 'fbfb19c355066a9afb030992231c4a363357f77345edd0f2e772359e5be59b02538e1fa6cae8f93f7d23355341cea2b93600dab6d6c3edcdced558fc6d739860' - gitlab_rails['otp_key_base'] = 'b719fe119132c7810908bba18315259ed12888d4f5ee5430c42a776d840a396799b0a5ef0a801348c8a357f07aa72bbd58e25a84b8f247a25c72f539c7a6c5fa' - gitlab_rails['secret_key_base'] = '6e657410d57c71b4fc3ed0d694e7842b1895a8b401d812c17fe61caf95b48a6d703cb53c112bc01ebd197a85da81b18e29682040e99b4f26594772a4a2c98c6d' - gitlab_rails['db_key_base'] = 'bf2e47b68d6cafaef1d767e628b619365becf27571e10f196f98dc85e7771042b9203199d39aff91fcb6837c8ed83f2a912b278da50999bb11a2fbc0fba52964' - ``` - -1. Run `touch /etc/gitlab/skip-auto-reconfigure` to prevent database migrations - from running on upgrade. Only the primary GitLab application server should - handle migrations. - -1. **Recommended** Configure host keys. Copy the contents (private and public keys) of `/etc/ssh/` on - the primary application server to `/etc/ssh` on all secondary servers. This - prevents false man-in-the-middle-attack alerts when accessing servers in your - High Availability cluster behind a load balancer. - -1. Run `sudo gitlab-ctl reconfigure` to compile the configuration. - -NOTE: **Note:** -You will need to restart the GitLab applications nodes after an update has occurred and database -migrations performed. - -## Enable Monitoring - -> [Introduced](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/3786) in GitLab 12.0. - -If you enable Monitoring, it must be enabled on **all** GitLab servers. - -1. Make sure to collect [`CONSUL_SERVER_NODES`](../postgresql/replication_and_failover.md#consul-information), which are the IP addresses or DNS records of the Consul server nodes, for the next step. Note they are presented as `Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z` - -1. Create/edit `/etc/gitlab/gitlab.rb` and add the following configuration: - - ```ruby - # Enable service discovery for Prometheus - consul['enable'] = true - consul['monitoring_service_discovery'] = true - - # Replace placeholders - # Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z - # with the addresses of the Consul server nodes - consul['configuration'] = { - retry_join: %w(Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z), - } - - # Set the network addresses that the exporters will listen on - node_exporter['listen_address'] = '0.0.0.0:9100' - gitlab_workhorse['prometheus_listen_addr'] = '0.0.0.0:9229' - sidekiq['listen_address'] = "0.0.0.0" - puma['listen'] = '0.0.0.0' - - # Add the monitoring node's IP address to the monitoring whitelist and allow it to - # scrape the NGINX metrics. Replace placeholder `monitoring.gitlab.example.com` with - # the address and/or subnets gathered from the monitoring node(s). - gitlab_rails['monitoring_whitelist'] = ['monitoring.gitlab.example.com', '127.0.0.0/8'] - nginx['status']['options']['allow'] = ['monitoring.gitlab.example.com', '127.0.0.0/8'] - ``` - -1. Run `sudo gitlab-ctl reconfigure` to compile the configuration. - - CAUTION: **Warning:** - If running Unicorn, after changing `unicorn['listen']` in `gitlab.rb`, and - running `sudo gitlab-ctl reconfigure`, it can take an extended period of time - for Unicorn to complete reloading after receiving a `HUP`. For more - information, see the - [issue](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/4401). - -## Troubleshooting - -- `mount: wrong fs type, bad option, bad superblock on` - -You have not installed the necessary NFS client utilities. See step 1 above. - -- `mount: mount point /var/opt/gitlab/... does not exist` - -This particular directory does not exist on the NFS server. Ensure -the share is exported and exists on the NFS server and try to remount. - ---- - -## Upgrading GitLab HA - -GitLab HA installations can be upgraded with no downtime, but the -upgrade process must be carefully coordinated to avoid failures. See the -[Omnibus GitLab multi-node upgrade -document](https://docs.gitlab.com/omnibus/update/#multi-node--ha-deployment) -for more details. - -Read more on high-availability configuration: - -1. [Configure the database](../postgresql/replication_and_failover.md) -1. [Configure Redis](redis.md) -1. [Configure NFS](nfs.md) -1. [Configure the load balancers](load_balancer.md) +This document was moved to [another location](../reference_architectures/index.md). diff --git a/doc/administration/high_availability/load_balancer.md b/doc/administration/high_availability/load_balancer.md index 75703327140..5cedc4e11ae 100644 --- a/doc/administration/high_availability/load_balancer.md +++ b/doc/administration/high_availability/load_balancer.md @@ -1,135 +1,5 @@ --- -type: reference +redirect_to: ../load_balancer.md --- -# Load Balancer for multi-node GitLab - -In an multi-node GitLab configuration, you will need a load balancer to route -traffic to the application servers. The specifics on which load balancer to use -or the exact configuration is beyond the scope of GitLab documentation. We hope -that if you're managing HA systems like GitLab you have a load balancer of -choice already. Some examples including HAProxy (open-source), F5 Big-IP LTM, -and Citrix Net Scaler. This documentation will outline what ports and protocols -you need to use with GitLab. - -## SSL - -How will you handle SSL in your multi-node environment? There are several different -options: - -- Each application node terminates SSL -- The load balancer(s) terminate SSL and communication is not secure between - the load balancer(s) and the application nodes -- The load balancer(s) terminate SSL and communication is *secure* between the - load balancer(s) and the application nodes - -### Application nodes terminate SSL - -Configure your load balancer(s) to pass connections on port 443 as 'TCP' rather -than 'HTTP(S)' protocol. This will pass the connection to the application nodes -NGINX service untouched. NGINX will have the SSL certificate and listen on port 443. - -See [NGINX HTTPS documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https) -for details on managing SSL certificates and configuring NGINX. - -### Load Balancer(s) terminate SSL without backend SSL - -Configure your load balancer(s) to use the 'HTTP(S)' protocol rather than 'TCP'. -The load balancer(s) will then be responsible for managing SSL certificates and -terminating SSL. - -Since communication between the load balancer(s) and GitLab will not be secure, -there is some additional configuration needed. See -[NGINX Proxied SSL documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#supporting-proxied-ssl) -for details. - -### Load Balancer(s) terminate SSL with backend SSL - -Configure your load balancer(s) to use the 'HTTP(S)' protocol rather than 'TCP'. -The load balancer(s) will be responsible for managing SSL certificates that -end users will see. - -Traffic will also be secure between the load balancer(s) and NGINX in this -scenario. There is no need to add configuration for proxied SSL since the -connection will be secure all the way. However, configuration will need to be -added to GitLab to configure SSL certificates. See -[NGINX HTTPS documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https) -for details on managing SSL certificates and configuring NGINX. - -## Ports - -### Basic ports - -| LB Port | Backend Port | Protocol | -| ------- | ------------ | ------------------------ | -| 80 | 80 | HTTP (*1*) | -| 443 | 443 | TCP or HTTPS (*1*) (*2*) | -| 22 | 22 | TCP | - -- (*1*): [Web terminal](../../ci/environments/index.md#web-terminals) support requires - your load balancer to correctly handle WebSocket connections. When using - HTTP or HTTPS proxying, this means your load balancer must be configured - to pass through the `Connection` and `Upgrade` hop-by-hop headers. See the - [web terminal](../integration/terminal.md) integration guide for - more details. -- (*2*): When using HTTPS protocol for port 443, you will need to add an SSL - certificate to the load balancers. If you wish to terminate SSL at the - GitLab application server instead, use TCP protocol. - -### GitLab Pages Ports - -If you're using GitLab Pages with custom domain support you will need some -additional port configurations. -GitLab Pages requires a separate virtual IP address. Configure DNS to point the -`pages_external_url` from `/etc/gitlab/gitlab.rb` at the new virtual IP address. See the -[GitLab Pages documentation](../pages/index.md) for more information. - -| LB Port | Backend Port | Protocol | -| ------- | ------------- | --------- | -| 80 | Varies (*1*) | HTTP | -| 443 | Varies (*1*) | TCP (*2*) | - -- (*1*): The backend port for GitLab Pages depends on the - `gitlab_pages['external_http']` and `gitlab_pages['external_https']` - setting. See [GitLab Pages documentation](../pages/index.md) for more details. -- (*2*): Port 443 for GitLab Pages should always use the TCP protocol. Users can - configure custom domains with custom SSL, which would not be possible - if SSL was terminated at the load balancer. - -### Alternate SSH Port - -Some organizations have policies against opening SSH port 22. In this case, -it may be helpful to configure an alternate SSH hostname that allows users -to use SSH on port 443. An alternate SSH hostname will require a new virtual IP address -compared to the other GitLab HTTP configuration above. - -Configure DNS for an alternate SSH hostname such as `altssh.gitlab.example.com`. - -| LB Port | Backend Port | Protocol | -| ------- | ------------ | -------- | -| 443 | 22 | TCP | - -## Readiness check - -It is strongly recommend that multi-node deployments configure load balancers to utilize the [readiness check](../../user/admin_area/monitoring/health_check.md#readiness) to ensure a node is ready to accept traffic, before routing traffic to it. This is especially important when utilizing Puma, as there is a brief period during a restart where Puma will not accept requests. - ---- - -Read more on high-availability configuration: - -1. [Configure the database](../postgresql/replication_and_failover.md) -1. [Configure Redis](redis.md) -1. [Configure NFS](nfs.md) -1. [Configure the GitLab application servers](gitlab.md) - -<!-- ## Troubleshooting - -Include any troubleshooting steps that you can foresee. If you know beforehand what issues -one might have when setting this up, or when something is changed, or on upgrading, it's -important to describe those, too. Think of things that may go wrong and include them here. -This is important to minimize requests for support, and to avoid doc comments with -questions that you know someone might ask. - -Each scenario can be a third-level heading, e.g. `### Getting error message X`. -If you have none to add when creating a doc, leave this section in place -but commented out to help encourage others to add to it in the future. --> +This document was moved to [another location](../load_balancer.md). diff --git a/doc/administration/high_availability/monitoring_node.md b/doc/administration/high_availability/monitoring_node.md index 6b6f0ae9ea3..76bcf6d0d40 100644 --- a/doc/administration/high_availability/monitoring_node.md +++ b/doc/administration/high_availability/monitoring_node.md @@ -1,104 +1,5 @@ --- -type: reference +redirect_to: ../monitoring/prometheus/index.md --- -# Configuring a Monitoring node for Scaling and High Availability - -> [Introduced](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/3786) in GitLab 12.0. - -You can configure a Prometheus node to monitor GitLab. - -## Standalone Monitoring node using Omnibus GitLab - -The Omnibus GitLab package can be used to configure a standalone Monitoring node running [Prometheus](../monitoring/prometheus/index.md) and [Grafana](../monitoring/performance/grafana_configuration.md). -The monitoring node is not highly available. See [Scaling and High Availability](../reference_architectures/index.md) -for an overview of GitLab scaling and high availability options. - -The steps below are the minimum necessary to configure a Monitoring node running Prometheus and Grafana with -Omnibus: - -1. SSH into the Monitoring node. -1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab - package you want using **steps 1 and 2** from the GitLab downloads page. - - Do not complete any other steps on the download page. - -1. Make sure to collect [`CONSUL_SERVER_NODES`](../postgresql/replication_and_failover.md#consul-information), which are the IP addresses or DNS records of the Consul server nodes, for the next step. Note they are presented as `Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z` - -1. Edit `/etc/gitlab/gitlab.rb` and add the contents: - - ```ruby - external_url 'http://gitlab.example.com' - - # Enable Prometheus - prometheus['enable'] = true - prometheus['listen_address'] = '0.0.0.0:9090' - prometheus['monitor_kubernetes'] = false - - # Enable Login form - grafana['disable_login_form'] = false - - # Enable Grafana - grafana['enable'] = true - grafana['admin_password'] = 'toomanysecrets' - - # Enable service discovery for Prometheus - consul['enable'] = true - consul['monitoring_service_discovery'] = true - - # Replace placeholders - # Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z - # with the addresses of the Consul server nodes - consul['configuration'] = { - retry_join: %w(Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z), - } - - # Disable all other services - gitlab_rails['auto_migrate'] = false - alertmanager['enable'] = false - gitaly['enable'] = false - gitlab_exporter['enable'] = false - gitlab_workhorse['enable'] = false - nginx['enable'] = true - postgres_exporter['enable'] = false - postgresql['enable'] = false - redis['enable'] = false - redis_exporter['enable'] = false - sidekiq['enable'] = false - puma['enable'] = false - node_exporter['enable'] = false - gitlab_exporter['enable'] = false - ``` - -1. Run `sudo gitlab-ctl reconfigure` to compile the configuration. - -The next step is to tell all the other nodes where the monitoring node is: - -1. Edit `/etc/gitlab/gitlab.rb`, and add, or find and uncomment the following line: - - ```ruby - gitlab_rails['prometheus_address'] = '10.0.0.1:9090' - ``` - - Where `10.0.0.1:9090` is the IP address and port of the Prometheus node. - -1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to - take effect. - -## Migrating to Service Discovery - -Once monitoring using Service Discovery is enabled with `consul['monitoring_service_discovery'] = true`, -ensure that `prometheus['scrape_configs']` is not set in `/etc/gitlab/gitlab.rb`. Setting both -`consul['monitoring_service_discovery'] = true` and `prometheus['scrape_configs']` in `/etc/gitlab/gitlab.rb` -will result in errors. - -<!-- ## Troubleshooting - -Include any troubleshooting steps that you can foresee. If you know beforehand what issues -one might have when setting this up, or when something is changed, or on upgrading, it's -important to describe those, too. Think of things that may go wrong and include them here. -This is important to minimize requests for support, and to avoid doc comments with -questions that you know someone might ask. - -Each scenario can be a third-level heading, e.g. `### Getting error message X`. -If you have none to add when creating a doc, leave this section in place -but commented out to help encourage others to add to it in the future. --> +This document was moved to [another location](../monitoring/prometheus/index.md). diff --git a/doc/administration/high_availability/nfs.md b/doc/administration/high_availability/nfs.md index 6e8dc2c6c57..e3342fa0813 100644 --- a/doc/administration/high_availability/nfs.md +++ b/doc/administration/high_availability/nfs.md @@ -1,322 +1,5 @@ --- -type: reference +redirect_to: ../nfs.md --- -# NFS - -You can view information and options set for each of the mounted NFS file -systems by running `nfsstat -m` and `cat /etc/fstab`. - -CAUTION: **Caution:** -From GitLab 13.0, using NFS for Git repositories is deprecated. In GitLab 14.0, -support for NFS for Git repositories is scheduled to be removed. Upgrade to -[Gitaly Cluster](../gitaly/praefect.md) as soon as possible. - -NOTE: **Note:** -Filesystem performance has a big impact on overall GitLab -performance, especially for actions that read or write to Git repositories. See -[Filesystem Performance Benchmarking](../operations/filesystem_benchmarking.md) -for steps to test filesystem performance. - -## Known kernel version incompatibilities - -RedHat Enterprise Linux (RHEL) and CentOS v7.7 and v7.8 ship with kernel -version `3.10.0-1127`, which [contains a -bug](https://bugzilla.redhat.com/show_bug.cgi?id=1783554) that causes -[uploads to fail to copy over NFS](https://gitlab.com/gitlab-org/gitlab/-/issues/218999). The -following GitLab versions include a fix to work properly with that -kernel version: - -1. [12.10.12](https://about.gitlab.com/releases/2020/06/25/gitlab-12-10-12-released/) -1. [13.0.7](https://about.gitlab.com/releases/2020/06/25/gitlab-13-0-7-released/) -1. [13.1.1](https://about.gitlab.com/releases/2020/06/24/gitlab-13-1-1-released/) -1. 13.2 and up - -If you are using that kernel version, be sure to upgrade GitLab to avoid -errors. - -## NFS Server features - -### Required features - -**File locking**: GitLab **requires** advisory file locking, which is only -supported natively in NFS version 4. NFSv3 also supports locking as long as -Linux Kernel 2.6.5+ is used. We recommend using version 4 and do not -specifically test NFSv3. - -### Recommended options - -When you define your NFS exports, we recommend you also add the following -options: - -- `no_root_squash` - NFS normally changes the `root` user to `nobody`. This is - a good security measure when NFS shares will be accessed by many different - users. However, in this case only GitLab will use the NFS share so it - is safe. GitLab recommends the `no_root_squash` setting because we need to - manage file permissions automatically. Without the setting you may receive - errors when the Omnibus package tries to alter permissions. Note that GitLab - and other bundled components do **not** run as `root` but as non-privileged - users. The recommendation for `no_root_squash` is to allow the Omnibus package - to set ownership and permissions on files, as needed. In some cases where the - `no_root_squash` option is not available, the `root` flag can achieve the same - result. -- `sync` - Force synchronous behavior. Default is asynchronous and under certain - circumstances it could lead to data loss if a failure occurs before data has - synced. - -Due to the complexities of running Omnibus with LDAP and the complexities of -maintaining ID mapping without LDAP, in most cases you should enable numeric UIDs -and GIDs (which is off by default in some cases) for simplified permission -management between systems: - -- [NetApp instructions](https://library.netapp.com/ecmdocs/ECMP1401220/html/GUID-24367A9F-E17B-4725-ADC1-02D86F56F78E.html) -- For non-NetApp devices, disable NFSv4 `idmapping` by performing opposite of [enable NFSv4 idmapper](https://wiki.archlinux.org/index.php/NFS#Enabling_NFSv4_idmapping) - -### Disable NFS server delegation - -We recommend that all NFS users disable the NFS server delegation feature. This -is to avoid a [Linux kernel bug](https://bugzilla.redhat.com/show_bug.cgi?id=1552203) -which causes NFS clients to slow precipitously due to -[excessive network traffic from numerous `TEST_STATEID` NFS messages](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/52017). - -To disable NFS server delegation, do the following: - -1. On the NFS server, run: - - ```shell - echo 0 > /proc/sys/fs/leases-enable - sysctl -w fs.leases-enable=0 - ``` - -1. Restart the NFS server process. For example, on CentOS run `service nfs restart`. - -#### Important notes - -The kernel bug may be fixed in -[more recent kernels with this commit](https://github.com/torvalds/linux/commit/95da1b3a5aded124dd1bda1e3cdb876184813140). - -Red Hat Enterprise 7 [shipped a kernel update](https://access.redhat.com/errata/RHSA-2019:2029) -on August 6, 2019 that may also have resolved this problem. - -You may not need to disable NFS server delegation if you know you are using a version of -the Linux kernel that has been fixed. That said, GitLab still encourages instance -administrators to keep NFS server delegation disabled. - -### Improving NFS performance with GitLab - -#### Improving NFS performance with Unicorn - -NOTE: **Note:** -From GitLab 12.1, it will automatically be detected if Rugged can and should be used per storage. - -If you previously enabled Rugged using the feature flag, you will need to unset the feature flag by using: - -```shell -sudo gitlab-rake gitlab:features:unset_rugged -``` - -If the Rugged feature flag is explicitly set to either true or false, GitLab will use the value explicitly set. - -#### Improving NFS performance with Puma - -NOTE: **Note:** -From GitLab 12.7, Rugged auto-detection is disabled if Puma thread count is greater than 1. - -If you want to use Rugged with Puma, it is recommended to [set Puma thread count to 1](https://docs.gitlab.com/omnibus/settings/puma.html#puma-settings). - -If you want to use Rugged with Puma thread count more than 1, Rugged can be enabled using the [feature flag](../../development/gitaly.md#legacy-rugged-code) - -If the Rugged feature flag is explicitly set to either true or false, GitLab will use the value explicitly set. - -### Known issues - -#### Avoid using AWS's Elastic File System (EFS) - -GitLab strongly recommends against using AWS Elastic File System (EFS). -Our support team will not be able to assist on performance issues related to -file system access. - -Customers and users have reported that AWS EFS does not perform well for GitLab's -use-case. Workloads where many small files are written in a serialized manner, like `git`, -are not well-suited for EFS. EBS with an NFS server on top will perform much better. - -If you do choose to use EFS, avoid storing GitLab log files (e.g. those in `/var/log/gitlab`) -there because this will also affect performance. We recommend that the log files be -stored on a local volume. - -For more details on another person's experience with EFS, see this [Commit Brooklyn 2019 video](https://youtu.be/K6OS8WodRBQ?t=313). - -#### Avoid using CephFS and GlusterFS - -GitLab strongly recommends against using CephFS and GlusterFS. -These distributed file systems are not well-suited for GitLab's input/output access patterns because Git uses many small files and access times and file locking times to propagate will make Git activity very slow. - -#### Avoid using PostgreSQL with NFS - -GitLab strongly recommends against running your PostgreSQL database -across NFS. The GitLab support team will not be able to assist on performance issues related to -this configuration. - -Additionally, this configuration is specifically warned against in the -[PostgreSQL Documentation](https://www.postgresql.org/docs/current/creating-cluster.html#CREATING-CLUSTER-NFS): - ->PostgreSQL does nothing special for NFS file systems, meaning it assumes NFS behaves exactly like ->locally-connected drives. If the client or server NFS implementation does not provide standard file ->system semantics, this can cause reliability problems. Specifically, delayed (asynchronous) writes ->to the NFS server can cause data corruption problems. - -For supported database architecture, please see our documentation on -[Configuring a Database for GitLab HA](../postgresql/replication_and_failover.md). - -## NFS Client mount options - -Here is an example snippet to add to `/etc/fstab`: - - ```plaintext - 10.1.0.1:/var/opt/gitlab/.ssh /var/opt/gitlab/.ssh nfs4 defaults,vers=4.1,hard,rsize=1048576,wsize=1048576,noatime,nofail,lookupcache=positive 0 2 - 10.1.0.1:/var/opt/gitlab/gitlab-rails/uploads /var/opt/gitlab/gitlab-rails/uploads nfs4 defaults,vers=4.1,hard,rsize=1048576,wsize=1048576,noatime,nofail,lookupcache=positive 0 2 - 10.1.0.1:/var/opt/gitlab/gitlab-rails/shared /var/opt/gitlab/gitlab-rails/shared nfs4 defaults,vers=4.1,hard,rsize=1048576,wsize=1048576,noatime,nofail,lookupcache=positive 0 2 - 10.1.0.1:/var/opt/gitlab/gitlab-ci/builds /var/opt/gitlab/gitlab-ci/builds nfs4 defaults,vers=4.1,hard,rsize=1048576,wsize=1048576,noatime,nofail,lookupcache=positive 0 2 - 10.1.0.1:/var/opt/gitlab/git-data /var/opt/gitlab/git-data nfs4 defaults,vers=4.1,hard,rsize=1048576,wsize=1048576,noatime,nofail,lookupcache=positive 0 2 - ``` - -Note there are several options that you should consider using: - -| Setting | Description | -| ------- | ----------- | -| `vers=4.1` |NFS v4.1 should be used instead of v4.0 because there is a Linux [NFS client bug in v4.0](https://gitlab.com/gitlab-org/gitaly/-/issues/1339) that can cause significant problems due to stale data. -| `nofail` | Don't halt boot process waiting for this mount to become available -| `lookupcache=positive` | Tells the NFS client to honor `positive` cache results but invalidates any `negative` cache results. Negative cache results cause problems with Git. Specifically, a `git push` can fail to register uniformly across all NFS clients. The negative cache causes the clients to 'remember' that the files did not exist previously. -| `hard` | Instead of `soft`. [Further details](#soft-mount-option). - -### soft mount option - -We recommend that you use `hard` in your mount options, unless you have a specific -reason to use `soft`. - -On GitLab.com, we use `soft` because there were times when we had NFS servers -reboot and `soft` improved availability, but everyone's infrastructure is different. -If your NFS is provided by on-premise storage arrays with redundant controllers, -for example, you shouldn't need to worry about NFS server availability. - -The NFS man page states: - -> "soft" timeout can cause silent data corruption in certain cases - -Read the [Linux man page](https://linux.die.net/man/5/nfs) to understand the difference, -and if you do use `soft`, ensure that you've taken steps to mitigate the risks. - -If you experience behavior that might have been caused by -writes to disk on the NFS server not occurring, such as commits going missing, -use the `hard` option, because (from the man page): - -> use the soft option only when client responsiveness is more important than data integrity - -Other vendors make similar recommendations, including -[SAP](http://wiki.scn.sap.com/wiki/x/PARnFQ) and NetApp's -[knowledge base](https://kb.netapp.com/Advice_and_Troubleshooting/Data_Storage_Software/ONTAP_OS/What_are_the_differences_between_hard_mount_and_soft_mount), -they highlight that if the NFS client driver caches data, `soft` means there is no certainty if -writes by GitLab are actually on disk. - -Mount points set with the option `hard` may not perform as well, and if the -NFS server goes down, `hard` will cause processes to hang when interacting with -the mount point. Use `SIGKILL` (`kill -9`) to deal with hung processes. -The `intr` option -[stopped working in the 2.6 kernel](https://access.redhat.com/solutions/157873). - -## A single NFS mount - -It's recommended to nest all GitLab data directories within a mount, that allows automatic -restore of backups without manually moving existing data. - -```plaintext -mountpoint -└── gitlab-data - ├── builds - ├── git-data - ├── shared - └── uploads -``` - -To do so, we'll need to configure Omnibus with the paths to each directory nested -in the mount point as follows: - -Mount `/gitlab-nfs` then use the following Omnibus -configuration to move each data location to a subdirectory: - -```ruby -git_data_dirs({"default" => { "path" => "/gitlab-nfs/gitlab-data/git-data"} }) -gitlab_rails['uploads_directory'] = '/gitlab-nfs/gitlab-data/uploads' -gitlab_rails['shared_path'] = '/gitlab-nfs/gitlab-data/shared' -gitlab_ci['builds_directory'] = '/gitlab-nfs/gitlab-data/builds' -``` - -Run `sudo gitlab-ctl reconfigure` to start using the central location. Please -be aware that if you had existing data you will need to manually copy/rsync it -to these new locations and then restart GitLab. - -## Bind mounts - -Alternatively to changing the configuration in Omnibus, bind mounts can be used -to store the data on an NFS mount. - -Bind mounts provide a way to specify just one NFS mount and then -bind the default GitLab data locations to the NFS mount. Start by defining your -single NFS mount point as you normally would in `/etc/fstab`. Let's assume your -NFS mount point is `/gitlab-nfs`. Then, add the following bind mounts in -`/etc/fstab`: - -```shell -/gitlab-nfs/gitlab-data/git-data /var/opt/gitlab/git-data none bind 0 0 -/gitlab-nfs/gitlab-data/.ssh /var/opt/gitlab/.ssh none bind 0 0 -/gitlab-nfs/gitlab-data/uploads /var/opt/gitlab/gitlab-rails/uploads none bind 0 0 -/gitlab-nfs/gitlab-data/shared /var/opt/gitlab/gitlab-rails/shared none bind 0 0 -/gitlab-nfs/gitlab-data/builds /var/opt/gitlab/gitlab-ci/builds none bind 0 0 -``` - -Using bind mounts will require manually making sure the data directories -are empty before attempting a restore. Read more about the -[restore prerequisites](../../raketasks/backup_restore.md). - -## Multiple NFS mounts - -When using default Omnibus configuration you will need to share 4 data locations -between all GitLab cluster nodes. No other locations should be shared. The -following are the 4 locations need to be shared: - -| Location | Description | Default configuration | -| -------- | ----------- | --------------------- | -| `/var/opt/gitlab/git-data` | Git repository data. This will account for a large portion of your data | `git_data_dirs({"default" => { "path" => "/var/opt/gitlab/git-data"} })` -| `/var/opt/gitlab/gitlab-rails/uploads` | User uploaded attachments | `gitlab_rails['uploads_directory'] = '/var/opt/gitlab/gitlab-rails/uploads'` -| `/var/opt/gitlab/gitlab-rails/shared` | Build artifacts, GitLab Pages, LFS objects, temp files, etc. If you're using LFS this may also account for a large portion of your data | `gitlab_rails['shared_path'] = '/var/opt/gitlab/gitlab-rails/shared'` -| `/var/opt/gitlab/gitlab-ci/builds` | GitLab CI/CD build traces | `gitlab_ci['builds_directory'] = '/var/opt/gitlab/gitlab-ci/builds'` - -Other GitLab directories should not be shared between nodes. They contain -node-specific files and GitLab code that does not need to be shared. To ship -logs to a central location consider using remote syslog. Omnibus GitLab packages -provide configuration for [UDP log shipping](https://docs.gitlab.com/omnibus/settings/logs.html#udp-log-shipping-gitlab-enterprise-edition-only). - -Having multiple NFS mounts will require manually making sure the data directories -are empty before attempting a restore. Read more about the -[restore prerequisites](../../raketasks/backup_restore.md). - ---- - -Read more on high-availability configuration: - -1. [Configure the database](../postgresql/replication_and_failover.md) -1. [Configure Redis](redis.md) -1. [Configure the GitLab application servers](gitlab.md) -1. [Configure the load balancers](load_balancer.md) - -<!-- ## Troubleshooting - -Include any troubleshooting steps that you can foresee. If you know beforehand what issues -one might have when setting this up, or when something is changed, or on upgrading, it's -important to describe those, too. Think of things that may go wrong and include them here. -This is important to minimize requests for support, and to avoid doc comments with -questions that you know someone might ask. - -Each scenario can be a third-level heading, e.g. `### Getting error message X`. -If you have none to add when creating a doc, leave this section in place -but commented out to help encourage others to add to it in the future. --> +This document was moved to [another location](../nfs.md). diff --git a/doc/administration/high_availability/nfs_host_client_setup.md b/doc/administration/high_availability/nfs_host_client_setup.md index 213680e2f64..e3342fa0813 100644 --- a/doc/administration/high_availability/nfs_host_client_setup.md +++ b/doc/administration/high_availability/nfs_host_client_setup.md @@ -1,157 +1,5 @@ --- -type: reference +redirect_to: ../nfs.md --- -# Configuring NFS for GitLab HA - -Setting up NFS for a GitLab HA setup allows all applications nodes in a cluster -to share the same files and maintain data consistency. Application nodes in an HA -setup act as clients while the NFS server plays host. - -> Note: The instructions provided in this documentation allow for setting a quick -proof of concept but will leave NFS as potential single point of failure and -therefore not recommended for use in production. Explore options such as [Pacemaker -and Corosync](https://clusterlabs.org) for highly available NFS in production. - -Below are instructions for setting up an application node(client) in an HA cluster -to read from and write to a central NFS server(host). - -NOTE: **Note:** -Using EFS may negatively impact performance. Please review the [relevant documentation](nfs.md#avoid-using-awss-elastic-file-system-efs) for additional details. - -## NFS Server Setup - -> Follow the instructions below to set up and configure your NFS server. - -### Step 1 - Install NFS Server on Host - -Installing the `nfs-kernel-server` package allows you to share directories with the clients running the GitLab application. - -```shell -apt-get update -apt-get install nfs-kernel-server -``` - -### Step 2 - Export Host's Home Directory to Client - -In this setup we will share the home directory on the host with the client. Edit the exports file as below to share the host's home directory with the client. If you have multiple clients running GitLab you must enter the client IP addresses in line in the `/etc/exports` file. - -```plaintext -#/etc/exports for one client -/home <client_ip_address>(rw,sync,no_root_squash,no_subtree_check) - -#/etc/exports for three clients -/home <client_ip_address>(rw,sync,no_root_squash,no_subtree_check) <client_2_ip_address>(rw,sync,no_root_squash,no_subtree_check) <client_3_ip_address>(rw,sync,no_root_squash,no_subtree_check) -``` - -Restart the NFS server after making changes to the `exports` file for the changes -to take effect. - -```shell -systemctl restart nfs-kernel-server -``` - -NOTE: **Note:** -You may need to update your server's firewall. See the [firewall section](#nfs-in-a-firewalled-environment) at the end of this guide. - -## Client / GitLab application node Setup - -> Follow the instructions below to connect any GitLab Rails application node running -inside your HA environment to the NFS server configured above. - -### Step 1 - Install NFS Common on Client - -The `nfs-common` provides NFS functionality without installing server components which -we don't need running on the application nodes. - -```shell -apt-get update -apt-get install nfs-common -``` - -### Step 2 - Create Mount Points on Client - -Create a directory on the client that we can mount the shared directory from the host. -Please note that if your mount point directory contains any files they will be hidden -once the remote shares are mounted. An empty/new directory on the client is recommended -for this purpose. - -```shell -mkdir -p /nfs/home -``` - -Confirm that the mount point works by mounting it on the client and checking that -it is mounted with the command below: - -```shell -mount <host_ip_address>:/home -df -h -``` - -### Step 3 - Set up Automatic Mounts on Boot - -Edit `/etc/fstab` on the client as below to mount the remote shares automatically at boot. -Note that GitLab requires advisory file locking, which is only supported natively in -NFS version 4. NFSv3 also supports locking as long as Linux Kernel 2.6.5+ is used. -We recommend using version 4 and do not specifically test NFSv3. -See [NFS documentation](nfs.md#nfs-client-mount-options) for guidance on mount options. - -```plaintext -#/etc/fstab -<host_ip_address>:/home /nfs/home nfs4 defaults,hard,vers=4.1,rsize=1048576,wsize=1048576,noatime,nofail,lookupcache=positive 0 2 -``` - -Reboot the client and confirm that the mount point is mounted automatically. - -NOTE: **Note:** -If you followed our guide to [GitLab Pages on a separate server](../pages/index.md#running-gitlab-pages-on-a-separate-server) -here, please continue there with the pages-specific NFS mounts. -The step below is for broader use-cases than only sharing pages data. - -### Step 4 - Set up GitLab to Use NFS mounts - -When using the default Omnibus configuration you will need to share 4 data locations -between all GitLab cluster nodes. No other locations should be shared. Changing the -default file locations in `gitlab.rb` on the client allows you to have one main mount -point and have all the required locations as subdirectories to use the NFS mount for -`git-data`. - -```plaintext -git_data_dirs({"default" => {"path" => "/nfs/home/var/opt/gitlab-data/git-data"}}) -gitlab_rails['uploads_directory'] = '/nfs/home/var/opt/gitlab-data/uploads' -gitlab_rails['shared_path'] = '/nfs/home/var/opt/gitlab-data/shared' -gitlab_ci['builds_directory'] = '/nfs/home/var/opt/gitlab-data/builds' -``` - -Save the changes in `gitlab.rb` and run `gitlab-ctl reconfigure`. - -## NFS in a Firewalled Environment - -If the traffic between your NFS server and NFS client(s) is subject to port filtering -by a firewall, then you will need to reconfigure that firewall to allow NFS communication. - -[This guide from TDLP](http://tldp.org/HOWTO/NFS-HOWTO/security.html#FIREWALLS) -covers the basics of using NFS in a firewalled environment. Additionally, we encourage you to -search for and review the specific documentation for your operating system or distribution and your firewall software. - -Example for Ubuntu: - -Check that NFS traffic from the client is allowed by the firewall on the host by running -the command: `sudo ufw status`. If it's being blocked, then you can allow traffic from a specific -client with the command below. - -```shell -sudo ufw allow from <client_ip_address> to any port nfs -``` - -<!-- ## Troubleshooting - -Include any troubleshooting steps that you can foresee. If you know beforehand what issues -one might have when setting this up, or when something is changed, or on upgrading, it's -important to describe those, too. Think of things that may go wrong and include them here. -This is important to minimize requests for support, and to avoid doc comments with -questions that you know someone might ask. - -Each scenario can be a third-level heading, e.g. `### Getting error message X`. -If you have none to add when creating a doc, leave this section in place -but commented out to help encourage others to add to it in the future. --> +This document was moved to [another location](../nfs.md). diff --git a/doc/administration/high_availability/pgbouncer.md b/doc/administration/high_availability/pgbouncer.md index 15e4da5b1f7..44f4aa37651 100644 --- a/doc/administration/high_availability/pgbouncer.md +++ b/doc/administration/high_availability/pgbouncer.md @@ -1,166 +1,5 @@ --- -type: reference +redirect_to: ../postgresql/pgbouncer.md --- -# Working with the bundled PgBouncer service **(PREMIUM ONLY)** - -As part of its High Availability stack, GitLab Premium includes a bundled version of [PgBouncer](http://www.pgbouncer.org/) that can be managed through `/etc/gitlab/gitlab.rb`. PgBouncer is used to seamlessly migrate database connections between servers in a failover scenario. Additionally, it can be used in a non-HA setup to pool connections, speeding up response time while reducing resource usage. - -In a HA setup, it's recommended to run a PgBouncer node separately for each database node with an internal load balancer (TCP) serving each accordingly. - -## Operations - -### Running PgBouncer as part of an HA GitLab installation - -This content has been moved to a [new location](../postgresql/replication_and_failover.md#configuring-the-pgbouncer-node). - -### Running PgBouncer as part of a non-HA GitLab installation - -1. Generate PGBOUNCER_USER_PASSWORD_HASH with the command `gitlab-ctl pg-password-md5 pgbouncer` - -1. Generate SQL_USER_PASSWORD_HASH with the command `gitlab-ctl pg-password-md5 gitlab`. We'll also need to enter the plaintext SQL_USER_PASSWORD later - -1. On your database node, ensure the following is set in your `/etc/gitlab/gitlab.rb` - - ```ruby - postgresql['pgbouncer_user_password'] = 'PGBOUNCER_USER_PASSWORD_HASH' - postgresql['sql_user_password'] = 'SQL_USER_PASSWORD_HASH' - postgresql['listen_address'] = 'XX.XX.XX.Y' # Where XX.XX.XX.Y is the ip address on the node postgresql should listen on - postgresql['md5_auth_cidr_addresses'] = %w(AA.AA.AA.B/32) # Where AA.AA.AA.B is the IP address of the pgbouncer node - ``` - -1. Run `gitlab-ctl reconfigure` - - **Note:** If the database was already running, it will need to be restarted after reconfigure by running `gitlab-ctl restart postgresql`. - -1. On the node you are running PgBouncer on, make sure the following is set in `/etc/gitlab/gitlab.rb` - - ```ruby - pgbouncer['enable'] = true - pgbouncer['databases'] = { - gitlabhq_production: { - host: 'DATABASE_HOST', - user: 'pgbouncer', - password: 'PGBOUNCER_USER_PASSWORD_HASH' - } - } - ``` - -1. Run `gitlab-ctl reconfigure` - -1. On the node running Puma, make sure the following is set in `/etc/gitlab/gitlab.rb` - - ```ruby - gitlab_rails['db_host'] = 'PGBOUNCER_HOST' - gitlab_rails['db_port'] = '6432' - gitlab_rails['db_password'] = 'SQL_USER_PASSWORD' - ``` - -1. Run `gitlab-ctl reconfigure` - -1. At this point, your instance should connect to the database through PgBouncer. If you are having issues, see the [Troubleshooting](#troubleshooting) section - -## Enable Monitoring - -> [Introduced](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/3786) in GitLab 12.0. - -If you enable Monitoring, it must be enabled on **all** PgBouncer servers. - -1. Create/edit `/etc/gitlab/gitlab.rb` and add the following configuration: - - ```ruby - # Enable service discovery for Prometheus - consul['enable'] = true - consul['monitoring_service_discovery'] = true - - # Replace placeholders - # Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z - # with the addresses of the Consul server nodes - consul['configuration'] = { - retry_join: %w(Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z), - } - - # Set the network addresses that the exporters will listen on - node_exporter['listen_address'] = '0.0.0.0:9100' - pgbouncer_exporter['listen_address'] = '0.0.0.0:9188' - ``` - -1. Run `sudo gitlab-ctl reconfigure` to compile the configuration. - -### Interacting with PgBouncer - -#### Administrative console - -As part of Omnibus GitLab, we provide a command `gitlab-ctl pgb-console` to automatically connect to the PgBouncer administrative console. Please see the [PgBouncer documentation](https://www.pgbouncer.org/usage.html#admin-console) for detailed instructions on how to interact with the console. - -To start a session, run - -```shell -# gitlab-ctl pgb-console -Password for user pgbouncer: -psql (11.7, server 1.7.2/bouncer) -Type "help" for help. - -pgbouncer=# -``` - -The password you will be prompted for is the PGBOUNCER_USER_PASSWORD - -To get some basic information about the instance, run - -```shell -pgbouncer=# show databases; show clients; show servers; - name | host | port | database | force_user | pool_size | reserve_pool | pool_mode | max_connections | current_connections ----------------------+-----------+------+---------------------+------------+-----------+--------------+-----------+-----------------+--------------------- - gitlabhq_production | 127.0.0.1 | 5432 | gitlabhq_production | | 100 | 5 | | 0 | 1 - pgbouncer | | 6432 | pgbouncer | pgbouncer | 2 | 0 | statement | 0 | 0 -(2 rows) - - type | user | database | state | addr | port | local_addr | local_port | connect_time | request_time | ptr | link -| remote_pid | tls -------+-----------+---------------------+--------+-----------+-------+------------+------------+---------------------+---------------------+-----------+------ -+------------+----- - C | gitlab | gitlabhq_production | active | 127.0.0.1 | 44590 | 127.0.0.1 | 6432 | 2018-04-24 22:13:10 | 2018-04-24 22:17:10 | 0x12444c0 | -| 0 | - C | gitlab | gitlabhq_production | active | 127.0.0.1 | 44592 | 127.0.0.1 | 6432 | 2018-04-24 22:13:10 | 2018-04-24 22:17:10 | 0x12447c0 | -| 0 | - C | gitlab | gitlabhq_production | active | 127.0.0.1 | 44594 | 127.0.0.1 | 6432 | 2018-04-24 22:13:10 | 2018-04-24 22:17:10 | 0x1244940 | -| 0 | - C | gitlab | gitlabhq_production | active | 127.0.0.1 | 44706 | 127.0.0.1 | 6432 | 2018-04-24 22:14:22 | 2018-04-24 22:16:31 | 0x1244ac0 | -| 0 | - C | gitlab | gitlabhq_production | active | 127.0.0.1 | 44708 | 127.0.0.1 | 6432 | 2018-04-24 22:14:22 | 2018-04-24 22:15:15 | 0x1244c40 | -| 0 | - C | gitlab | gitlabhq_production | active | 127.0.0.1 | 44794 | 127.0.0.1 | 6432 | 2018-04-24 22:15:15 | 2018-04-24 22:15:15 | 0x1244dc0 | -| 0 | - C | gitlab | gitlabhq_production | active | 127.0.0.1 | 44798 | 127.0.0.1 | 6432 | 2018-04-24 22:15:15 | 2018-04-24 22:16:31 | 0x1244f40 | -| 0 | - C | pgbouncer | pgbouncer | active | 127.0.0.1 | 44660 | 127.0.0.1 | 6432 | 2018-04-24 22:13:51 | 2018-04-24 22:17:12 | 0x1244640 | -| 0 | -(8 rows) - - type | user | database | state | addr | port | local_addr | local_port | connect_time | request_time | ptr | link | rem -ote_pid | tls -------+--------+---------------------+-------+-----------+------+------------+------------+---------------------+---------------------+-----------+------+---- ---------+----- - S | gitlab | gitlabhq_production | idle | 127.0.0.1 | 5432 | 127.0.0.1 | 35646 | 2018-04-24 22:15:15 | 2018-04-24 22:17:10 | 0x124dca0 | | - 19980 | -(1 row) -``` - -## Troubleshooting - -In case you are experiencing any issues connecting through PgBouncer, the first place to check is always the logs: - -```shell -# gitlab-ctl tail pgbouncer -``` - -Additionally, you can check the output from `show databases` in the [Administrative console](#administrative-console). In the output, you would expect to see values in the `host` field for the `gitlabhq_production` database. Additionally, `current_connections` should be greater than 1. - -### Message: `LOG: invalid CIDR mask in address` - -See the suggested fix [in Geo documentation](../geo/replication/troubleshooting.md#message-log--invalid-cidr-mask-in-address). - -### Message: `LOG: invalid IP mask "md5": Name or service not known` - -See the suggested fix [in Geo documentation](../geo/replication/troubleshooting.md#message-log--invalid-ip-mask-md5-name-or-service-not-known). +This document was moved to [another location](../postgresql/pgbouncer.md). diff --git a/doc/administration/high_availability/sidekiq.md b/doc/administration/high_availability/sidekiq.md index 98a9af64e5e..ac92ae2eaaa 100644 --- a/doc/administration/high_availability/sidekiq.md +++ b/doc/administration/high_availability/sidekiq.md @@ -1,190 +1,5 @@ --- -type: reference +redirect_to: ../sidekiq.md --- -# Configuring Sidekiq - -This section discusses how to configure an external Sidekiq instance. - -Sidekiq requires connection to the Redis, PostgreSQL and Gitaly instance. -To configure the Sidekiq node: - -1. SSH into the Sidekiq server. - -1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab package -you want using steps 1 and 2 from the GitLab downloads page. -**Do not complete any other steps on the download page.** - -1. Open `/etc/gitlab/gitlab.rb` with your editor. - -1. Generate the Sidekiq configuration: - - ```ruby - sidekiq['listen_address'] = "10.10.1.48" - - ## Optional: Enable extra Sidekiq processes - sidekiq_cluster['enable'] = true - sidekiq_cluster['enable'] = true - "elastic_indexer" - ] - ``` - -1. Setup Sidekiq's connection to Redis: - - ```ruby - ## Must be the same in every sentinel node - redis['master_name'] = 'gitlab-redis' - - ## The same password for Redis authentication you set up for the master node. - redis['master_password'] = 'YOUR_PASSOWORD' - - ## A list of sentinels with `host` and `port` - gitlab_rails['redis_sentinels'] = [ - {'host' => '10.10.1.34', 'port' => 26379}, - {'host' => '10.10.1.35', 'port' => 26379}, - {'host' => '10.10.1.36', 'port' => 26379}, - ] - ``` - -1. Setup Sidekiq's connection to Gitaly: - - ```ruby - git_data_dirs({ - 'default' => { 'gitaly_address' => 'tcp://gitaly:8075' }, - }) - gitlab_rails['gitaly_token'] = 'YOUR_TOKEN' - ``` - -1. Setup Sidekiq's connection to PostgreSQL: - - ```ruby - gitlab_rails['db_host'] = '10.10.1.30' - gitlab_rails['db_password'] = 'YOUR_PASSOWORD' - gitlab_rails['db_port'] = '5432' - gitlab_rails['db_adapter'] = 'postgresql' - gitlab_rails['db_encoding'] = 'unicode' - gitlab_rails['auto_migrate'] = false - ``` - - Remember to add the Sidekiq nodes to the PostgreSQL whitelist: - - ```ruby - postgresql['trust_auth_cidr_addresses'] = %w(127.0.0.1/32 10.10.1.30/32 10.10.1.31/32 10.10.1.32/32 10.10.1.33/32 10.10.1.38/32) - ``` - -1. Disable other services: - - ```ruby - nginx['enable'] = false - grafana['enable'] = false - prometheus['enable'] = false - gitlab_rails['auto_migrate'] = false - alertmanager['enable'] = false - gitaly['enable'] = false - gitlab_monitor['enable'] = false - gitlab_workhorse['enable'] = false - nginx['enable'] = false - postgres_exporter['enable'] = false - postgresql['enable'] = false - redis['enable'] = false - redis_exporter['enable'] = false - puma['enable'] = false - gitlab_exporter['enable'] = false - ``` - -1. Run `gitlab-ctl reconfigure`. - -NOTE: **Note:** -You will need to restart the Sidekiq nodes after an update has occurred and database -migrations performed. - -## Example configuration - -Here's what the ending `/etc/gitlab/gitlab.rb` would look like: - -```ruby -######################################## -##### Services Disabled ### -######################################## - -nginx['enable'] = false -grafana['enable'] = false -prometheus['enable'] = false -gitlab_rails['auto_migrate'] = false -alertmanager['enable'] = false -gitaly['enable'] = false -gitlab_monitor['enable'] = false -gitlab_workhorse['enable'] = false -nginx['enable'] = false -postgres_exporter['enable'] = false -postgresql['enable'] = false -redis['enable'] = false -redis_exporter['enable'] = false -puma['enable'] = false -gitlab_exporter['enable'] = false - -######################################## -#### Redis ### -######################################## - -## Must be the same in every sentinel node -redis['master_name'] = 'gitlab-redis' - -## The same password for Redis authentication you set up for the master node. -redis['master_password'] = 'YOUR_PASSOWORD' - -## A list of sentinels with `host` and `port` -gitlab_rails['redis_sentinels'] = [ - {'host' => '10.10.1.34', 'port' => 26379}, - {'host' => '10.10.1.35', 'port' => 26379}, - {'host' => '10.10.1.36', 'port' => 26379}, - ] - -####################################### -### Gitaly ### -####################################### - -git_data_dirs({ - 'default' => { 'gitaly_address' => 'tcp://gitaly:8075' }, -}) -gitlab_rails['gitaly_token'] = 'YOUR_TOKEN' - -####################################### -### Postgres ### -####################################### -gitlab_rails['db_host'] = '10.10.1.30' -gitlab_rails['db_password'] = 'YOUR_PASSOWORD' -gitlab_rails['db_port'] = '5432' -gitlab_rails['db_adapter'] = 'postgresql' -gitlab_rails['db_encoding'] = 'unicode' -gitlab_rails['auto_migrate'] = false - -####################################### -### Sidekiq configuration ### -####################################### -sidekiq['listen_address'] = "10.10.1.48" - -####################################### -### Monitoring configuration ### -####################################### -consul['enable'] = true -consul['monitoring_service_discovery'] = true - -consul['configuration'] = { - bind_addr: '10.10.1.48', - retry_join: %w(10.10.1.34 10.10.1.35 10.10.1.36) -} - -# Set the network addresses that the exporters will listen on -node_exporter['listen_address'] = '10.10.1.48:9100' - -# Rails Status for prometheus -gitlab_rails['monitoring_whitelist'] = ['10.10.1.42', '127.0.0.1'] -``` - -## Further reading - -Related Sidekiq configuration: - -1. [Extra Sidekiq processes](../operations/extra_sidekiq_processes.md) -1. [Using the GitLab-Sidekiq chart](https://docs.gitlab.com/charts/charts/gitlab/sidekiq/) +This document was moved to [another location](../sidekiq.md). diff --git a/doc/administration/img/repository_storages_admin_ui_v13_1.png b/doc/administration/img/repository_storages_admin_ui_v13_1.png Binary files differindex f8c13a35369..a2b88d14a36 100644 --- a/doc/administration/img/repository_storages_admin_ui_v13_1.png +++ b/doc/administration/img/repository_storages_admin_ui_v13_1.png diff --git a/doc/administration/index.md b/doc/administration/index.md index fa415e5f78d..ed079abf708 100644 --- a/doc/administration/index.md +++ b/doc/administration/index.md @@ -138,7 +138,7 @@ Learn how to install, configure, update, and maintain your GitLab instance. ## Package Registry administration - [Container Registry](packages/container_registry.md): Configure Container Registry with GitLab. -- [Package Registry](packages/index.md): Enable GitLab to act as an NPM Registry and a Maven Repository. **(PREMIUM ONLY)** +- [Package Registry](packages/index.md): Enable GitLab to act as an NPM Registry and a Maven Repository. - [Dependency Proxy](packages/dependency_proxy.md): Configure the Dependency Proxy, a local proxy for frequently used upstream images/packages. **(PREMIUM ONLY)** ### Repository settings @@ -163,7 +163,11 @@ Learn how to install, configure, update, and maintain your GitLab instance. ## Snippet settings -- [Setting snippet content size limit](snippets/index.md): Set a maximum size limit for snippets' content. +- [Setting snippet content size limit](snippets/index.md): Set a maximum content size limit for snippets. + +## Wiki settings + +- [Setting wiki page content size limit](wikis/index.md): Set a maximum content size limit for wiki pages. ## Git configuration options diff --git a/doc/administration/instance_limits.md b/doc/administration/instance_limits.md index 6d2fbd95d5e..f30dba331b8 100644 --- a/doc/administration/instance_limits.md +++ b/doc/administration/instance_limits.md @@ -160,7 +160,7 @@ There is a limit when embedding metrics in GFM for performance reasons. ## Number of webhooks -On GitLab.com, the [maximum number of webhooks](../user/gitlab_com/index.md#maximum-number-of-webhooks) per project, and per group, is limited. +On GitLab.com, the [maximum number of webhooks and their size](../user/gitlab_com/index.md#webhooks) per project, and per group, is limited. To set this limit on a self-managed installation, run the following in the [GitLab Rails console](troubleshooting/debug.md#starting-a-rails-console-session): @@ -314,6 +314,58 @@ To update this limit to a new value on a self-managed installation, run the foll Plan.default.actual_limits.update!(ci_instance_level_variables: 30) ``` +### Maximum file size per type of artifact + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/37226) in GitLab 13.3. + +Job artifacts defined with [`artifacts:reports`](../ci/pipelines/job_artifacts.md#artifactsreports) +that are uploaded by the Runner are rejected if the file size exceeds the maximum +file size limit. The limit is determined by comparing the project's +[maximum artifact size setting](../user/admin_area/settings/continuous_integration.md#maximum-artifacts-size-core-only) +with the instance limit for the given artifact type, and choosing the smaller value. + +Limits are set in megabytes, so the smallest possible value that can be defined is `1 MB`. + +Each type of artifact has a size limit that can be set. A default of `0` means there +is no limit for that specific artifact type, and the project's maximum artifact size +setting is used: + +| Artifact limit name | Default value | +|---------------------------------------------|---------------| +| `ci_max_artifact_size_accessibility` | 0 | +| `ci_max_artifact_size_archive` | 0 | +| `ci_max_artifact_size_browser_performance` | 0 | +| `ci_max_artifact_size_cluster_applications` | 0 | +| `ci_max_artifact_size_cobertura` | 0 | +| `ci_max_artifact_size_codequality` | 0 | +| `ci_max_artifact_size_container_scanning` | 0 | +| `ci_max_artifact_size_coverage_fuzzing` | 0 | +| `ci_max_artifact_size_dast` | 0 | +| `ci_max_artifact_size_dependency_scanning` | 0 | +| `ci_max_artifact_size_dotenv` | 0 | +| `ci_max_artifact_size_junit` | 0 | +| `ci_max_artifact_size_license_management` | 0 | +| `ci_max_artifact_size_license_scanning` | 0 | +| `ci_max_artifact_size_load_performance` | 0 | +| `ci_max_artifact_size_lsif` | 20 MB ([introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/37226) in GitLab 13.3) | +| `ci_max_artifact_size_metadata` | 0 | +| `ci_max_artifact_size_metrics_referee` | 0 | +| `ci_max_artifact_size_metrics` | 0 | +| `ci_max_artifact_size_network_referee` | 0 | +| `ci_max_artifact_size_performance` | 0 | +| `ci_max_artifact_size_requirements` | 0 | +| `ci_max_artifact_size_sast` | 0 | +| `ci_max_artifact_size_secret_detection` | 0 | +| `ci_max_artifact_size_terraform` | 5 MB ([introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/37018) in GitLab 13.3) | +| `ci_max_artifact_size_trace` | 0 | + +For example, to set the `ci_max_artifact_size_junit` limit to 10MB on a self-managed +installation, run the following in the [GitLab Rails console](troubleshooting/debug.md#starting-a-rails-console-session): + +```ruby +Plan.default.actual_limits.update!(ci_max_artifact_size_junit: 10) +``` + ## Instance monitoring and metrics ### Incident Management inbound alert limits @@ -388,6 +440,24 @@ Reports that go over the 20 MB limit won't be loaded. Affected reports: ## Advanced Global Search limits +### Maximum file size indexed + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/8638) in GitLab 13.3. + +You can set a limit on the content of repository files that are indexed in +Elasticsearch. Any files larger than this limit will not be indexed, and thus +will not be searchable. + +Setting a limit helps reduce the memory usage of the indexing processes as well +as the overall index size. This value defaults to `1024 KiB` (1 MiB) as any +text files larger than this likely aren't meant to be read by humans. + +NOTE: **Note:** +You must set a limit, as an unlimited file size is not supported. Setting this +value to be greater than the amount of memory on GitLab's Sidekiq nodes will +lead to GitLab's Sidekiq nodes running out of memory as they will pre-allocate +this amount of memory during indexing. + ### Maximum field length > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/201826) in GitLab 12.8. @@ -396,6 +466,9 @@ You can set a limit on the content of text fields indexed for Global Search. Setting a maximum helps to reduce the load of the indexing processes. If any text field exceeds this limit then the text will be truncated to this number of characters and the rest will not be indexed and hence will not be searchable. +This is applicable to all indexed data except repository files that get +indexed, which have a separate limit (see [Maximum file size +indexed](#maximum-file-size-indexed)). - On GitLab.com this is limited to 20000 characters - For self-managed installations it is unlimited by default @@ -408,6 +481,7 @@ Set the limit to `0` to disable it. ## Wiki limits +- [Wiki page content size limit](wikis/index.md#wiki-page-content-size-limit). - [Length restrictions for file and directory names](../user/project/wiki/index.md#length-restrictions-for-file-and-directory-names). ## Snippets limits diff --git a/doc/administration/integration/plantuml.md b/doc/administration/integration/plantuml.md index 2a30eced7c4..49ea59d239c 100644 --- a/doc/administration/integration/plantuml.md +++ b/doc/administration/integration/plantuml.md @@ -1,3 +1,10 @@ +--- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference, howto +--- + # PlantUML & GitLab > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/8537) in GitLab 8.16. diff --git a/doc/administration/integration/terminal.md b/doc/administration/integration/terminal.md index fd9e09dc17a..c363bd30543 100644 --- a/doc/administration/integration/terminal.md +++ b/doc/administration/integration/terminal.md @@ -57,7 +57,7 @@ through to the next one in the chain. If you installed GitLab using Omnibus, or from source, starting with GitLab 8.15, this should be done by the default configuration, so there's no need for you to do anything. -However, if you run a [load balancer](../high_availability/load_balancer.md) in +However, if you run a [load balancer](../load_balancer.md) in front of GitLab, you may need to make some changes to your configuration. These guides document the necessary steps for a selection of popular reverse proxies: @@ -98,4 +98,4 @@ they will receive a `Connection failed` message. > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/8413) in GitLab 8.17. Terminal sessions, by default, do not expire. -You can limit terminal session lifetime in your GitLab instance. To do so, navigate to **{admin}** [**Admin Area > Settings > Web terminal**](../../user/admin_area/settings/index.md#general), and set a `max session time`. +You can limit terminal session lifetime in your GitLab instance. To do so, navigate to [**Admin Area > Settings > Web terminal**](../../user/admin_area/settings/index.md#general), and set a `max session time`. diff --git a/doc/administration/invalidate_markdown_cache.md b/doc/administration/invalidate_markdown_cache.md index 5fd804e11dc..ac43d0eae63 100644 --- a/doc/administration/invalidate_markdown_cache.md +++ b/doc/administration/invalidate_markdown_cache.md @@ -1,3 +1,10 @@ +--- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference +--- + # Invalidate Markdown Cache For performance reasons, GitLab caches the HTML version of Markdown text diff --git a/doc/administration/issue_closing_pattern.md b/doc/administration/issue_closing_pattern.md index 579b957eb47..7abe0f725f2 100644 --- a/doc/administration/issue_closing_pattern.md +++ b/doc/administration/issue_closing_pattern.md @@ -1,6 +1,13 @@ +--- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference +--- + # Issue closing pattern **(CORE ONLY)** ->**Note:** +NOTE: **Note:** This is the administration documentation. There is a separate [user documentation](../user/project/issues/managing_issues.md#closing-issues-automatically) on issue closing pattern. @@ -16,7 +23,7 @@ is installed on. The default pattern can be located in [`gitlab.yml.example`](https://gitlab.com/gitlab-org/gitlab/blob/master/config/gitlab.yml.example) under the "Automatic issue closing" section. -> **Tip:** +TIP: **Tip:** You are advised to use <https://rubular.com> to test the issue closing pattern. Because Rubular doesn't understand `%{issue_ref}`, you can replace this by `#\d+` when testing your patterns, which matches only local issue references like `#123`. diff --git a/doc/administration/job_artifacts.md b/doc/administration/job_artifacts.md index cb31a8934b1..66a7bcb90f6 100644 --- a/doc/administration/job_artifacts.md +++ b/doc/administration/job_artifacts.md @@ -164,9 +164,30 @@ _The artifacts are stored by default in gitlab-rake gitlab:artifacts:migrate ``` +1. Optional: Verify all files migrated properly. + From [PostgreSQL console](https://docs.gitlab.com/omnibus/settings/database.html#connecting-to-the-bundled-postgresql-database) + (`sudo gitlab-psql -d gitlabhq_production`) verify `objectstg` below (where `file_store=2`) has count of all artifacts: + + ```shell + gitlabhq_production=# SELECT count(*) AS total, sum(case when file_store = '1' then 1 else 0 end) AS filesystem, sum(case when file_store = '2' then 1 else 0 end) AS objectstg FROM ci_job_artifacts; + + total | filesystem | objectstg + ------+------------+----------- + 2409 | 0 | 2409 + ``` + + Verify no files on disk in `artifacts` folder: + + ```shell + sudo find /var/opt/gitlab/gitlab-rails/shared/artifacts -type f | grep -v tmp/cache | wc -l + ``` + + In some cases, you may need to run the [orphan artifact file cleanup Rake task](../raketasks/cleanup.md#remove-orphan-artifact-files) + to clean up orphaned artifacts. + CAUTION: **Caution:** JUnit test report artifact (`junit.xml.gz`) migration -[is not supported](https://gitlab.com/gitlab-org/gitlab/-/issues/27698) +[was not supported until GitLab 12.8](https://gitlab.com/gitlab-org/gitlab/-/issues/27698#note_317190991) by the `gitlab:artifacts:migrate` script. **In installations from source:** @@ -197,9 +218,29 @@ _The artifacts are stored by default in sudo -u git -H bundle exec rake gitlab:artifacts:migrate RAILS_ENV=production ``` +1. Optional: Verify all files migrated properly. + From PostgreSQL console (`sudo -u git -H psql -d gitlabhq_production`) verify `objectstg` below (where `file_store=2`) has count of all artifacts: + + ```shell + gitlabhq_production=# SELECT count(*) AS total, sum(case when file_store = '1' then 1 else 0 end) AS filesystem, sum(case when file_store = '2' then 1 else 0 end) AS objectstg FROM ci_job_artifacts; + + total | filesystem | objectstg + ------+------------+----------- + 2409 | 0 | 2409 + ``` + + Verify no files on disk in `artifacts` folder: + + ```shell + sudo find /var/opt/gitlab/gitlab-rails/shared/artifacts -type f | grep -v tmp/cache | wc -l + ``` + + In some cases, you may need to run the [orphan artifact file cleanup Rake task](../raketasks/cleanup.md#remove-orphan-artifact-files) + to clean up orphaned artifacts. + CAUTION: **Caution:** JUnit test report artifact (`junit.xml.gz`) migration -[is not supported](https://gitlab.com/gitlab-org/gitlab/-/issues/27698) +[was not supported until GitLab 12.8](https://gitlab.com/gitlab-org/gitlab/-/issues/27698#note_317190991) by the `gitlab:artifacts:migrate` script. ### OpenStack example diff --git a/doc/administration/lfs/index.md b/doc/administration/lfs/index.md index 4a8151bd091..5c1a9519a35 100644 --- a/doc/administration/lfs/index.md +++ b/doc/administration/lfs/index.md @@ -1,4 +1,8 @@ --- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference, howto disqus_identifier: 'https://docs.gitlab.com/ee/workflow/lfs/lfs_administration.html' --- @@ -152,7 +156,24 @@ On Omnibus installations, the settings are prefixed by `lfs_object_store_`: This will migrate existing LFS objects to object storage. New LFS objects will be forwarded to object storage unless - `gitlab_rails['lfs_object_store_background_upload']` is set to false. + `gitlab_rails['lfs_object_store_background_upload']` and `gitlab_rails['lfs_object_store_direct_upload']` is set to `false`. +1. Optional: Verify all files migrated properly. + From [PostgreSQL console](https://docs.gitlab.com/omnibus/settings/database.html#connecting-to-the-bundled-postgresql-database) + (`sudo gitlab-psql -d gitlabhq_production`) verify `objectstg` below (where `file_store=2`) has count of all artifacts: + + ```shell + gitlabhq_production=# SELECT count(*) AS total, sum(case when file_store = '1' then 1 else 0 end) AS filesystem, sum(case when file_store = '2' then 1 else 0 end) AS objectstg FROM lfs_objects; + + total | filesystem | objectstg + ------+------------+----------- + 2409 | 0 | 2409 + ``` + + Verify no files on disk in `artifacts` folder: + + ```shell + sudo find /var/opt/gitlab/gitlab-rails/shared/lfs-objects -type f | grep -v tmp/cache | wc -l + ``` ### S3 for installations from source @@ -187,14 +208,30 @@ For source installations the settings are nested under `lfs:` and then ``` This will migrate existing LFS objects to object storage. New LFS objects - will be forwarded to object storage unless `background_upload` is set to - false. + will be forwarded to object storage unless `background_upload` and `direct_upload` is set to + `false`. +1. Optional: Verify all files migrated properly. + From PostgreSQL console (`sudo -u git -H psql -d gitlabhq_production`) verify `objectstg` below (where `file_store=2`) has count of all artifacts: + + ```shell + gitlabhq_production=# SELECT count(*) AS total, sum(case when file_store = '1' then 1 else 0 end) AS filesystem, sum(case when file_store = '2' then 1 else 0 end) AS objectstg FROM lfs_objects; + + total | filesystem | objectstg + ------+------------+----------- + 2409 | 0 | 2409 + ``` + + Verify no files on disk in `artifacts` folder: + + ```shell + sudo find /var/opt/gitlab/gitlab-rails/shared/lfs-objects -type f | grep -v tmp/cache | wc -l + ``` ### Migrating back to local storage In order to migrate back to local storage: -1. Set both `direct_upload` and `background_upload` to false under the LFS object storage settings. Don't forget to restart GitLab. +1. Set both `direct_upload` and `background_upload` to `false` under the LFS object storage settings. Don't forget to restart GitLab. 1. Run `rake gitlab:lfs:migrate_to_local` on your console. 1. Disable `object_storage` for LFS objects in `gitlab.rb`. Remember to restart GitLab afterwards. diff --git a/doc/administration/load_balancer.md b/doc/administration/load_balancer.md new file mode 100644 index 00000000000..fe534f30f66 --- /dev/null +++ b/doc/administration/load_balancer.md @@ -0,0 +1,126 @@ +--- +type: reference +--- + +# Load Balancer for multi-node GitLab + +In an multi-node GitLab configuration, you will need a load balancer to route +traffic to the application servers. The specifics on which load balancer to use +or the exact configuration is beyond the scope of GitLab documentation. We hope +that if you're managing HA systems like GitLab you have a load balancer of +choice already. Some examples including HAProxy (open-source), F5 Big-IP LTM, +and Citrix Net Scaler. This documentation will outline what ports and protocols +you need to use with GitLab. + +## SSL + +How will you handle SSL in your multi-node environment? There are several different +options: + +- Each application node terminates SSL +- The load balancer(s) terminate SSL and communication is not secure between + the load balancer(s) and the application nodes +- The load balancer(s) terminate SSL and communication is *secure* between the + load balancer(s) and the application nodes + +### Application nodes terminate SSL + +Configure your load balancer(s) to pass connections on port 443 as 'TCP' rather +than 'HTTP(S)' protocol. This will pass the connection to the application nodes +NGINX service untouched. NGINX will have the SSL certificate and listen on port 443. + +See [NGINX HTTPS documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https) +for details on managing SSL certificates and configuring NGINX. + +### Load Balancer(s) terminate SSL without backend SSL + +Configure your load balancer(s) to use the 'HTTP(S)' protocol rather than 'TCP'. +The load balancer(s) will then be responsible for managing SSL certificates and +terminating SSL. + +Since communication between the load balancer(s) and GitLab will not be secure, +there is some additional configuration needed. See +[NGINX Proxied SSL documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#supporting-proxied-ssl) +for details. + +### Load Balancer(s) terminate SSL with backend SSL + +Configure your load balancer(s) to use the 'HTTP(S)' protocol rather than 'TCP'. +The load balancer(s) will be responsible for managing SSL certificates that +end users will see. + +Traffic will also be secure between the load balancer(s) and NGINX in this +scenario. There is no need to add configuration for proxied SSL since the +connection will be secure all the way. However, configuration will need to be +added to GitLab to configure SSL certificates. See +[NGINX HTTPS documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https) +for details on managing SSL certificates and configuring NGINX. + +## Ports + +### Basic ports + +| LB Port | Backend Port | Protocol | +| ------- | ------------ | ------------------------ | +| 80 | 80 | HTTP (*1*) | +| 443 | 443 | TCP or HTTPS (*1*) (*2*) | +| 22 | 22 | TCP | + +- (*1*): [Web terminal](../ci/environments/index.md#web-terminals) support requires + your load balancer to correctly handle WebSocket connections. When using + HTTP or HTTPS proxying, this means your load balancer must be configured + to pass through the `Connection` and `Upgrade` hop-by-hop headers. See the + [web terminal](integration/terminal.md) integration guide for + more details. +- (*2*): When using HTTPS protocol for port 443, you will need to add an SSL + certificate to the load balancers. If you wish to terminate SSL at the + GitLab application server instead, use TCP protocol. + +### GitLab Pages Ports + +If you're using GitLab Pages with custom domain support you will need some +additional port configurations. +GitLab Pages requires a separate virtual IP address. Configure DNS to point the +`pages_external_url` from `/etc/gitlab/gitlab.rb` at the new virtual IP address. See the +[GitLab Pages documentation](pages/index.md) for more information. + +| LB Port | Backend Port | Protocol | +| ------- | ------------- | --------- | +| 80 | Varies (*1*) | HTTP | +| 443 | Varies (*1*) | TCP (*2*) | + +- (*1*): The backend port for GitLab Pages depends on the + `gitlab_pages['external_http']` and `gitlab_pages['external_https']` + setting. See [GitLab Pages documentation](pages/index.md) for more details. +- (*2*): Port 443 for GitLab Pages should always use the TCP protocol. Users can + configure custom domains with custom SSL, which would not be possible + if SSL was terminated at the load balancer. + +### Alternate SSH Port + +Some organizations have policies against opening SSH port 22. In this case, +it may be helpful to configure an alternate SSH hostname that allows users +to use SSH on port 443. An alternate SSH hostname will require a new virtual IP address +compared to the other GitLab HTTP configuration above. + +Configure DNS for an alternate SSH hostname such as `altssh.gitlab.example.com`. + +| LB Port | Backend Port | Protocol | +| ------- | ------------ | -------- | +| 443 | 22 | TCP | + +## Readiness check + +It is strongly recommend that multi-node deployments configure load balancers to utilize the [readiness check](../user/admin_area/monitoring/health_check.md#readiness) to ensure a node is ready to accept traffic, before routing traffic to it. This is especially important when utilizing Puma, as there is a brief period during a restart where Puma will not accept requests. + +<!-- ## Troubleshooting + +Include any troubleshooting steps that you can foresee. If you know beforehand what issues +one might have when setting this up, or when something is changed, or on upgrading, it's +important to describe those, too. Think of things that may go wrong and include them here. +This is important to minimize requests for support, and to avoid doc comments with +questions that you know someone might ask. + +Each scenario can be a third-level heading, e.g. `### Getting error message X`. +If you have none to add when creating a doc, leave this section in place +but commented out to help encourage others to add to it in the future. --> diff --git a/doc/administration/logs.md b/doc/administration/logs.md index 3db9d32563e..2e8d0bf7461 100644 --- a/doc/administration/logs.md +++ b/doc/administration/logs.md @@ -14,6 +14,11 @@ Find more about them [in Audit Events documentation](audit_events.md). System log files are typically plain text in a standard log file format. This guide talks about how to read and use these system log files. +[Read more about how to customise logging on Omnibus GitLab +installations](https://docs.gitlab.com/omnibus/settings/logs.html) +including adjusting log retention, log forwarding, +switching logs from JSON to plain text logging, and more. + ## `production_json.log` This file lives in `/var/log/gitlab/gitlab-rails/production_json.log` for @@ -832,6 +837,29 @@ For example: This message shows that Geo detected that a repository update was needed for project `1`. +## `update_mirror_service_json.log` + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/commit/7f637e2af7006dc2b1b2649d9affc0b86cfb33c4) in GitLab 11.12. + +This file is stored in: + +- `/var/log/gitlab/gitlab-rails/update_mirror_service_json.log` for Omnibus GitLab installations. +- `/home/git/gitlab/log/update_mirror_service_json.log` for installations from source. + +This file contains information about any errors that occurred during project mirroring. + +```json +{ + "severity":"ERROR", + "time":"2020-07-28T23:29:29.473Z", + "correlation_id":"5HgIkCJsO53", + "user_id":"x", + "project_id":"x", + "import_url":"https://mirror-source/group/project.git", + "error_message":"The LFS objects download list couldn't be imported. Error: Unauthorized" +} +``` + ## Registry Logs For Omnibus installations, Container Registry logs reside in `/var/log/gitlab/registry/current`. diff --git a/doc/administration/merge_request_diffs.md b/doc/administration/merge_request_diffs.md index 93cdbff6621..3f4cd6e2751 100644 --- a/doc/administration/merge_request_diffs.md +++ b/doc/administration/merge_request_diffs.md @@ -1,3 +1,10 @@ +--- +stage: Create +group: Editor +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference +--- + # Merge request diffs storage **(CORE ONLY)** > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/52568) in GitLab 11.8. diff --git a/doc/administration/monitoring/github_imports.md b/doc/administration/monitoring/github_imports.md index 21cc4c708a8..0d79684c951 100644 --- a/doc/administration/monitoring/github_imports.md +++ b/doc/administration/monitoring/github_imports.md @@ -6,8 +6,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Monitoring GitHub imports ->**Note:** -Available since [GitLab 10.2](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/14731). +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/14731) in GitLab 10.2. The GitHub importer exposes various Prometheus metrics that you can use to monitor the health and progress of the importer. diff --git a/doc/administration/monitoring/gitlab_self_monitoring_project/img/self_monitoring_default_dashboard.png b/doc/administration/monitoring/gitlab_self_monitoring_project/img/self_monitoring_overview_dashboard.png Binary files differindex 1d61823ce41..1d61823ce41 100644 --- a/doc/administration/monitoring/gitlab_self_monitoring_project/img/self_monitoring_default_dashboard.png +++ b/doc/administration/monitoring/gitlab_self_monitoring_project/img/self_monitoring_overview_dashboard.png diff --git a/doc/administration/monitoring/gitlab_self_monitoring_project/index.md b/doc/administration/monitoring/gitlab_self_monitoring_project/index.md index 44ba26296b9..e272cccb7ce 100644 --- a/doc/administration/monitoring/gitlab_self_monitoring_project/index.md +++ b/doc/administration/monitoring/gitlab_self_monitoring_project/index.md @@ -55,10 +55,10 @@ panels, provide a regular expression in the **Instance label regex** field. The dashboard uses metrics available in [Omnibus GitLab](https://docs.gitlab.com/omnibus/) installations. - + You can also -[create your own dashboards](../../../operations/metrics/dashboards/index.md#defining-custom-dashboards-per-project). +[create your own dashboards](../../../operations/metrics/dashboards/index.md). ## Connection to Prometheus @@ -83,8 +83,8 @@ Once the webhook is setup, you can You can add custom metrics in the self monitoring project by: -1. [Duplicating](../../../operations/metrics/dashboards/index.md#duplicating-a-gitlab-defined-dashboard) the default dashboard. -1. [Editing](../../../operations/metrics/dashboards/index.md#view-and-edit-the-source-file-of-a-custom-dashboard) the newly created dashboard file and configuring it with [dashboard YAML properties](../../../operations/metrics/dashboards/yaml.md). +1. [Duplicating](../../../operations/metrics/dashboards/index.md#duplicate-a-gitlab-defined-dashboard) the overview dashboard. +1. [Editing](../../../operations/metrics/index.md) the newly created dashboard file and configuring it with [dashboard YAML properties](../../../operations/metrics/dashboards/yaml.md). ## Troubleshooting diff --git a/doc/administration/monitoring/performance/grafana_configuration.md b/doc/administration/monitoring/performance/grafana_configuration.md index 96f1377fb73..136a2749e80 100644 --- a/doc/administration/monitoring/performance/grafana_configuration.md +++ b/doc/administration/monitoring/performance/grafana_configuration.md @@ -68,7 +68,7 @@ repository. After setting up Grafana, you can enable a link to access it easily from the GitLab sidebar: -1. Navigate to the **{admin}** **Admin Area > Settings > Metrics and profiling**. +1. Navigate to the **Admin Area > Settings > Metrics and profiling**. 1. Expand **Metrics - Grafana**. 1. Check the **Enable access to Grafana** checkbox. 1. Configure the **Grafana URL**: @@ -77,7 +77,7 @@ GitLab sidebar: - *Otherwise,* enter the full URL of the Grafana instance. 1. Click **Save changes**. -GitLab displays your link in the **{admin}** **Admin Area > Monitoring > Metrics Dashboard**. +GitLab displays your link in the **Admin Area > Monitoring > Metrics Dashboard**. ## Security Update diff --git a/doc/administration/monitoring/performance/performance_bar.md b/doc/administration/monitoring/performance/performance_bar.md index 8002a9ab296..e247ec3708c 100644 --- a/doc/administration/monitoring/performance/performance_bar.md +++ b/doc/administration/monitoring/performance/performance_bar.md @@ -23,7 +23,7 @@ From left to right, it displays: details:  - **Rugged calls**: the time taken (in milliseconds) and the total number of - [Rugged](../../high_availability/nfs.md#improving-nfs-performance-with-gitlab) calls. + [Rugged](../../nfs.md#improving-nfs-performance-with-gitlab) calls. Click to display a modal window with more details:  - **Redis calls**: the time taken (in milliseconds) and the total number of @@ -72,8 +72,8 @@ Requests with warnings display `(!)` after their path in the **Request selector* The GitLab Performance Bar is disabled by default. To enable it for a given group: 1. Sign in as a user with Administrator [permissions](../../../user/permissions.md). -1. In the menu bar, click the **{admin}** **Admin Area** icon. -1. Navigate to **{settings}** **Settings > Metrics and profiling** +1. In the menu bar, click **Admin Area**. +1. Navigate to **Settings > Metrics and profiling** (`admin/application_settings/metrics_and_profiling`), and expand the section **Profiling - Performance bar**. 1. Click **Enable access to the Performance Bar**. diff --git a/doc/administration/monitoring/performance/request_profiling.md b/doc/administration/monitoring/performance/request_profiling.md index a3b29493d84..5746b95eb44 100644 --- a/doc/administration/monitoring/performance/request_profiling.md +++ b/doc/administration/monitoring/performance/request_profiling.md @@ -9,8 +9,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w To profile a request: 1. Sign in to GitLab as a user with Administrator or Maintainer [permissions](../../../user/permissions.md). -1. In the navigation bar, click **{admin}** **Admin area**. -1. Navigate to **{monitor}** **Monitoring > Requests Profiles**. +1. In the navigation bar, click **Admin area**. +1. Navigate to **Monitoring > Requests Profiles**. 1. In the **Requests Profiles** section, copy the token. 1. Pass the headers `X-Profile-Token: <token>` and `X-Profile-Mode: <mode>`(where `<mode>` can be `execution` or `memory`) to the request you want to profile. When @@ -29,7 +29,7 @@ To profile a request: Profiled requests can take longer than usual. After the request completes, you can view the profiling output from the -**{monitor}** **Monitoring > Requests Profiles** administration page: +**Monitoring > Requests Profiles** administration page:  diff --git a/doc/administration/monitoring/prometheus/gitlab_metrics.md b/doc/administration/monitoring/prometheus/gitlab_metrics.md index fa5e5da80c3..bff689c0c0c 100644 --- a/doc/administration/monitoring/prometheus/gitlab_metrics.md +++ b/doc/administration/monitoring/prometheus/gitlab_metrics.md @@ -9,7 +9,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w To enable the GitLab Prometheus metrics: 1. Log into GitLab as a user with [administrator permissions](../../../user/permissions.md). -1. Navigate to **{admin}** **Admin Area > Settings > Metrics and profiling**. +1. Navigate to **Admin Area > Settings > Metrics and profiling**. 1. Find the **Metrics - Prometheus** section, and click **Enable Prometheus Metrics**. 1. [Restart GitLab](../../restart_gitlab.md#omnibus-gitlab-restart) for the changes to take effect. @@ -50,7 +50,8 @@ The following metrics are available: | `gitlab_page_out_of_bounds` | Counter | 12.8 | Counter for the PageLimiter pagination limit being hit | `controller`, `action`, `bot` | | `gitlab_rails_queue_duration_seconds` | Histogram | 9.4 | Measures latency between GitLab Workhorse forwarding a request to Rails | | | `gitlab_sql_duration_seconds` | Histogram | 10.2 | SQL execution time, excluding `SCHEMA` operations and `BEGIN` / `COMMIT` | | -| `gitlab_transaction_allocated_memory_bytes` | Histogram | 10.2 | Allocated memory for all transactions (`gitlab_transaction_*` metrics) | | +| `gitlab_ruby_threads_max_expected_threads` | Gauge | 13.3 | Maximum number of threads expected to be running and performing application work | +| `gitlab_ruby_threads_running_threads` | Gauge | 13.3 | Number of running Ruby threads by name | | `gitlab_transaction_cache_<key>_count_total` | Counter | 10.2 | Counter for total Rails cache calls (per key) | | | `gitlab_transaction_cache_<key>_duration_total` | Counter | 10.2 | Counter for total time (seconds) spent in Rails cache calls (per key) | | | `gitlab_transaction_cache_count_total` | Counter | 10.2 | Counter for total Rails cache calls (aggregate) | | @@ -95,8 +96,6 @@ The following metrics are available: | `gitlab_transaction_db_count_total` | Counter | 13.1 | Counter for total number of SQL calls | `controller`, `action` | | `gitlab_transaction_db_write_count_total` | Counter | 13.1 | Counter for total number of write SQL calls | `controller`, `action` | | `gitlab_transaction_db_cached_count_total` | Counter | 13.1 | Counter for total number of cached SQL calls | `controller`, `action` | -| `http_redis_requests_duration_seconds` | Histogram | 13.1 | Redis requests duration during web transactions | `controller`, `action` | -| `http_redis_requests_total` | Counter | 13.1 | Redis requests count during web transactions | `controller`, `action` | | `http_elasticsearch_requests_duration_seconds` **(STARTER)** | Histogram | 13.1 | Elasticsearch requests duration during web transactions | `controller`, `action` | | `http_elasticsearch_requests_total` **(STARTER)** | Counter | 13.1 | Elasticsearch requests count during web transactions | `controller`, `action` | | `pipelines_created_total` | Counter | 9.4 | Counter of pipelines created | | diff --git a/doc/administration/monitoring/prometheus/index.md b/doc/administration/monitoring/prometheus/index.md index f0ad0a1a2e6..7d93e9797be 100644 --- a/doc/administration/monitoring/prometheus/index.md +++ b/doc/administration/monitoring/prometheus/index.md @@ -109,6 +109,81 @@ prometheus['scrape_configs'] = [ ] ``` +### Standalone Prometheus using Omnibus GitLab + +The Omnibus GitLab package can be used to configure a standalone Monitoring node running Prometheus and [Grafana](../performance/grafana_configuration.md). + +The steps below are the minimum necessary to configure a Monitoring node running Prometheus and Grafana with Omnibus GitLab: + +1. SSH into the Monitoring node. +1. [Install](https://about.gitlab.com/install/) the Omnibus GitLab + package you want using **steps 1 and 2** from the GitLab downloads page, but + do not follow the remaining steps. +1. Make sure to collect the IP addresses or DNS records of the Consul server nodes, for the next step. +1. Edit `/etc/gitlab/gitlab.rb` and add the contents: + + ```ruby + external_url 'http://gitlab.example.com' + + # Enable Prometheus + prometheus['enable'] = true + prometheus['listen_address'] = '0.0.0.0:9090' + prometheus['monitor_kubernetes'] = false + + # Enable Login form + grafana['disable_login_form'] = false + + # Enable Grafana + grafana['enable'] = true + grafana['admin_password'] = 'toomanysecrets' + + # Enable service discovery for Prometheus + consul['enable'] = true + consul['monitoring_service_discovery'] = true + + # The addresses can be IPs or FQDNs + consul['configuration'] = { + retry_join: %w(10.0.0.1 10.0.0.2 10.0.0.3), + } + + # Disable all other services + gitlab_rails['auto_migrate'] = false + alertmanager['enable'] = false + gitaly['enable'] = false + gitlab_exporter['enable'] = false + gitlab_workhorse['enable'] = false + nginx['enable'] = true + postgres_exporter['enable'] = false + postgresql['enable'] = false + redis['enable'] = false + redis_exporter['enable'] = false + sidekiq['enable'] = false + puma['enable'] = false + node_exporter['enable'] = false + gitlab_exporter['enable'] = false + ``` + +1. Run `sudo gitlab-ctl reconfigure` to compile the configuration. + +The next step is to tell all the other nodes where the monitoring node is: + +1. Edit `/etc/gitlab/gitlab.rb`, and add, or find and uncomment the following line: + + ```ruby + gitlab_rails['prometheus_address'] = '10.0.0.1:9090' + ``` + + Where `10.0.0.1:9090` is the IP address and port of the Prometheus node. + +1. Save the file and [reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to + take effect. + +NOTE: **Note:** +Once monitoring using Service Discovery is enabled with `consul['monitoring_service_discovery'] = true`, +ensure that `prometheus['scrape_configs']` is not set in `/etc/gitlab/gitlab.rb`. Setting both +`consul['monitoring_service_discovery'] = true` and `prometheus['scrape_configs']` in `/etc/gitlab/gitlab.rb` +will result in errors. + ### Using an external Prometheus server NOTE: **Note:** @@ -128,14 +203,24 @@ To use an external Prometheus server: 1. Set each bundled service's [exporter](#bundled-software-metrics) to listen on a network address, for example: ```ruby + node_exporter['listen_address'] = '0.0.0.0:9100' + gitlab_workhorse['prometheus_listen_addr'] = "0.0.0.0:9229" + + # Rails nodes gitlab_exporter['listen_address'] = '0.0.0.0' - sidekiq['listen_address'] = '0.0.0.0' gitlab_exporter['listen_port'] = '9168' - node_exporter['listen_address'] = '0.0.0.0:9100' + + # Sidekiq nodes + sidekiq['listen_address'] = '0.0.0.0' + + # Redis nodes redis_exporter['listen_address'] = '0.0.0.0:9121' + + # PostgreSQL nodes postgres_exporter['listen_address'] = '0.0.0.0:9187' + + # Gitaly nodes gitaly['prometheus_listen_addr'] = "0.0.0.0:9236" - gitlab_workhorse['prometheus_listen_addr'] = "0.0.0.0:9229" ``` 1. Install and set up a dedicated Prometheus instance, if necessary, using the [official installation instructions](https://prometheus.io/docs/prometheus/latest/installation/). @@ -227,7 +312,7 @@ To use an external Prometheus server: You can visit `http://localhost:9090` for the dashboard that Prometheus offers by default. ->**Note:** +NOTE: **Note:** If SSL has been enabled on your GitLab instance, you may not be able to access Prometheus on the same browser as GitLab if using the same FQDN due to [HSTS](https://en.wikipedia.org/wiki/HTTP_Strict_Transport_Security). We plan to [provide access via GitLab](https://gitlab.com/gitlab-org/multi-user-prometheus), but in the interim there are diff --git a/doc/administration/nfs.md b/doc/administration/nfs.md new file mode 100644 index 00000000000..bae6bd0dd6c --- /dev/null +++ b/doc/administration/nfs.md @@ -0,0 +1,368 @@ +--- +type: reference +--- + +# Using NFS with GitLab + +NFS can be used as an alternative for object storage but this isn't typically +recommended for performance reasons. Note however it is required for [GitLab +Pages](https://gitlab.com/gitlab-org/gitlab-pages/-/issues/196). + +For data objects such as LFS, Uploads, Artifacts, etc., an [Object Storage service](object_storage.md) +is recommended over NFS where possible, due to better performance. + +CAUTION: **Caution:** +From GitLab 13.0, using NFS for Git repositories is deprecated. In GitLab 14.0, +support for NFS for Git repositories is scheduled to be removed. Upgrade to +[Gitaly Cluster](gitaly/praefect.md) as soon as possible. + +NOTE: **Note:** +Filesystem performance has a big impact on overall GitLab +performance, especially for actions that read or write to Git repositories. See +[Filesystem Performance Benchmarking](operations/filesystem_benchmarking.md) +for steps to test filesystem performance. + +## Known kernel version incompatibilities + +RedHat Enterprise Linux (RHEL) and CentOS v7.7 and v7.8 ship with kernel +version `3.10.0-1127`, which [contains a +bug](https://bugzilla.redhat.com/show_bug.cgi?id=1783554) that causes +[uploads to fail to copy over NFS](https://gitlab.com/gitlab-org/gitlab/-/issues/218999). The +following GitLab versions include a fix to work properly with that +kernel version: + +1. [12.10.12](https://about.gitlab.com/releases/2020/06/25/gitlab-12-10-12-released/) +1. [13.0.7](https://about.gitlab.com/releases/2020/06/25/gitlab-13-0-7-released/) +1. [13.1.1](https://about.gitlab.com/releases/2020/06/24/gitlab-13-1-1-released/) +1. 13.2 and up + +If you are using that kernel version, be sure to upgrade GitLab to avoid +errors. + +## Fast lookup of authorized SSH keys + +The [fast SSH key lookup](operations/fast_ssh_key_lookup.md) feature can improve +performance of GitLab instances even if they're using block storage. + +[Fast SSH key lookup](operations/fast_ssh_key_lookup.md) is a replacement for +`authorized_keys` (in `/var/opt/gitlab/.ssh`) using the GitLab database. + +NFS increases latency, so fast lookup is recommended if `/var/opt/gitlab` +is moved to NFS. + +We are investigating the use of +[fast lookup as the default](https://gitlab.com/groups/gitlab-org/-/epics/3104). + +## NFS server + +Installing the `nfs-kernel-server` package allows you to share directories with +the clients running the GitLab application: + +```shell +sudo apt-get update +sudo apt-get install nfs-kernel-server +``` + +### Required features + +**File locking**: GitLab **requires** advisory file locking, which is only +supported natively in NFS version 4. NFSv3 also supports locking as long as +Linux Kernel 2.6.5+ is used. We recommend using version 4 and do not +specifically test NFSv3. + +### Recommended options + +When you define your NFS exports, we recommend you also add the following +options: + +- `no_root_squash` - NFS normally changes the `root` user to `nobody`. This is + a good security measure when NFS shares will be accessed by many different + users. However, in this case only GitLab will use the NFS share so it + is safe. GitLab recommends the `no_root_squash` setting because we need to + manage file permissions automatically. Without the setting you may receive + errors when the Omnibus package tries to alter permissions. Note that GitLab + and other bundled components do **not** run as `root` but as non-privileged + users. The recommendation for `no_root_squash` is to allow the Omnibus package + to set ownership and permissions on files, as needed. In some cases where the + `no_root_squash` option is not available, the `root` flag can achieve the same + result. +- `sync` - Force synchronous behavior. Default is asynchronous and under certain + circumstances it could lead to data loss if a failure occurs before data has + synced. + +Due to the complexities of running Omnibus with LDAP and the complexities of +maintaining ID mapping without LDAP, in most cases you should enable numeric UIDs +and GIDs (which is off by default in some cases) for simplified permission +management between systems: + +- [NetApp instructions](https://library.netapp.com/ecmdocs/ECMP1401220/html/GUID-24367A9F-E17B-4725-ADC1-02D86F56F78E.html) +- For non-NetApp devices, disable NFSv4 `idmapping` by performing opposite of [enable NFSv4 idmapper](https://wiki.archlinux.org/index.php/NFS#Enabling_NFSv4_idmapping) + +### Disable NFS server delegation + +We recommend that all NFS users disable the NFS server delegation feature. This +is to avoid a [Linux kernel bug](https://bugzilla.redhat.com/show_bug.cgi?id=1552203) +which causes NFS clients to slow precipitously due to +[excessive network traffic from numerous `TEST_STATEID` NFS messages](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/52017). + +To disable NFS server delegation, do the following: + +1. On the NFS server, run: + + ```shell + echo 0 > /proc/sys/fs/leases-enable + sysctl -w fs.leases-enable=0 + ``` + +1. Restart the NFS server process. For example, on CentOS run `service nfs restart`. + +NOTE: **Important note:** +The kernel bug may be fixed in +[more recent kernels with this commit](https://github.com/torvalds/linux/commit/95da1b3a5aded124dd1bda1e3cdb876184813140). +Red Hat Enterprise 7 [shipped a kernel update](https://access.redhat.com/errata/RHSA-2019:2029) +on August 6, 2019 that may also have resolved this problem. +You may not need to disable NFS server delegation if you know you are using a version of +the Linux kernel that has been fixed. That said, GitLab still encourages instance +administrators to keep NFS server delegation disabled. + +### Improving NFS performance with GitLab + +#### Improving NFS performance with Unicorn + +NOTE: **Note:** +From GitLab 12.1, it will automatically be detected if Rugged can and should be used per storage. + +If you previously enabled Rugged using the feature flag, you will need to unset the feature flag by using: + +```shell +sudo gitlab-rake gitlab:features:unset_rugged +``` + +If the Rugged feature flag is explicitly set to either true or false, GitLab will use the value explicitly set. + +#### Improving NFS performance with Puma + +NOTE: **Note:** +From GitLab 12.7, Rugged auto-detection is disabled if Puma thread count is greater than 1. + +If you want to use Rugged with Puma, it is recommended to [set Puma thread count to 1](https://docs.gitlab.com/omnibus/settings/puma.html#puma-settings). + +If you want to use Rugged with Puma thread count more than 1, Rugged can be enabled using the [feature flag](../development/gitaly.md#legacy-rugged-code) + +If the Rugged feature flag is explicitly set to either true or false, GitLab will use the value explicitly set. + +## NFS client + +The `nfs-common` provides NFS functionality without installing server components which +we don't need running on the application nodes. + +```shell +apt-get update +apt-get install nfs-common +``` + +### Mount options + +Here is an example snippet to add to `/etc/fstab`: + +```plaintext +10.1.0.1:/var/opt/gitlab/.ssh /var/opt/gitlab/.ssh nfs4 defaults,vers=4.1,hard,rsize=1048576,wsize=1048576,noatime,nofail,lookupcache=positive 0 2 +10.1.0.1:/var/opt/gitlab/gitlab-rails/uploads /var/opt/gitlab/gitlab-rails/uploads nfs4 defaults,vers=4.1,hard,rsize=1048576,wsize=1048576,noatime,nofail,lookupcache=positive 0 2 +10.1.0.1:/var/opt/gitlab/gitlab-rails/shared /var/opt/gitlab/gitlab-rails/shared nfs4 defaults,vers=4.1,hard,rsize=1048576,wsize=1048576,noatime,nofail,lookupcache=positive 0 2 +10.1.0.1:/var/opt/gitlab/gitlab-ci/builds /var/opt/gitlab/gitlab-ci/builds nfs4 defaults,vers=4.1,hard,rsize=1048576,wsize=1048576,noatime,nofail,lookupcache=positive 0 2 +10.1.0.1:/var/opt/gitlab/git-data /var/opt/gitlab/git-data nfs4 defaults,vers=4.1,hard,rsize=1048576,wsize=1048576,noatime,nofail,lookupcache=positive 0 2 +``` + +Note there are several options that you should consider using: + +| Setting | Description | +| ------- | ----------- | +| `vers=4.1` |NFS v4.1 should be used instead of v4.0 because there is a Linux [NFS client bug in v4.0](https://gitlab.com/gitlab-org/gitaly/-/issues/1339) that can cause significant problems due to stale data. +| `nofail` | Don't halt boot process waiting for this mount to become available +| `lookupcache=positive` | Tells the NFS client to honor `positive` cache results but invalidates any `negative` cache results. Negative cache results cause problems with Git. Specifically, a `git push` can fail to register uniformly across all NFS clients. The negative cache causes the clients to 'remember' that the files did not exist previously. +| `hard` | Instead of `soft`. [Further details](#soft-mount-option). + +#### `soft` mount option + +It's recommended that you use `hard` in your mount options, unless you have a specific +reason to use `soft`. + +On GitLab.com, we use `soft` because there were times when we had NFS servers +reboot and `soft` improved availability, but everyone's infrastructure is different. +If your NFS is provided by on-premise storage arrays with redundant controllers, +for example, you shouldn't need to worry about NFS server availability. + +The NFS man page states: + +> "soft" timeout can cause silent data corruption in certain cases + +Read the [Linux man page](https://linux.die.net/man/5/nfs) to understand the difference, +and if you do use `soft`, ensure that you've taken steps to mitigate the risks. + +If you experience behavior that might have been caused by +writes to disk on the NFS server not occurring, such as commits going missing, +use the `hard` option, because (from the man page): + +> use the soft option only when client responsiveness is more important than data integrity + +Other vendors make similar recommendations, including +[SAP](http://wiki.scn.sap.com/wiki/x/PARnFQ) and NetApp's +[knowledge base](https://kb.netapp.com/Advice_and_Troubleshooting/Data_Storage_Software/ONTAP_OS/What_are_the_differences_between_hard_mount_and_soft_mount), +they highlight that if the NFS client driver caches data, `soft` means there is no certainty if +writes by GitLab are actually on disk. + +Mount points set with the option `hard` may not perform as well, and if the +NFS server goes down, `hard` will cause processes to hang when interacting with +the mount point. Use `SIGKILL` (`kill -9`) to deal with hung processes. +The `intr` option +[stopped working in the 2.6 kernel](https://access.redhat.com/solutions/157873). + +### A single NFS mount + +It's recommended to nest all GitLab data directories within a mount, that allows automatic +restore of backups without manually moving existing data. + +```plaintext +mountpoint +└── gitlab-data + ├── builds + ├── git-data + ├── shared + └── uploads +``` + +To do so, we'll need to configure Omnibus with the paths to each directory nested +in the mount point as follows: + +Mount `/gitlab-nfs` then use the following Omnibus +configuration to move each data location to a subdirectory: + +```ruby +git_data_dirs({"default" => { "path" => "/gitlab-nfs/gitlab-data/git-data"} }) +gitlab_rails['uploads_directory'] = '/gitlab-nfs/gitlab-data/uploads' +gitlab_rails['shared_path'] = '/gitlab-nfs/gitlab-data/shared' +gitlab_ci['builds_directory'] = '/gitlab-nfs/gitlab-data/builds' +``` + +Run `sudo gitlab-ctl reconfigure` to start using the central location. Please +be aware that if you had existing data you will need to manually copy/rsync it +to these new locations and then restart GitLab. + +### Bind mounts + +Alternatively to changing the configuration in Omnibus, bind mounts can be used +to store the data on an NFS mount. + +Bind mounts provide a way to specify just one NFS mount and then +bind the default GitLab data locations to the NFS mount. Start by defining your +single NFS mount point as you normally would in `/etc/fstab`. Let's assume your +NFS mount point is `/gitlab-nfs`. Then, add the following bind mounts in +`/etc/fstab`: + +```shell +/gitlab-nfs/gitlab-data/git-data /var/opt/gitlab/git-data none bind 0 0 +/gitlab-nfs/gitlab-data/.ssh /var/opt/gitlab/.ssh none bind 0 0 +/gitlab-nfs/gitlab-data/uploads /var/opt/gitlab/gitlab-rails/uploads none bind 0 0 +/gitlab-nfs/gitlab-data/shared /var/opt/gitlab/gitlab-rails/shared none bind 0 0 +/gitlab-nfs/gitlab-data/builds /var/opt/gitlab/gitlab-ci/builds none bind 0 0 +``` + +Using bind mounts will require manually making sure the data directories +are empty before attempting a restore. Read more about the +[restore prerequisites](../raketasks/backup_restore.md). + +You can view information and options set for each of the mounted NFS file +systems by running `nfsstat -m` and `cat /etc/fstab`. + +### Multiple NFS mounts + +When using default Omnibus configuration you will need to share 4 data locations +between all GitLab cluster nodes. No other locations should be shared. The +following are the 4 locations need to be shared: + +| Location | Description | Default configuration | +| -------- | ----------- | --------------------- | +| `/var/opt/gitlab/git-data` | Git repository data. This will account for a large portion of your data | `git_data_dirs({"default" => { "path" => "/var/opt/gitlab/git-data"} })` +| `/var/opt/gitlab/gitlab-rails/uploads` | User uploaded attachments | `gitlab_rails['uploads_directory'] = '/var/opt/gitlab/gitlab-rails/uploads'` +| `/var/opt/gitlab/gitlab-rails/shared` | Build artifacts, GitLab Pages, LFS objects, temp files, etc. If you're using LFS this may also account for a large portion of your data | `gitlab_rails['shared_path'] = '/var/opt/gitlab/gitlab-rails/shared'` +| `/var/opt/gitlab/gitlab-ci/builds` | GitLab CI/CD build traces | `gitlab_ci['builds_directory'] = '/var/opt/gitlab/gitlab-ci/builds'` + +Other GitLab directories should not be shared between nodes. They contain +node-specific files and GitLab code that does not need to be shared. To ship +logs to a central location consider using remote syslog. Omnibus GitLab packages +provide configuration for [UDP log shipping](https://docs.gitlab.com/omnibus/settings/logs.html#udp-log-shipping-gitlab-enterprise-edition-only). + +Having multiple NFS mounts will require manually making sure the data directories +are empty before attempting a restore. Read more about the +[restore prerequisites](../raketasks/backup_restore.md). + +## NFS in a Firewalled Environment + +If the traffic between your NFS server and NFS client(s) is subject to port filtering +by a firewall, then you will need to reconfigure that firewall to allow NFS communication. + +[This guide from TDLP](http://tldp.org/HOWTO/NFS-HOWTO/security.html#FIREWALLS) +covers the basics of using NFS in a firewalled environment. Additionally, we encourage you to +search for and review the specific documentation for your operating system or distribution and your firewall software. + +Example for Ubuntu: + +Check that NFS traffic from the client is allowed by the firewall on the host by running +the command: `sudo ufw status`. If it's being blocked, then you can allow traffic from a specific +client with the command below. + +```shell +sudo ufw allow from <client_ip_address> to any port nfs +``` + +## Known issues + +### Avoid using AWS's Elastic File System (EFS) + +GitLab strongly recommends against using AWS Elastic File System (EFS). +Our support team will not be able to assist on performance issues related to +file system access. + +Customers and users have reported that AWS EFS does not perform well for GitLab's +use-case. Workloads where many small files are written in a serialized manner, like `git`, +are not well-suited for EFS. EBS with an NFS server on top will perform much better. + +If you do choose to use EFS, avoid storing GitLab log files (e.g. those in `/var/log/gitlab`) +there because this will also affect performance. We recommend that the log files be +stored on a local volume. + +For more details on another person's experience with EFS, see this [Commit Brooklyn 2019 video](https://youtu.be/K6OS8WodRBQ?t=313). + +### Avoid using CephFS and GlusterFS + +GitLab strongly recommends against using CephFS and GlusterFS. +These distributed file systems are not well-suited for GitLab's input/output access patterns because Git uses many small files and access times and file locking times to propagate will make Git activity very slow. + +### Avoid using PostgreSQL with NFS + +GitLab strongly recommends against running your PostgreSQL database +across NFS. The GitLab support team will not be able to assist on performance issues related to +this configuration. + +Additionally, this configuration is specifically warned against in the +[PostgreSQL Documentation](https://www.postgresql.org/docs/current/creating-cluster.html#CREATING-CLUSTER-NFS): + +>PostgreSQL does nothing special for NFS file systems, meaning it assumes NFS behaves exactly like +>locally-connected drives. If the client or server NFS implementation does not provide standard file +>system semantics, this can cause reliability problems. Specifically, delayed (asynchronous) writes +>to the NFS server can cause data corruption problems. + +For supported database architecture, please see our documentation on +[Configuring a Database for GitLab HA](postgresql/replication_and_failover.md). + +<!-- ## Troubleshooting + +Include any troubleshooting steps that you can foresee. If you know beforehand what issues +one might have when setting this up, or when something is changed, or on upgrading, it's +important to describe those, too. Think of things that may go wrong and include them here. +This is important to minimize requests for support, and to avoid doc comments with +questions that you know someone might ask. + +Each scenario can be a third-level heading, e.g. `### Getting error message X`. +If you have none to add when creating a doc, leave this section in place +but commented out to help encourage others to add to it in the future. --> diff --git a/doc/administration/object_storage.md b/doc/administration/object_storage.md index b51b722fbd7..49716883310 100644 --- a/doc/administration/object_storage.md +++ b/doc/administration/object_storage.md @@ -15,12 +15,18 @@ GitLab has been tested on a number of object storage providers: - [Amazon S3](https://aws.amazon.com/s3/) - [Google Cloud Storage](https://cloud.google.com/storage) -- [Digital Ocean Spaces](https://www.digitalocean.com/products/spaces) +- [Digital Ocean Spaces](https://www.digitalocean.com/products/spaces/) - [Oracle Cloud Infrastructure](https://docs.cloud.oracle.com/en-us/iaas/Content/Object/Tasks/s3compatibleapi.htm) - [Openstack Swift](https://docs.openstack.org/swift/latest/s3_compat.html) - On-premises hardware and appliances from various storage vendors. - MinIO. We have [a guide to deploying this](https://docs.gitlab.com/charts/advanced/external-object-storage/minio.html) within our Helm Chart documentation. +### Known compatibility issues + +- Dell EMC ECS: Prior to GitLab 13.3, there is a [known bug in GitLab Workhorse that prevents + HTTP Range Requests from working with CI job artifacts](https://gitlab.com/gitlab-org/gitlab/-/issues/223806). + Be sure to upgrade to GitLab v13.3.0 or above if you use S3 storage with this hardware. + ## Configuration guides There are two ways of specifying object storage configuration in GitLab: @@ -55,6 +61,10 @@ NOTE: **Note:** Consolidated object storage configuration cannot be used for backups or Mattermost. See [the full table for a complete list](#storage-specific-configuration). +NOTE: **Note:** +Enabling consolidated object storage will enable object storage for all object types. +If you wish to use local storage for specific object types, you can [selectively disable object storages](#selectively-disabling-object-storage). + Most types of objects, such as CI artifacts, LFS files, upload attachments, and so on can be saved in object storage by specifying a single credential for object storage with multiple buckets. A [different bucket @@ -84,6 +94,11 @@ See the section on [ETag mismatch errors](#etag-mismatch) for more details. 'aws_access_key_id' => '<AWS_ACCESS_KEY_ID>', 'aws_secret_access_key' => '<AWS_SECRET_ACCESS_KEY>' } + # OPTIONAL: The following lines are only needed if server side encryption is required + gitlab_rails['object_store']['storage_options'] = { + 'server_side_encryption' => '<AES256 or aws:kms>', + 'server_side_encryption_kms_key_id' => '<arn:s3:aws:xxx>' + } gitlab_rails['object_store']['objects']['artifacts']['bucket'] = '<artifacts>' gitlab_rails['object_store']['objects']['external_diffs']['bucket'] = '<external-diffs>' gitlab_rails['object_store']['objects']['lfs']['bucket'] = '<lfs-objects>' @@ -119,6 +134,9 @@ See the section on [ETag mismatch errors](#etag-mismatch) for more details. aws_access_key_id: <AWS_ACCESS_KEY_ID> aws_secret_access_key: <AWS_SECRET_ACCESS_KEY> region: <eu-central-1> + storage_options: + server_side_encryption: <AES256 or aws:kms> + server_side_encryption_key_kms_id: <arn:s3:aws:xxx> objects: artifacts: bucket: <artifacts> @@ -184,7 +202,8 @@ gitlab_rails['object_store']['connection'] = { |---------|-------------| | `enabled` | Enable/disable object storage | | `proxy_download` | Set to `true` to [enable proxying all files served](#proxy-download). Option allows to reduce egress traffic as this allows clients to download directly from remote storage instead of proxying all data | -| `connection` | Various connection options described below | +| `connection` | Various [connection options](#connection-settings) described below | +| `storage_options` | Options to use when saving new objects, such as [server side encryption](#server-side-encryption-headers). Introduced in GitLab 13.3 | | `objects` | [Object-specific configuration](#object-specific-configuration) ### Connection settings @@ -494,16 +513,18 @@ If you configure GitLab to use object storage for CI logs and artifacts, ### Proxy Download -A number of the use cases for object storage allow client traffic to be redirected to the -object storage back end, like when Git clients request large files via LFS or when -downloading CI artifacts and logs. +Clients can download files in object storage by receiving a pre-signed, time-limited URL, +or by GitLab proxying the data from object storage to the client. +Downloading files from object storage directly +helps reduce the amount of egress traffic GitLab +needs to process. When the files are stored on local block storage or NFS, GitLab has to act as a proxy. This is not the default behavior with object storage. The `proxy_download` setting controls this behavior: the default is generally `false`. -Verify this in the documentation for each use case. Set it to `true` so that GitLab proxies -the files. +Verify this in the documentation for each use case. Set it to `true` if you want +GitLab to proxy the files. When not proxying files, GitLab returns an [HTTP 302 redirect with a pre-signed, time-limited object storage URL](https://gitlab.com/gitlab-org/gitlab/-/issues/32117#note_218532298). @@ -524,7 +545,9 @@ certificate, or may return common TLS errors such as: x509: certificate signed by unknown authority ``` -- Clients will need network access to the object storage. Errors that might result +- Clients will need network access to the object storage. +Network firewalls could block access. +Errors that might result if this access is not in place include: ```plaintext @@ -535,6 +558,10 @@ Getting a `403 Forbidden` response is specifically called out on the [package repository documentation](packages/index.md#using-object-storage) as a side effect of how some build tools work. +Additionally for a short time period users could share pre-signed, time-limited object storage URLs +with others without authentication. Also bandwidth charges may be incurred +between the object storage provider and the client. + ### ETag mismatch Using the default GitLab settings, some object storage back-ends such as @@ -576,21 +603,46 @@ configuration. #### Encrypted S3 buckets -> - Introduced in [GitLab 13.1](https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/466) for instance profiles only. -> - Introduced in [GitLab 13.2](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/34460) for static credentials when [consolidated object storage configuration](#consolidated-object-storage-configuration) is used. +> - Introduced in [GitLab 13.1](https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/466) for instance profiles only and [S3 default encryption](https://docs.aws.amazon.com/AmazonS3/latest/dev/bucket-encryption.html). +> - Introduced in [GitLab 13.2](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/34460) for static credentials when [consolidated object storage configuration](#consolidated-object-storage-configuration) and [S3 default encryption](https://docs.aws.amazon.com/AmazonS3/latest/dev/bucket-encryption.html) are used. When configured either with an instance profile or with the consolidated -object configuration, GitLab Workhorse properly uploads files to S3 buckets -that have [SSE-S3 or SSE-KMS encryption enabled by +object configuration, GitLab Workhorse properly uploads files to S3 +buckets that have [SSE-S3 or SSE-KMS encryption enabled by default](https://docs.aws.amazon.com/kms/latest/developerguide/services-s3.html). -Note that customer master keys (CMKs) and -SSE-C encryption are [not yet supported since this requires supplying -keys to the GitLab configuration](https://gitlab.com/gitlab-org/gitlab/-/issues/226006). +Note that customer master keys (CMKs) and SSE-C encryption are [not +supported since this requires sending the encryption keys in every request](https://gitlab.com/gitlab-org/gitlab/-/issues/226006). + +##### Server-side encryption headers + +> Introduced in [GitLab 13.3](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/38240). + +Setting a default encryption on an S3 bucket is the easiest way to +enable encryption, but you may want to [set a bucket policy to ensure +only encrypted objects are uploaded](https://aws.amazon.com/premiumsupport/knowledge-center/s3-bucket-store-kms-encrypted-objects/). +To do this, you must configure GitLab to send the proper encryption headers +in the `storage_options` configuration section: + +| Setting | Description | +|-------------------------------------|-------------| +| `server_side_encryption` | Encryption mode (AES256 or aws:kms) | +| `server_side_encryption_kms_key_id` | Amazon Resource Name. Only needed when `aws:kms` is used in `server_side_encryption`. See the [Amazon documentation on using KMS encryption](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html) | + +As with the case for default encryption, these options only work when +the Workhorse S3 client is enabled. One of the following two conditions +must be fulfilled: + +- `use_iam_profile` is `true` in the connection settings. +- Consolidated object storage settings are in use. + +[ETag mismatch errors](#etag-mismatch) will occur if server side +encryption headers are used without enabling the Workhorse S3 client. ##### Disabling the feature The Workhorse S3 client is enabled by default when the -[`use_iam_profile` configuration option](#iam-permissions) is set to `true`. +[`use_iam_profile` configuration option](#iam-permissions) is set to `true` or consolidated +object storage settings are configured. The feature can be disabled using the `:use_workhorse_s3_client` feature flag. To disable the feature, ask a GitLab administrator with diff --git a/doc/administration/operations/cleaning_up_redis_sessions.md b/doc/administration/operations/cleaning_up_redis_sessions.md index 38fac8a0eca..d2aec2f7c47 100644 --- a/doc/administration/operations/cleaning_up_redis_sessions.md +++ b/doc/administration/operations/cleaning_up_redis_sessions.md @@ -15,7 +15,8 @@ prefixed with `session:gitlab:`, so they would look like `session:gitlab:976aa289e2189b17d7ef525a6702ace9`. Below we describe how to remove the keys in the old format. -**Note:** the instructions below must be modified in accordance with your +NOTE: **Note:** +The instructions below must be modified in accordance with your configuration settings if you have used the advanced Redis settings outlined in [Configuration Files Documentation](https://gitlab.com/gitlab-org/gitlab/blob/master/config/README.md). diff --git a/doc/administration/operations/extra_sidekiq_processes.md b/doc/administration/operations/extra_sidekiq_processes.md index 155680354da..e589ecc4216 100644 --- a/doc/administration/operations/extra_sidekiq_processes.md +++ b/doc/administration/operations/extra_sidekiq_processes.md @@ -82,7 +82,7 @@ To start multiple processes: ``` After the extra Sidekiq processes are added, navigate to -**{admin}** **Admin Area > Monitoring > Background Jobs** (`/admin/background_jobs`) in GitLab. +**Admin Area > Monitoring > Background Jobs** (`/admin/background_jobs`) in GitLab.  diff --git a/doc/administration/operations/filesystem_benchmarking.md b/doc/administration/operations/filesystem_benchmarking.md index 856061348ed..64afd1b44f3 100644 --- a/doc/administration/operations/filesystem_benchmarking.md +++ b/doc/administration/operations/filesystem_benchmarking.md @@ -26,7 +26,7 @@ To install: Then run the following: ```shell -fio --randrepeat=1 --ioengine=libaio --direct=1 --gtod_reduce=1 --name=test --filename=/path/to/git-data/testfile --bs=4k --iodepth=64 --size=4G --readwrite=randrw --rwmixread=75 +fio --randrepeat=1 --ioengine=libaio --direct=1 --gtod_reduce=1 --name=test --bs=4k --iodepth=64 --readwrite=randrw --rwmixread=75 --size=4G --filename=/path/to/git-data/testfile ``` This will create a 4GB file in `/path/to/git-data/testfile`. It performs diff --git a/doc/administration/operations/moving_repositories.md b/doc/administration/operations/moving_repositories.md index 960005fe25d..4763c012538 100644 --- a/doc/administration/operations/moving_repositories.md +++ b/doc/administration/operations/moving_repositories.md @@ -9,16 +9,17 @@ We will look at three scenarios: the target directory is empty, the target directory contains an outdated copy of the repositories, and how to deal with thousands of repositories. -**Each of the approaches we list can/will overwrite data in the +DANGER: **Danger:** +Each of the approaches we list can/will overwrite data in the target directory `/mnt/gitlab/repositories`. Do not mix up the -source and the target.** +source and the target. -## Target directory is empty: use a tar pipe +## Target directory is empty: use a `tar` pipe If the target directory `/mnt/gitlab/repositories` is empty the -simplest thing to do is to use a tar pipe. This method has low -overhead and tar is almost always already installed on your system. -However, it is not possible to resume an interrupted tar pipe: if +simplest thing to do is to use a `tar` pipe. This method has low +overhead and `tar` is almost always already installed on your system. +However, it is not possible to resume an interrupted `tar` pipe: if that happens then all data must be copied again. ```shell @@ -28,9 +29,9 @@ sudo -u git sh -c 'tar -C /var/opt/gitlab/git-data/repositories -cf - -- . |\ If you want to see progress, replace `-xf` with `-xvf`. -### Tar pipe to another server +### `tar` pipe to another server -You can also use a tar pipe to copy data to another server. If your +You can also use a `tar` pipe to copy data to another server. If your `git` user has SSH access to the new server as `git@newserver`, you can pipe the data through SSH. @@ -42,13 +43,13 @@ sudo -u git sh -c 'tar -C /var/opt/gitlab/git-data/repositories -cf - -- . |\ If you want to compress the data before it goes over the network (which will cost you CPU cycles) you can replace `ssh` with `ssh -C`. -## The target directory contains an outdated copy of the repositories: use rsync +## The target directory contains an outdated copy of the repositories: use `rsync` If the target directory already contains a partial / outdated copy of the repositories it may be wasteful to copy all the data again -with tar. In this scenario it is better to use rsync. This utility +with `tar`. In this scenario it is better to use `rsync`. This utility is either already installed on your system or easily installable -via apt, yum etc. +via `apt`, `yum`, etc. ```shell sudo -u git sh -c 'rsync -a --delete /var/opt/gitlab/git-data/repositories/. \ @@ -59,30 +60,30 @@ The `/.` in the command above is very important, without it you can easily get the wrong directory structure in the target directory. If you want to see progress, replace `-a` with `-av`. -### Single rsync to another server +### Single `rsync` to another server If the `git` user on your source system has SSH access to the target -server you can send the repositories over the network with rsync. +server you can send the repositories over the network with `rsync`. ```shell sudo -u git sh -c 'rsync -a --delete /var/opt/gitlab/git-data/repositories/. \ git@newserver:/mnt/gitlab/repositories' ``` -## Thousands of Git repositories: use one rsync per repository +## Thousands of Git repositories: use one `rsync` per repository -Every time you start an rsync job it has to inspect all files in +Every time you start an `rsync` job it has to inspect all files in the source directory, all files in the target directory, and then decide what files to copy or not. If the source or target directory -has many contents this startup phase of rsync can become a burden -for your GitLab server. In cases like this you can make rsync's +has many contents this startup phase of `rsync` can become a burden +for your GitLab server. In cases like this you can make `rsync`'s life easier by dividing its work in smaller pieces, and sync one repository at a time. -In addition to rsync we will use [GNU +In addition to `rsync` we will use [GNU Parallel](http://www.gnu.org/software/parallel/). This utility is -not included in GitLab so you need to install it yourself with apt -or yum. Also note that the GitLab scripts we used below were added +not included in GitLab so you need to install it yourself with `apt` +or `yum`. Also note that the GitLab scripts we used below were added in GitLab 8.1. **This process does not clean up repositories at the target location that no @@ -90,9 +91,9 @@ longer exist at the source.** If you start using your GitLab instance with `/mnt/gitlab/repositories`, you need to run `gitlab-rake gitlab:cleanup:repos` after switching to the new repository storage directory. -### Parallel rsync for all repositories known to GitLab +### Parallel `rsync` for all repositories known to GitLab -This will sync repositories with 10 rsync processes at a time. We keep +This will sync repositories with 10 `rsync` processes at a time. We keep track of progress so that the transfer can be restarted if necessary. First we create a new directory, owned by `git`, to hold transfer @@ -147,7 +148,7 @@ cat /home/git/transfer-logs/* | sort | uniq -u |\ ` ``` -### Parallel rsync only for repositories with recent activity +### Parallel `rsync` only for repositories with recent activity Suppose you have already done one sync that started after 2015-10-1 12:00 UTC. Then you might only want to sync repositories that were changed via GitLab diff --git a/doc/administration/operations/puma.md b/doc/administration/operations/puma.md index 62b93d40a6b..e7b4bb88faf 100644 --- a/doc/administration/operations/puma.md +++ b/doc/administration/operations/puma.md @@ -27,7 +27,7 @@ will _not_ carry over automatically, due to differences between the two applicat deployments, see [Configuring Puma Settings](https://docs.gitlab.com/omnibus/settings/puma.html#configuring-puma-settings). For Helm based deployments, see the [Webservice Chart documentation](https://docs.gitlab.com/charts/charts/gitlab/webservice/index.html). -Additionally we strongly recommend that multi-node deployments [configure their load balancers to utilize the readiness check](../high_availability/load_balancer.md#readiness-check) due to a difference between Unicorn and Puma in how they handle connections during a restart of the service. +Additionally we strongly recommend that multi-node deployments [configure their load balancers to utilize the readiness check](../load_balancer.md#readiness-check) due to a difference between Unicorn and Puma in how they handle connections during a restart of the service. ## Performance caveat when using Puma with Rugged diff --git a/doc/administration/operations/sidekiq_memory_killer.md b/doc/administration/operations/sidekiq_memory_killer.md index fdccfacc8a9..e829d735c4f 100644 --- a/doc/administration/operations/sidekiq_memory_killer.md +++ b/doc/administration/operations/sidekiq_memory_killer.md @@ -71,5 +71,5 @@ The MemoryKiller is controlled using environment variables. If the process hard shutdown/restart is not performed by Sidekiq, the Sidekiq process will be forcefully terminated after - `Sidekiq.options[:timeout] * 2` seconds. An external supervision mechanism + `Sidekiq.options[:timeout] + 2` seconds. An external supervision mechanism (e.g. runit) must restart Sidekiq afterwards. diff --git a/doc/administration/packages/container_registry.md b/doc/administration/packages/container_registry.md index 44f1d075a5e..1883f6659e6 100644 --- a/doc/administration/packages/container_registry.md +++ b/doc/administration/packages/container_registry.md @@ -331,6 +331,9 @@ The different supported drivers are: | swift | OpenStack Swift Object Storage | | oss | Aliyun OSS | +NOTE: **Note:** +Although most S3 compatible services (like [MinIO](https://min.io/)) should work with the registry, we only guarantee support for AWS S3. Because we cannot assert the correctness of third-party S3 implementations, we can debug issues, but we cannot patch the registry unless an issue is reproducible against an AWS S3 bucket. + Read more about the individual driver's configuration options in the [Docker Registry docs](https://docs.docker.com/registry/configuration/#storage). @@ -446,27 +449,62 @@ to be in read-only mode for a while. During this time, you can pull from the Container Registry, but you cannot push. 1. Optional: To reduce the amount of data to be migrated, run the [garbage collection tool without downtime](#performing-garbage-collection-without-downtime). -1. Copy initial data to your S3 bucket, for example with the AWS CLI [`cp`](https://awscli.amazonaws.com/v2/documentation/api/latest/reference/s3/cp.html) +1. This example uses the `aws` CLI. If you haven't configured the + CLI before, you have to configure your credentials by running `sudo aws configure`. + Because a non-admin user likely can't access the Container Registry folder, + ensure you use `sudo`. To check your credential configuration, run + [`ls`](https://awscli.amazonaws.com/v2/documentation/api/latest/reference/s3/ls.html) to list + all buckets. + + ```shell + sudo aws --endpoint-url https://your-object-storage-backend.com s3 ls + ``` + + If you are using AWS as your back end, you do not need the [`--endpoint-url`](https://docs.aws.amazon.com/cli/latest/reference/#options). +1. Copy initial data to your S3 bucket, for example with the `aws` CLI + [`cp`](https://awscli.amazonaws.com/v2/documentation/api/latest/reference/s3/cp.html) or [`sync`](https://awscli.amazonaws.com/v2/documentation/api/latest/reference/s3/sync.html) command. Make sure to keep the `docker` folder as the top-level folder inside the bucket. ```shell - aws s3 sync registry s3://mybucket + sudo aws --endpoint-url https://your-object-storage-backend.com s3 sync registry s3://mybucket ``` + TIP: **Tip:** + If you have a lot of data, you may be able to improve performance by + [running parallel sync operations](https://aws.amazon.com/premiumsupport/knowledge-center/s3-improve-transfer-sync-command/). + 1. To perform the final data sync, [put the Container Registry in `read-only` mode](#performing-garbage-collection-without-downtime) and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). 1. Sync any changes since the initial data load to your S3 bucket and delete files that exist in the destination bucket but not in the source: ```shell - aws s3 sync registry s3://mybucket --delete + sudo aws --endpoint-url https://your-object-storage-backend.com s3 sync registry s3://mybucket --delete --dryrun ``` + After verifying the command is going to perform as expected, remove the + [`--dryrun`](https://docs.aws.amazon.com/cli/latest/reference/s3/sync.html) + flag and run the command. + DANGER: **Danger:** - The `--delete` flag will delete files that exist in the destination but not in the source. + The [`--delete`](https://docs.aws.amazon.com/cli/latest/reference/s3/sync.html) + flag will delete files that exist in the destination but not in the source. Make sure not to swap the source and destination, or you will delete all data in the Registry. +1. Verify all Container Registry files have been uploaded to object storage + by looking at the file count returned by these two commands: + + ```shell + sudo find registry -type f | wc -l + ``` + + ```shell + sudo aws --endpoint-url https://your-object-storage-backend.com s3 ls s3://mybucket --recursive | wc -l + ``` + + The output of these commands should match, except for the content in the + `_uploads` directories and sub-directories. 1. Configure your registry to [use the S3 bucket for storage](#use-object-storage). 1. For the changes to take effect, set the Registry back to `read-write` mode and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). @@ -693,6 +731,35 @@ notifications: backoff: 1000 ``` +## Run the Cleanup policy now + +To reduce the amount of [Container Registry disk space used by a given project](../troubleshooting/gitlab_rails_cheat_sheet.md#registry-disk-space-usage-by-project), +administrators can clean up image tags +and [run garbage collection](#container-registry-garbage-collection). + +To remove image tags by running the cleanup policy, run the following commands in the +[GitLab Rails console](../troubleshooting/navigating_gitlab_via_rails_console.md): + +```ruby +# Numeric ID of the project whose container registry should be cleaned up +P = <project_id> + +# Numeric ID of a developer, maintainer or owner in that project +U = <user_id> + +# Get required details / objects +user = User.find_by_id(U) +project = Project.find_by_id(P) +repo = ContainerRepository.find_by(project_id: P) +policy = ContainerExpirationPolicy.find_by(project_id: P) + +# Start the tag cleanup +Projects::ContainerRepository::CleanupTagsService.new(project, user, policy.attributes.except("created_at", "updated_at")).execute(repo) +``` + +NOTE: **Note:** +You can also [run cleanup on a schedule](../../user/packages/container_registry/index.md#cleanup-policy). + ## Container Registry garbage collection NOTE: **Note:** diff --git a/doc/administration/packages/index.md b/doc/administration/packages/index.md index 5d07136ef40..3af1f0c933b 100644 --- a/doc/administration/packages/index.md +++ b/doc/administration/packages/index.md @@ -4,7 +4,7 @@ group: Package info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# GitLab Package Registry administration **(PREMIUM ONLY)** +# GitLab Package Registry administration GitLab Packages allows organizations to utilize GitLab as a private repository for a variety of common package managers. Users are able to build and publish diff --git a/doc/administration/pages/index.md b/doc/administration/pages/index.md index 4efd92eaa07..9e2aa602767 100644 --- a/doc/administration/pages/index.md +++ b/doc/administration/pages/index.md @@ -114,7 +114,7 @@ since that is needed in all configurations. --- -URL scheme: `http://page.example.io` +URL scheme: `http://<namespace>.example.io/<project_slug>` This is the minimum setup that you can use Pages with. It is the base for all other setups as described below. NGINX will proxy all requests to the daemon. @@ -139,7 +139,7 @@ Watch the [video tutorial](https://youtu.be/dD8c7WNcc6s) for this configuration. --- -URL scheme: `https://page.example.io` +URL scheme: `https://<namespace>.example.io/<project_slug>` NGINX will proxy all requests to the daemon. Pages daemon doesn't listen to the outside world. @@ -204,6 +204,7 @@ control over how the Pages daemon runs and serves content in your environment. | `external_https` | Configure Pages to bind to one or more secondary IP addresses, serving HTTPS requests. Multiple addresses can be given as an array, along with exact ports, for example `['1.2.3.4', '1.2.3.5:8063']`. Sets value for `listen_https`. | `gitlab_client_http_timeout` | GitLab API HTTP client connection timeout in seconds (default: 10s). | `gitlab_client_jwt_expiry` | JWT Token expiry time in seconds (default: 30s). +| `domain_config_source` | Domain configuration source (default: `disk`) | `gitlab_id` | The OAuth application public ID. Leave blank to automatically fill when Pages authenticates with GitLab. | `gitlab_secret` | The OAuth application secret. Leave blank to automatically fill when Pages authenticates with GitLab. | `gitlab_server` | Server to use for authentication when access control is enabled; defaults to GitLab `external_url`. @@ -254,7 +255,7 @@ you have IPv6 as well as IPv4 addresses, you can use them both. --- -URL scheme: `http://page.example.io` and `http://domain.com` +URL scheme: `http://<namespace>.example.io/<project_slug>` and `http://custom-domain.com` In that case, the Pages daemon is running, NGINX still proxies requests to the daemon but the daemon is also able to receive requests from the outside @@ -285,7 +286,7 @@ world. Custom domains are supported, but no TLS. --- -URL scheme: `https://page.example.io` and `https://domain.com` +URL scheme: `https://<namespace>.example.io/<project_slug>` and `https://custom-domain.com` In that case, the Pages daemon is running, NGINX still proxies requests to the daemon but the daemon is also able to receive requests from the outside @@ -388,9 +389,9 @@ To do that: 1. Click **Save changes**. CAUTION: **Warning:** -This action will not make all currently public web-sites private until they redeployed. -This issue among others will be resolved by -[changing GitLab Pages configuration mechanism](https://gitlab.com/gitlab-org/gitlab-pages/-/issues/282). +For self-managed installations, all public websites remain private until they are +redeployed. This issue will be resolved by +[sourcing domain configuration from the GitLab API](https://gitlab.com/gitlab-org/gitlab/-/issues/218357). ### Running behind a proxy @@ -409,7 +410,7 @@ pages: ### Using a custom Certificate Authority (CA) NOTE: **Note:** -[Before 13.2](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/4289), when using Omnibus, a [workaround was required](https://docs.gitlab.com/13.1/ee/administration/pages/index.html#using-a-custom-certificate-authority-ca). +[Before 13.3](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/4411), when using Omnibus, a [workaround was required](https://docs.gitlab.com/13.1/ee/administration/pages/index.html#using-a-custom-certificate-authority-ca). When using certificates issued by a custom CA, [Access Control](../../user/project/pages/pages_access_control.md#gitlab-pages-access-control) and the [online view of HTML job artifacts](../../ci/pipelines/job_artifacts.md#browsing-artifacts) @@ -536,7 +537,7 @@ database encryption. Proceed with caution. 1. Set up a new server. This will become the **Pages server**. -1. Create an [NFS share](../high_availability/nfs_host_client_setup.md) +1. Create an [NFS share](../nfs.md) on the **Pages server** and configure this share to allow access from your main **GitLab server**. Note that the example there is more general and @@ -601,6 +602,43 @@ configuring a load balancer to work at the IP level, and so on. If you wish to set up GitLab Pages on multiple servers, perform the above procedure for each Pages server. +## Domain source configuration + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/217912) in GitLab 13.3. + +GitLab Pages can use different sources to get domain configuration. +The default value is `nil`; however, GitLab Pages will default to `disk`. + + ```ruby + gitlab_pages['domain_config_source'] = nil + ``` + +You can specify `gitlab` to enable [API-based configuration](#gitlab-api-based-configuration). + +For more details see this [blog post](https://about.gitlab.com/blog/2020/08/03/how-gitlab-pages-uses-the-gitlab-api-to-serve-content/). + +### GitLab API-based configuration + +GitLab Pages can use an API-based configuration. This replaces disk source configuration, which +was used prior to GitLab 13.0. Follow these steps to enable it: + +1. Add the following to your `/etc/gitlab/gitlab.erb` file: + + ```ruby + gitlab_pages['domain_config_source'] = "gitlab" + ``` + +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + +If you encounter an issue, you can disable it by choosing `disk` or `nil`: + +```ruby +gitlab_pages['domain_config_source'] = nil +``` + +For other common issues, see the [troubleshooting section](#failed-to-connect-to-the-internal-gitlab-api) +or report an issue. + ## Backup GitLab Pages are part of the [regular backup](../../raketasks/backup_restore.md), so there is no separate backup to configure. @@ -696,3 +734,24 @@ date > /var/opt/gitlab/gitlab-rails/shared/pages/.update ``` If you've customized the Pages storage path, adjust the command above to use your custom path. + +### Failed to connect to the internal GitLab API + +If you have enabled [API-based configuration](#gitlab-api-based-configuration) and see the following error: + +```plaintext +ERRO[0010] Failed to connect to the internal GitLab API after 0.50s error="failed to connect to internal Pages API: HTTP status: 401" +``` + +If you are [Running GitLab Pages on a separate server](#running-gitlab-pages-on-a-separate-server) +you must copy the `/etc/gitlab/gitlab-secrets.json` file +from the **GitLab server** to the **Pages server** after upgrading to GitLab 13.3, +as described in that section. + +Other reasons may include network connectivity issues between your +**GitLab server** and your **Pages server** such as firewall configurations or closed ports. +For example, if there is a connection timeout: + +```plaintext +error="failed to connect to internal Pages API: Get \"https://gitlab.example.com:3000/api/v4/internal/pages/status\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)" +``` diff --git a/doc/administration/pages/source.md b/doc/administration/pages/source.md index d5b49bdf839..486bc7a8777 100644 --- a/doc/administration/pages/source.md +++ b/doc/administration/pages/source.md @@ -6,7 +6,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w # GitLab Pages administration for source installations ->**Note:** +NOTE: **Note:** Before attempting to enable GitLab Pages, first make sure you have [installed GitLab](../../install/installation.md) successfully. @@ -77,7 +77,7 @@ host that GitLab runs. For example, an entry would look like this: where `example.io` is the domain under which GitLab Pages will be served and `192.0.2.1` is the IP address of your GitLab instance. -> **Note:** +NOTE: **Note:** You should not use the GitLab domain to serve user pages. For more information see the [security section](#security). @@ -94,7 +94,7 @@ since that is needed in all configurations. - [Wildcard DNS setup](#dns-configuration) -URL scheme: `http://page.example.io` +URL scheme: `http://<namespace>.example.io/<project_slug>` This is the minimum setup that you can use Pages with. It is the base for all other setups as described below. NGINX will proxy all requests to the daemon. @@ -157,7 +157,7 @@ The Pages daemon doesn't listen to the outside world. - [Wildcard DNS setup](#dns-configuration) - Wildcard TLS certificate -URL scheme: `https://page.example.io` +URL scheme: `https://<namespace>.example.io/<project_slug>` NGINX will proxy all requests to the daemon. Pages daemon doesn't listen to the outside world. @@ -221,7 +221,7 @@ that without TLS certificates. - [Wildcard DNS setup](#dns-configuration) - Secondary IP -URL scheme: `http://page.example.io` and `http://domain.com` +URL scheme: `http://<namespace>.example.io/<project_slug>` and `http://custom-domain.com` In that case, the pages daemon is running, NGINX still proxies requests to the daemon but the daemon is also able to receive requests from the outside @@ -286,7 +286,7 @@ world. Custom domains are supported, but no TLS. - Wildcard TLS certificate - Secondary IP -URL scheme: `https://page.example.io` and `https://domain.com` +URL scheme: `https://<namespace>.example.io/<project_slug>` and `https://custom-domain.com` In that case, the pages daemon is running, NGINX still proxies requests to the daemon but the daemon is also able to receive requests from the outside @@ -349,7 +349,7 @@ world. Custom domains and TLS are supported. ## NGINX caveats ->**Note:** +NOTE: **Note:** The following information applies only for installations from source. Be extra careful when setting up the domain name in the NGINX configuration. You must diff --git a/doc/administration/postgresql/external.md b/doc/administration/postgresql/external.md index 6e2bbc0aae1..e2cfb95ec48 100644 --- a/doc/administration/postgresql/external.md +++ b/doc/administration/postgresql/external.md @@ -32,7 +32,7 @@ If you use a cloud-managed service, or provide your own PostgreSQL instance: gitlab_rails['db_password'] = 'DB password' ``` - For more information on GitLab HA setups, refer to [configuring GitLab for HA](../high_availability/gitlab.md). + For more information on GitLab multi-node setups, refer to the [reference architectures](../reference_architectures/index.md). 1. Reconfigure for the changes to take effect: diff --git a/doc/administration/postgresql/index.md b/doc/administration/postgresql/index.md index 7e0a2f3cae1..2720d8e696b 100644 --- a/doc/administration/postgresql/index.md +++ b/doc/administration/postgresql/index.md @@ -13,7 +13,7 @@ There are essentially three setups to choose from. This setup is for when you have installed GitLab using the [Omnibus GitLab **Enterprise Edition** (EE) package](https://about.gitlab.com/install/?version=ee). -All the tools that are needed like PostgreSQL, PgBouncer, Repmgr are bundled in +All the tools that are needed like PostgreSQL, PgBouncer, Patroni, and repmgr are bundled in the package, so you can it to set up the whole PostgreSQL infrastructure (primary, replica). [> Read how to set up PostgreSQL replication and failover using Omnibus GitLab](replication_and_failover.md) diff --git a/doc/administration/postgresql/pgbouncer.md b/doc/administration/postgresql/pgbouncer.md new file mode 100644 index 00000000000..9db3e017359 --- /dev/null +++ b/doc/administration/postgresql/pgbouncer.md @@ -0,0 +1,168 @@ +--- +type: reference +--- + +# Working with the bundled PgBouncer service **(PREMIUM ONLY)** + +[PgBouncer](http://www.pgbouncer.org/) is used to seamlessly migrate database +connections between servers in a failover scenario. Additionally, it can be used +in a non-fault-tolerant setup to pool connections, speeding up response time +while reducing resource usage. + +GitLab Premium includes a bundled version of PgBouncer that can be managed +through `/etc/gitlab/gitlab.rb`. + +## PgBouncer as part of a fault-tolerant GitLab installation + +This content has been moved to a [new location](replication_and_failover.md#configuring-the-pgbouncer-node). + +## PgBouncer as part of a non-fault-tolerant GitLab installation + +1. Generate PGBOUNCER_USER_PASSWORD_HASH with the command `gitlab-ctl pg-password-md5 pgbouncer` + +1. Generate SQL_USER_PASSWORD_HASH with the command `gitlab-ctl pg-password-md5 gitlab`. We'll also need to enter the plaintext SQL_USER_PASSWORD later + +1. On your database node, ensure the following is set in your `/etc/gitlab/gitlab.rb` + + ```ruby + postgresql['pgbouncer_user_password'] = 'PGBOUNCER_USER_PASSWORD_HASH' + postgresql['sql_user_password'] = 'SQL_USER_PASSWORD_HASH' + postgresql['listen_address'] = 'XX.XX.XX.Y' # Where XX.XX.XX.Y is the ip address on the node postgresql should listen on + postgresql['md5_auth_cidr_addresses'] = %w(AA.AA.AA.B/32) # Where AA.AA.AA.B is the IP address of the pgbouncer node + ``` + +1. Run `gitlab-ctl reconfigure` + + NOTE: **Note:** + If the database was already running, it will need to be restarted after reconfigure by running `gitlab-ctl restart postgresql`. + +1. On the node you are running PgBouncer on, make sure the following is set in `/etc/gitlab/gitlab.rb` + + ```ruby + pgbouncer['enable'] = true + pgbouncer['databases'] = { + gitlabhq_production: { + host: 'DATABASE_HOST', + user: 'pgbouncer', + password: 'PGBOUNCER_USER_PASSWORD_HASH' + } + } + ``` + +1. Run `gitlab-ctl reconfigure` + +1. On the node running Puma, make sure the following is set in `/etc/gitlab/gitlab.rb` + + ```ruby + gitlab_rails['db_host'] = 'PGBOUNCER_HOST' + gitlab_rails['db_port'] = '6432' + gitlab_rails['db_password'] = 'SQL_USER_PASSWORD' + ``` + +1. Run `gitlab-ctl reconfigure` + +1. At this point, your instance should connect to the database through PgBouncer. If you are having issues, see the [Troubleshooting](#troubleshooting) section + +## Enable Monitoring + +> [Introduced](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/3786) in GitLab 12.0. + +If you enable Monitoring, it must be enabled on **all** PgBouncer servers. + +1. Create/edit `/etc/gitlab/gitlab.rb` and add the following configuration: + + ```ruby + # Enable service discovery for Prometheus + consul['enable'] = true + consul['monitoring_service_discovery'] = true + + # Replace placeholders + # Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z + # with the addresses of the Consul server nodes + consul['configuration'] = { + retry_join: %w(Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z), + } + + # Set the network addresses that the exporters will listen on + node_exporter['listen_address'] = '0.0.0.0:9100' + pgbouncer_exporter['listen_address'] = '0.0.0.0:9188' + ``` + +1. Run `sudo gitlab-ctl reconfigure` to compile the configuration. + +## Administrative console + +As part of Omnibus GitLab, a command is provided to automatically connect to the +PgBouncer administrative console. See the +[PgBouncer documentation](https://www.pgbouncer.org/usage.html#admin-console) +for detailed instructions on how to interact with the console. + +To start a session run the following and provide the password for the `pgbouncer` +user: + +```shell +sudo gitlab-ctl pgb-console +``` + +To get some basic information about the instance: + +```shell +pgbouncer=# show databases; show clients; show servers; + name | host | port | database | force_user | pool_size | reserve_pool | pool_mode | max_connections | current_connections +---------------------+-----------+------+---------------------+------------+-----------+--------------+-----------+-----------------+--------------------- + gitlabhq_production | 127.0.0.1 | 5432 | gitlabhq_production | | 100 | 5 | | 0 | 1 + pgbouncer | | 6432 | pgbouncer | pgbouncer | 2 | 0 | statement | 0 | 0 +(2 rows) + + type | user | database | state | addr | port | local_addr | local_port | connect_time | request_time | ptr | link +| remote_pid | tls +------+-----------+---------------------+--------+-----------+-------+------------+------------+---------------------+---------------------+-----------+------ ++------------+----- + C | gitlab | gitlabhq_production | active | 127.0.0.1 | 44590 | 127.0.0.1 | 6432 | 2018-04-24 22:13:10 | 2018-04-24 22:17:10 | 0x12444c0 | +| 0 | + C | gitlab | gitlabhq_production | active | 127.0.0.1 | 44592 | 127.0.0.1 | 6432 | 2018-04-24 22:13:10 | 2018-04-24 22:17:10 | 0x12447c0 | +| 0 | + C | gitlab | gitlabhq_production | active | 127.0.0.1 | 44594 | 127.0.0.1 | 6432 | 2018-04-24 22:13:10 | 2018-04-24 22:17:10 | 0x1244940 | +| 0 | + C | gitlab | gitlabhq_production | active | 127.0.0.1 | 44706 | 127.0.0.1 | 6432 | 2018-04-24 22:14:22 | 2018-04-24 22:16:31 | 0x1244ac0 | +| 0 | + C | gitlab | gitlabhq_production | active | 127.0.0.1 | 44708 | 127.0.0.1 | 6432 | 2018-04-24 22:14:22 | 2018-04-24 22:15:15 | 0x1244c40 | +| 0 | + C | gitlab | gitlabhq_production | active | 127.0.0.1 | 44794 | 127.0.0.1 | 6432 | 2018-04-24 22:15:15 | 2018-04-24 22:15:15 | 0x1244dc0 | +| 0 | + C | gitlab | gitlabhq_production | active | 127.0.0.1 | 44798 | 127.0.0.1 | 6432 | 2018-04-24 22:15:15 | 2018-04-24 22:16:31 | 0x1244f40 | +| 0 | + C | pgbouncer | pgbouncer | active | 127.0.0.1 | 44660 | 127.0.0.1 | 6432 | 2018-04-24 22:13:51 | 2018-04-24 22:17:12 | 0x1244640 | +| 0 | +(8 rows) + + type | user | database | state | addr | port | local_addr | local_port | connect_time | request_time | ptr | link | rem +ote_pid | tls +------+--------+---------------------+-------+-----------+------+------------+------------+---------------------+---------------------+-----------+------+---- +--------+----- + S | gitlab | gitlabhq_production | idle | 127.0.0.1 | 5432 | 127.0.0.1 | 35646 | 2018-04-24 22:15:15 | 2018-04-24 22:17:10 | 0x124dca0 | | + 19980 | +(1 row) +``` + +## Troubleshooting + +In case you are experiencing any issues connecting through PgBouncer, the first +place to check is always the logs: + +```shell +sudo gitlab-ctl tail pgbouncer +``` + +Additionally, you can check the output from `show databases` in the +[administrative console](#administrative-console). In the output, you would expect +to see values in the `host` field for the `gitlabhq_production` database. +Additionally, `current_connections` should be greater than 1. + +### Message: `LOG: invalid CIDR mask in address` + +See the suggested fix [in Geo documentation](../geo/replication/troubleshooting.md#message-log--invalid-cidr-mask-in-address). + +### Message: `LOG: invalid IP mask "md5": Name or service not known` + +See the suggested fix [in Geo documentation](../geo/replication/troubleshooting.md#message-log--invalid-ip-mask-md5-name-or-service-not-known). diff --git a/doc/administration/postgresql/replication_and_failover.md b/doc/administration/postgresql/replication_and_failover.md index 5f550f09e5b..bc2af167e6c 100644 --- a/doc/administration/postgresql/replication_and_failover.md +++ b/doc/administration/postgresql/replication_and_failover.md @@ -29,6 +29,11 @@ You also need to take into consideration the underlying network topology, making sure you have redundant connectivity between all Database and GitLab instances to avoid the network becoming a single point of failure. +NOTE: **Note:** +As of GitLab 13.3, PostgreSQL 12 is shipped with Omnibus GitLab. Clustering for PostgreSQL 12 is only supported with +Patroni. See the [Patroni](#patroni) section for further details. The support for repmgr will not be extended beyond +PostgreSQL 11. + ### Database node Each database node runs three services: @@ -97,7 +102,7 @@ This is why you will need: When using default setup, minimum configuration requires: -- `CONSUL_USERNAME`. Defaults to `gitlab-consul` +- `CONSUL_USERNAME`. The default user for Omnibus GitLab is `gitlab-consul` - `CONSUL_DATABASE_PASSWORD`. Password for the database user. - `CONSUL_PASSWORD_HASH`. This is a hash generated out of Consul username/password pair. Can be generated with: @@ -140,7 +145,7 @@ server nodes. We will need the following password information for the application's database user: -- `POSTGRESQL_USERNAME`. Defaults to `gitlab` +- `POSTGRESQL_USERNAME`. The default user for Omnibus GitLab is `gitlab` - `POSTGRESQL_USER_PASSWORD`. The password for the database user - `POSTGRESQL_PASSWORD_HASH`. This is a hash generated out of the username/password pair. Can be generated with: @@ -153,7 +158,7 @@ We will need the following password information for the application's database u When using default setup, minimum configuration requires: -- `PGBOUNCER_USERNAME`. Defaults to `pgbouncer` +- `PGBOUNCER_USERNAME`. The default user for Omnibus GitLab is `pgbouncer` - `PGBOUNCER_PASSWORD`. This is a password for PgBouncer service. - `PGBOUNCER_PASSWORD_HASH`. This is a hash generated out of PgBouncer username/password pair. Can be generated with: @@ -198,7 +203,7 @@ When installing the GitLab package, do not supply `EXTERNAL_URL` value. ### Configuring the Database nodes -1. Make sure to [configure the Consul nodes](../high_availability/consul.md). +1. Make sure to [configure the Consul nodes](../consul.md). 1. Make sure you collect [`CONSUL_SERVER_NODES`](#consul-information), [`PGBOUNCER_PASSWORD_HASH`](#pgbouncer-information), [`POSTGRESQL_PASSWORD_HASH`](#postgresql-information), the [number of db nodes](#postgresql-information), and the [network address](#network-information) before executing the next step. 1. On the master database node, edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: @@ -305,6 +310,12 @@ Select one node as a primary node. CREATE EXTENSION pg_trgm; ``` +1. Enable the `btree_gist` extension: + + ```shell + CREATE EXTENSION btree_gist; + ``` + 1. Exit the database prompt by typing `\q` and Enter. 1. Verify the cluster is initialized with one node: @@ -455,7 +466,7 @@ Check the [Troubleshooting section](#troubleshooting) before proceeding. gitlab-ctl write-pgpass --host 127.0.0.1 --database pgbouncer --user pgbouncer --hostuser gitlab-consul ``` -1. [Enable monitoring](../high_availability/pgbouncer.md#enable-monitoring) +1. [Enable monitoring](../postgresql/pgbouncer.md#enable-monitoring) #### PgBouncer Checkpoint @@ -736,9 +747,9 @@ consul['configuration'] = { After deploying the configuration follow these steps: -1. On `10.6.0.31`, our primary database +1. On `10.6.0.31`, our primary database: - Enable the `pg_trgm` extension + Enable the `pg_trgm` and `btree_gist` extensions: ```shell gitlab-psql -d gitlabhq_production @@ -746,33 +757,34 @@ After deploying the configuration follow these steps: ```shell CREATE EXTENSION pg_trgm; + CREATE EXTENSION btree_gist; ``` -1. On `10.6.0.32`, our first standby database +1. On `10.6.0.32`, our first standby database: - Make this node a standby of the primary + Make this node a standby of the primary: ```shell gitlab-ctl repmgr standby setup 10.6.0.21 ``` -1. On `10.6.0.33`, our second standby database +1. On `10.6.0.33`, our second standby database: - Make this node a standby of the primary + Make this node a standby of the primary: ```shell gitlab-ctl repmgr standby setup 10.6.0.21 ``` -1. On `10.6.0.41`, our application server +1. On `10.6.0.41`, our application server: - Set `gitlab-consul` user's PgBouncer password to `toomanysecrets` + Set `gitlab-consul` user's PgBouncer password to `toomanysecrets`: ```shell gitlab-ctl write-pgpass --host 127.0.0.1 --database pgbouncer --user pgbouncer --hostuser gitlab-consul ``` - Run database migrations + Run database migrations: ```shell gitlab-rake gitlab:db:configure @@ -783,7 +795,7 @@ After deploying the configuration follow these steps: This example uses 3 PostgreSQL servers, and 1 application node (with PgBouncer setup alongside). It differs from the [recommended setup](#example-recommended-setup) by moving the Consul servers into the same servers we use for PostgreSQL. -The trade-off is between reducing server counts, against the increased operational complexity of needing to deal with PostgreSQL [failover](#failover-procedure) and [restore](#restore-procedure) procedures in addition to [Consul outage recovery](../high_availability/consul.md#outage-recovery) on the same set of machines. +The trade-off is between reducing server counts, against the increased operational complexity of needing to deal with PostgreSQL [failover](#failover-procedure) and [restore](#restore-procedure) procedures in addition to [Consul outage recovery](../consul.md#outage-recovery) on the same set of machines. In this example we start with all servers on the same 10.6.0.0/16 private network range, they can connect to each freely other on those addresses. @@ -1075,7 +1087,7 @@ To restart either service, run `gitlab-ctl restart SERVICE` For PostgreSQL, it is usually safe to restart the master node by default. Automatic failover defaults to a 1 minute timeout. Provided the database returns before then, nothing else needs to be done. To be safe, you can stop `repmgrd` on the standby nodes first with `gitlab-ctl stop repmgrd`, then start afterwards with `gitlab-ctl start repmgrd`. -On the Consul server nodes, it is important to restart the Consul service in a controlled fashion. Read our [Consul documentation](../high_availability/consul.md#restarting-the-server-cluster) for instructions on how to restart the service. +On the Consul server nodes, it is important to [restart the Consul service](../consul.md#restart-consul) in a controlled manner. ### `gitlab-ctl repmgr-check-master` command produces errors @@ -1122,11 +1134,10 @@ postgresql['trust_auth_cidr_addresses'] = %w(123.123.123.123/32 <other_cidrs>) ### Issues with other components -If you're running into an issue with a component not outlined here, be sure to check the troubleshooting section of their specific documentation page. +If you're running into an issue with a component not outlined here, be sure to check the troubleshooting section of their specific documentation page: -- [Consul](../high_availability/consul.md#troubleshooting) +- [Consul](../consul.md#troubleshooting-consul) - [PostgreSQL](https://docs.gitlab.com/omnibus/settings/database.html#troubleshooting) -- [GitLab application](../high_availability/gitlab.md#troubleshooting) ## Patroni @@ -1324,7 +1335,7 @@ You can switch an exiting database cluster to use Patroni instead of repmgr with NOTE: **Note:** Ensure that there is no `walsender` process running on the primary node. - `ps aux | grep walsender` must not show any running process. + `ps aux | grep walsender` must not show any running process. 1. On the primary node, [configure Patroni](#configuring-patroni-cluster). Remove `repmgr` and any other repmgr-specific configuration. Also remove any configuration that is related to PostgreSQL replication. diff --git a/doc/administration/raketasks/check.md b/doc/administration/raketasks/check.md index da5caf3966f..15014fffd01 100644 --- a/doc/administration/raketasks/check.md +++ b/doc/administration/raketasks/check.md @@ -135,3 +135,22 @@ The LDAP check Rake task will test the bind DN and password credentials (if configured) and will list a sample of LDAP users. This task is also executed as part of the `gitlab:check` task, but can run independently. See [LDAP Rake Tasks - LDAP Check](ldap.md#check) for details. + +## Troubleshooting + +The following are solutions to problems you might discover using the Rake tasks documented +above. + +### Dangling commits + +`gitlab:git:fsck` can find dangling commits. To fix them, try +[manually triggering housekeeping](../housekeeping.md#manual-housekeeping) +for the affected project(s). + +If the issue persists, try triggering `gc` via the +[Rails Console](../troubleshooting/navigating_gitlab_via_rails_console.md#starting-a-rails-console-session): + +```ruby +p = Project.find_by_path("project-name") +Projects::HousekeepingService.new(p, :gc).execute +``` diff --git a/doc/administration/raketasks/maintenance.md b/doc/administration/raketasks/maintenance.md index 19781b6a5db..553afba78e3 100644 --- a/doc/administration/raketasks/maintenance.md +++ b/doc/administration/raketasks/maintenance.md @@ -23,32 +23,39 @@ Example output: ```plaintext System information -System: Debian 7.8 -Current User: git -Using RVM: no -Ruby Version: 2.1.5p273 -Gem Version: 2.4.3 -Bundler Version: 1.7.6 -Rake Version: 10.3.2 -Redis Version: 3.2.5 -Sidekiq Version: 2.17.8 +System: Ubuntu 20.04 +Proxy: no +Current User: git +Using RVM: no +Ruby Version: 2.6.6p146 +Gem Version: 2.7.10 +Bundler Version:1.17.3 +Rake Version: 12.3.3 +Redis Version: 5.0.9 +Git Version: 2.27.0 +Sidekiq Version:5.2.9 +Go Version: unknown GitLab information -Version: 7.7.1 -Revision: 41ab9e1 -Directory: /home/git/gitlab -DB Adapter: postgresql -URL: https://gitlab.example.com -HTTP Clone URL: https://gitlab.example.com/some-project.git -SSH Clone URL: git@gitlab.example.com:some-project.git -Using LDAP: no -Using Omniauth: no +Version: 13.2.2-ee +Revision: 618883a1f9d +Directory: /opt/gitlab/embedded/service/gitlab-rails +DB Adapter: PostgreSQL +DB Version: 11.7 +URL: http://gitlab.example.com +HTTP Clone URL: http://gitlab.example.com/some-group/some-project.git +SSH Clone URL: git@gitlab.example.com:some-group/some-project.git +Elasticsearch: no +Geo: no +Using LDAP: no +Using Omniauth: yes +Omniauth Providers: GitLab Shell -Version: 2.4.1 -Repositories: /home/git/repositories/ -Hooks: /home/git/gitlab-shell/hooks/ -Git: /usr/bin/git +Version: 13.3.0 +Repository storage paths: +- default: /var/opt/gitlab/git-data/repositories +GitLab Shell path: /opt/gitlab/embedded/service/gitlab-shell ``` ## Show GitLab license information **(STARTER ONLY)** diff --git a/doc/administration/raketasks/project_import_export.md b/doc/administration/raketasks/project_import_export.md index b807e03b01f..51e0e2e46b6 100644 --- a/doc/administration/raketasks/project_import_export.md +++ b/doc/administration/raketasks/project_import_export.md @@ -46,6 +46,6 @@ Note the following: compatible as described in the [Version history](../../user/project/settings/import_export.md#version-history). - The project import option must be enabled in application settings (`/admin/application_settings/general`) under **Import sources**, which is available - under **{admin}** **Admin Area >** **{settings}** **Settings > Visibility and access controls**. + under **Admin Area > Settings > Visibility and access controls**. - The exports are stored in a temporary [shared directory](../../development/shared_files.md) and are deleted every 24 hours by a specific worker. diff --git a/doc/administration/raketasks/storage.md b/doc/administration/raketasks/storage.md index 74fd2c2ebb6..a97bff83290 100644 --- a/doc/administration/raketasks/storage.md +++ b/doc/administration/raketasks/storage.md @@ -100,7 +100,7 @@ to project IDs 50 to 100 in an Omnibus GitLab installation: sudo gitlab-rake gitlab:storage:migrate_to_hashed ID_FROM=50 ID_TO=100 ``` -You can monitor the progress in the **{admin}** **Admin Area > Monitoring > Background Jobs** page. +You can monitor the progress in the **Admin Area > Monitoring > Background Jobs** page. There is a specific queue you can watch to see how long it will take to finish: `hashed_storage:hashed_storage_project_migrate`. @@ -150,7 +150,7 @@ to project IDs 50 to 100 in an Omnibus GitLab installation: sudo gitlab-rake gitlab:storage:rollback_to_legacy ID_FROM=50 ID_TO=100 ``` -You can monitor the progress in the **{admin}** **Admin Area > Monitoring > Background Jobs** page. +You can monitor the progress in the **Admin Area > Monitoring > Background Jobs** page. On the **Queues** tab, you can watch the `hashed_storage:hashed_storage_project_rollback` queue to see how long the process will take to finish. After it reaches zero, you can confirm every project has been rolled back by running the commands bellow. diff --git a/doc/administration/raketasks/uploads/migrate.md b/doc/administration/raketasks/uploads/migrate.md index b164c4e744d..8c020e91a15 100644 --- a/doc/administration/raketasks/uploads/migrate.md +++ b/doc/administration/raketasks/uploads/migrate.md @@ -141,7 +141,15 @@ keeping in mind the task name in this case is `gitlab:uploads:migrate_to_local`. To migrate uploads from object storage to local storage: -1. Disable both `direct_upload` and `background_upload` under `uploads` settings in `gitlab.rb`. +1. Disable both `direct_upload` and `background_upload` under `uploads` settings in `gitlab.rb`: + + ```ruby + gitlab_rails['uploads_object_store_direct_upload'] = false + gitlab_rails['uploads_object_store_background_upload'] = false + ``` + + Save the file and [reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure). + 1. Run the Rake task: **Omnibus Installation** diff --git a/doc/administration/redis/replication_and_failover.md b/doc/administration/redis/replication_and_failover.md index ac31b909c89..ca041adb1d8 100644 --- a/doc/administration/redis/replication_and_failover.md +++ b/doc/administration/redis/replication_and_failover.md @@ -466,7 +466,7 @@ While it doesn't require a list of all Sentinel nodes, in case of a failure, it needs to access at least one of the listed. NOTE: **Note:** -The following steps should be performed in the [GitLab application server](../high_availability/gitlab.md) +The following steps should be performed in the GitLab application server which ideally should not have Redis or Sentinels on it for a HA setup. 1. SSH into the server where the GitLab application is installed. @@ -637,47 +637,63 @@ lives easier. If you want to know what happens underneath keep reading. ### Running multiple Redis clusters -GitLab supports running [separate Redis clusters for different persistent -classes](https://docs.gitlab.com/omnibus/settings/redis.html#running-with-multiple-redis-instances): -cache, queues, and shared_state. To make this work with Sentinel: +Omnibus GitLab supports running separate Redis and Sentinel instances for different +persistence classes. -1. Set the appropriate variable in `/etc/gitlab/gitlab.rb` for each instance you are using: +| Class | Purpose | +| -------------- | ------------------------------------------------ | +| `cache` | Store cached data. | +| `queues` | Store Sidekiq background jobs. | +| `shared_state` | Store session-related and other persistent data. | +| `actioncable` | Pub/Sub queue backend for ActionCable. | + +To make this work with Sentinel: + +1. [Configure the different Redis/Sentinels](#configuring-redis) instances based on your needs. +1. For each Rails application instance, edit its `/etc/gitlab/gitlab.rb` file: ```ruby gitlab_rails['redis_cache_instance'] = REDIS_CACHE_URL gitlab_rails['redis_queues_instance'] = REDIS_QUEUES_URL gitlab_rails['redis_shared_state_instance'] = REDIS_SHARED_STATE_URL - ``` - - **Note**: Redis URLs should be in the format: `redis://:PASSWORD@SENTINEL_PRIMARY_NAME` + gitlab_rails['redis_actioncable_instance'] = REDIS_ACTIONCABLE_URL - 1. PASSWORD is the plaintext password for the Redis instance - 1. SENTINEL_PRIMARY_NAME is the Sentinel primary name (e.g. `gitlab-redis-cache`) - -1. Include an array of hashes with host/port combinations, such as the following: - - ```ruby + # Configure the Sentinels gitlab_rails['redis_cache_sentinels'] = [ - { host: REDIS_CACHE_SENTINEL_HOST, port: PORT1 }, - { host: REDIS_CACHE_SENTINEL_HOST2, port: PORT2 } + { host: REDIS_CACHE_SENTINEL_HOST, port: 26379 }, + { host: REDIS_CACHE_SENTINEL_HOST2, port: 26379 } ] gitlab_rails['redis_queues_sentinels'] = [ - { host: REDIS_QUEUES_SENTINEL_HOST, port: PORT1 }, - { host: REDIS_QUEUES_SENTINEL_HOST2, port: PORT2 } + { host: REDIS_QUEUES_SENTINEL_HOST, port: 26379 }, + { host: REDIS_QUEUES_SENTINEL_HOST2, port: 26379 } ] gitlab_rails['redis_shared_state_sentinels'] = [ - { host: SHARED_STATE_SENTINEL_HOST, port: PORT1 }, - { host: SHARED_STATE_SENTINEL_HOST2, port: PORT2 } + { host: SHARED_STATE_SENTINEL_HOST, port: 26379 }, + { host: SHARED_STATE_SENTINEL_HOST2, port: 26379 } + ] + gitlab_rails['redis_actioncable_sentinels'] = [ + { host: ACTIONCABLE_SENTINEL_HOST, port: 26379 }, + { host: ACTIONCABLE_SENTINEL_HOST2, port: 26379 } ] ``` -1. Note that for each persistence class, GitLab will default to using the - configuration specified in `gitlab_rails['redis_sentinels']` unless - overridden by the settings above. -1. Be sure to include BOTH configuration options for each persistent classes. For example, - if you choose to configure a cache instance, you must specify both `gitlab_rails['redis_cache_instance']` - and `gitlab_rails['redis_cache_sentinels']` for GitLab to generate the proper configuration files. -1. Run `gitlab-ctl reconfigure` + Note that: + + - Redis URLs should be in the format: `redis://:PASSWORD@SENTINEL_PRIMARY_NAME`, where: + - `PASSWORD` is the plaintext password for the Redis instance. + - `SENTINEL_PRIMARY_NAME` is the Sentinel primary name set with `redis['master_name']`, + for example `gitlab-redis-cache`. + +1. Save the file and reconfigure GitLab for the change to take effect: + + ```shell + sudo gitlab-ctl reconfigure + ``` + +NOTE: **Note:** +For each persistence class, GitLab will default to using the +configuration specified in `gitlab_rails['redis_sentinels']` unless +overridden by the previously described settings. ### Control running services @@ -736,6 +752,5 @@ Read more: 1. [Reference architectures](../reference_architectures/index.md) 1. [Configure the database](../postgresql/replication_and_failover.md) -1. [Configure NFS](../high_availability/nfs.md) -1. [Configure the GitLab application servers](../high_availability/gitlab.md) -1. [Configure the load balancers](../high_availability/load_balancer.md) +1. [Configure NFS](../nfs.md) +1. [Configure the load balancers](../load_balancer.md) diff --git a/doc/administration/redis/replication_and_failover_external.md b/doc/administration/redis/replication_and_failover_external.md index 244b44dd76a..d530e6a8fd7 100644 --- a/doc/administration/redis/replication_and_failover_external.md +++ b/doc/administration/redis/replication_and_failover_external.md @@ -19,13 +19,7 @@ The following are the requirements for providing your own Redis instance: - Redis version 5.0 or higher is recommended, as this is what ships with Omnibus GitLab packages starting with GitLab 12.7. -- Support for Redis 3.2 is deprecated with GitLab 12.10 and will be completely - removed in GitLab 13.0. -- GitLab 12.0 and later requires Redis version 3.2 or higher. Older Redis - versions do not support an optional count argument to SPOP which is now - required for [Merge Trains](../../ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md). -- In addition, if Redis 4 or later is available, GitLab makes use of certain - commands like `UNLINK` and `USAGE` which were introduced only in Redis 4. +- GitLab 13.0 and later requires Redis version 4.0 or higher. - Standalone Redis or Redis high availability with Sentinel are supported. Redis Cluster is not supported. - Managed Redis from cloud providers such as AWS ElastiCache will work. If these @@ -221,7 +215,7 @@ the correct credentials for the Sentinel nodes. While it doesn't require a list of all Sentinel nodes, in case of a failure, it needs to access at least one of listed ones. -The following steps should be performed in the [GitLab application server](../high_availability/gitlab.md) +The following steps should be performed in the GitLab application server which ideally should not have Redis or Sentinels in the same machine: 1. Edit `/home/git/gitlab/config/resque.yml` following the example in diff --git a/doc/administration/redis/standalone.md b/doc/administration/redis/standalone.md index 12e932dbc5e..ea5a7850244 100644 --- a/doc/administration/redis/standalone.md +++ b/doc/administration/redis/standalone.md @@ -15,7 +15,7 @@ is generally stable and can handle many requests, so it is an acceptable trade off to have only a single instance. See the [reference architectures](../reference_architectures/index.md) page for an overview of GitLab scaling options. -## Set up a standalone Redis instance +## Set up the standalone Redis instance The steps below are the minimum necessary to configure a Redis server with Omnibus GitLab: @@ -28,36 +28,49 @@ Omnibus GitLab: 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby - ## Enable Redis - redis['enable'] = true - - ## Disable all other services - sidekiq['enable'] = false - gitlab_workhorse['enable'] = false - puma['enable'] = false - postgresql['enable'] = false - nginx['enable'] = false - prometheus['enable'] = false - alertmanager['enable'] = false - pgbouncer_exporter['enable'] = false - gitlab_exporter['enable'] = false - gitaly['enable'] = false + ## Enable Redis and disable all other services + ## https://docs.gitlab.com/omnibus/roles/ + roles ['redis_master_role'] + ## Redis configuration redis['bind'] = '0.0.0.0' redis['port'] = 6379 - redis['password'] = 'SECRET_PASSWORD_HERE' + redis['password'] = '<redis_password>' - gitlab_rails['enable'] = false + ## Disable automatic database migrations + ## Only the primary GitLab application server should handle migrations + gitlab_rails['auto_migrate'] = false ``` 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. 1. Note the Redis node's IP address or hostname, port, and - Redis password. These will be necessary when configuring the GitLab - application servers later. + Redis password. These will be necessary when [configuring the GitLab + application servers](#set-up-the-gitlab-rails-application-instance). [Advanced configuration options](https://docs.gitlab.com/omnibus/settings/redis.html) are supported and can be added if needed. +## Set up the GitLab Rails application instance + +On the instance where GitLab is installed: + +1. Edit the `/etc/gitlab/gitlab.rb` file and add the following contents: + + ```ruby + ## Disable Redis + redis['enable'] = false + + gitlab_rails['redis_host'] = 'redis.example.com' + gitlab_rails['redis_port'] = 6379 + + ## Required if Redis authentication is configured on the Redis node + gitlab_rails['redis_password'] = '<redis_password>' + ``` + +1. Save your changes to `/etc/gitlab/gitlab.rb`. + +1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + ## Troubleshooting See the [Redis troubleshooting guide](troubleshooting.md). diff --git a/doc/administration/reference_architectures/10k_users.md b/doc/administration/reference_architectures/10k_users.md index 5367021af4e..fe2dad41066 100644 --- a/doc/administration/reference_architectures/10k_users.md +++ b/doc/administration/reference_architectures/10k_users.md @@ -1,76 +1,2047 @@ -# Reference architecture: up to 10,000 users +--- +reading_time: true +stage: Enablement +group: Distribution +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- -This page describes GitLab reference architecture for up to 10,000 users. -For a full list of reference architectures, see +# Reference architecture: up to 10,000 users **(PREMIUM ONLY)** + +This page describes GitLab reference architecture for up to 10,000 users. For a +full list of reference architectures, see [Available reference architectures](index.md#available-reference-architectures). > - **Supported users (approximate):** 10,000 -> - **High Availability:** True -> - **Test RPS rates:** API: 200 RPS, Web: 20 RPS, Git: 20 RPS - -| Service | Nodes | Configuration ([8](#footnotes)) | GCP | AWS | Azure | -|--------------------------------------------------------------|-------|---------------------------------|----------------|-----------------------|----------------| -| GitLab Rails ([1](#footnotes)) | 3 | 32 vCPU, 28.8GB Memory | n1-highcpu-32 | c5.9xlarge | F32s v2 | -| PostgreSQL | 3 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | D4s v3 | -| PgBouncer | 3 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | F2s v2 | -| Gitaly ([2](#footnotes)) ([5](#footnotes)) ([7](#footnotes)) | X | 16 vCPU, 60GB Memory | n1-standard-16 | m5.4xlarge | D16s v3 | -| Redis ([3](#footnotes)) - Cache | 3 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | D4s v3 | -| Redis ([3](#footnotes)) - Queues / Shared State | 3 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | D4s v3 | -| Redis Sentinel ([3](#footnotes)) - Cache | 3 | 1 vCPU, 1.7GB Memory | g1-small | t2.small | B1MS | -| Redis Sentinel ([3](#footnotes)) - Queues / Shared State | 3 | 1 vCPU, 1.7GB Memory | g1-small | t2.small | B1MS | -| Consul | 3 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | F2s v2 | -| Sidekiq | 4 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | D4s v3 | -| Object Storage ([4](#footnotes)) | - | - | - | - | - | -| NFS Server ([5](#footnotes)) ([7](#footnotes)) | 1 | 4 vCPU, 3.6GB Memory | n1-highcpu-4 | c5.xlarge | F4s v2 | -| Monitoring node | 1 | 4 vCPU, 3.6GB Memory | n1-highcpu-4 | c5.xlarge | F4s v2 | -| External load balancing node ([6](#footnotes)) | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | F2s v2 | -| Internal load balancing node ([6](#footnotes)) | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | F2s v2 | - -## Footnotes - -1. In our architectures we run each GitLab Rails node using the Puma webserver - and have its number of workers set to 90% of available CPUs along with four threads. For - nodes that are running Rails with other components the worker value should be reduced - accordingly where we've found 50% achieves a good balance but this is dependent - on workload. - -1. Gitaly node requirements are dependent on customer data, specifically the number of - projects and their sizes. We recommend two nodes as an absolute minimum for HA environments - and at least four nodes should be used when supporting 50,000 or more users. - We also recommend that each Gitaly node should store no more than 5TB of data - and have the number of [`gitaly-ruby` workers](../gitaly/index.md#gitaly-ruby) - set to 20% of available CPUs. Additional nodes should be considered in conjunction - with a review of expected data size and spread based on the recommendations above. - -1. Recommended Redis setup differs depending on the size of the architecture. - For smaller architectures (less than 3,000 users) a single instance should suffice. - For medium sized installs (3,000 - 5,000) we suggest one Redis cluster for all - classes and that Redis Sentinel is hosted alongside Consul. - For larger architectures (10,000 users or more) we suggest running a separate - [Redis Cluster](../redis/replication_and_failover.md#running-multiple-redis-clusters) for the Cache class - and another for the Queues and Shared State classes respectively. We also recommend - that you run the Redis Sentinel clusters separately for each Redis Cluster. - -1. For data objects such as LFS, Uploads, Artifacts, etc. We recommend an [Object Storage service](../object_storage.md) - over NFS where possible, due to better performance and availability. - -1. NFS can be used as an alternative for both repository data (replacing Gitaly) and - object storage but this isn't typically recommended for performance reasons. Note however it is required for - [GitLab Pages](https://gitlab.com/gitlab-org/gitlab-pages/-/issues/196). - -1. Our architectures have been tested and validated with [HAProxy](https://www.haproxy.org/) - as the load balancer. Although other load balancers with similar feature sets - could also be used, those load balancers have not been validated. - -1. We strongly recommend that any Gitaly or NFS nodes be set up with SSD disks over - HDD with a throughput of at least 8,000 IOPS for read operations and 2,000 IOPS for write - as these components have heavy I/O. These IOPS values are recommended only as a starter - as with time they may be adjusted higher or lower depending on the scale of your - environment's workload. If you're running the environment on a Cloud provider - you may need to refer to their documentation on how configure IOPS correctly. - -1. The architectures were built and tested with the [Intel Xeon E5 v3 (Haswell)](https://cloud.google.com/compute/docs/cpu-platforms) - CPU platform on GCP. On different hardware you may find that adjustments, either lower - or higher, are required for your CPU or Node counts accordingly. For more information, a - [Sysbench](https://github.com/akopytov/sysbench) benchmark of the CPU can be found - [here](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Reference-Architectures/GCP-CPU-Benchmarks). +> - **High Availability:** Yes +> - **Test requests per second (RPS) rates:** API: 200 RPS, Web: 20 RPS, Git: 20 RPS + +| Service | Nodes | Configuration | GCP | AWS | Azure | +|--------------------------------------------|-------------|-------------------------|-----------------|-------------|----------| +| External load balancing node | 1 | 2 vCPU, 1.8GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| Consul | 3 | 2 vCPU, 1.8GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| PostgreSQL | 3 | 4 vCPU, 15GB memory | n1-standard-4 | m5.xlarge | D4s v3 | +| PgBouncer | 3 | 2 vCPU, 1.8GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| Internal load balancing node | 1 | 2 vCPU, 1.8GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| Redis - Cache | 3 | 4 vCPU, 15GB memory | n1-standard-4 | m5.xlarge | D4s v3 | +| Redis - Queues / Shared State | 3 | 4 vCPU, 15GB memory | n1-standard-4 | m5.xlarge | D4s v3 | +| Redis Sentinel - Cache | 3 | 1 vCPU, 1.7GB memory | g1-small | t2.small | B1MS | +| Redis Sentinel - Queues / Shared State | 3 | 1 vCPU, 1.7GB memory | g1-small | t2.small | B1MS | +| Gitaly | 2 (minimum) | 16 vCPU, 60GB memory | n1-standard-16 | m5.4xlarge | D16s v3 | +| Sidekiq | 4 | 4 vCPU, 15GB memory | n1-standard-4 | m5.xlarge | D4s v3 | +| GitLab Rails | 3 | 32 vCPU, 28.8GB memory | n1-highcpu-32 | c5.9xlarge | F32s v2 | +| Monitoring node | 1 | 4 vCPU, 3.6GB memory | n1-highcpu-4 | c5.xlarge | F4s v2 | +| Object Storage | n/a | n/a | n/a | n/a | n/a | +| NFS Server | 1 | 4 vCPU, 3.6GB memory | n1-highcpu-4 | c5.xlarge | F4s v2 | + +The Google Cloud Platform (GCP) architectures were built and tested using the +[Intel Xeon E5 v3 (Haswell)](https://cloud.google.com/compute/docs/cpu-platforms) +CPU platform. On different hardware you may find that adjustments, either lower +or higher, are required for your CPU or node counts. For more information, see +our [Sysbench](https://github.com/akopytov/sysbench)-based +[CPU benchmark](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Reference-Architectures/GCP-CPU-Benchmarks). + +For data objects (such as LFS, Uploads, or Artifacts), an +[object storage service](#configure-the-object-storage) is recommended instead +of NFS where possible, due to better performance and availability. Since this +doesn't require a node to be set up, *Object Storage* is noted as not +applicable (n/a) in the previous table. + +## Setup components + +To set up GitLab and its components to accommodate up to 10,000 users: + +1. [Configure the external load balancing node](#configure-the-external-load-balancer) + that will handle the load balancing of the three GitLab application services nodes. +1. [Configure Consul](#configure-consul). +1. [Configure PostgreSQL](#configure-postgresql), the database for GitLab. +1. [Configure PgBouncer](#configure-pgbouncer). +1. [Configure the internal load balancing node](#configure-the-internal-load-balancer) +1. [Configure Redis](#configure-redis). +1. [Configure Gitaly](#configure-gitaly), + which provides access to the Git repositories. +1. [Configure Sidekiq](#configure-sidekiq). +1. [Configure the main GitLab Rails application](#configure-gitlab-rails) + to run Puma/Unicorn, Workhorse, GitLab Shell, and to serve all frontend requests (UI, API, Git + over HTTP/SSH). +1. [Configure Prometheus](#configure-prometheus) to monitor your GitLab environment. +1. [Configure the Object Storage](#configure-the-object-storage) + used for shared data objects. +1. [Configure NFS (Optional)](#configure-nfs-optional) + to have shared disk storage service as an alternative to Gitaly and/or Object Storage (although + not recommended). NFS is required for GitLab Pages, you can skip this step if you're not using + that feature. + +We start with all servers on the same 10.6.0.0/24 private network range, they +can connect to each other freely on those addresses. + +Here is a list and description of each machine and the assigned IP: + +- `10.6.0.10`: External Load Balancer +- `10.6.0.11`: Consul 1 +- `10.6.0.12`: Consul 2 +- `10.6.0.13`: Consul 3 +- `10.6.0.21`: PostgreSQL primary +- `10.6.0.22`: PostgreSQL secondary 1 +- `10.6.0.23`: PostgreSQL secondary 2 +- `10.6.0.31`: PgBouncer 1 +- `10.6.0.32`: PgBouncer 2 +- `10.6.0.33`: PgBouncer 3 +- `10.6.0.40`: Internal Load Balancer +- `10.6.0.51`: Redis - Cache Primary +- `10.6.0.52`: Redis - Cache Replica 1 +- `10.6.0.53`: Redis - Cache Replica 2 +- `10.6.0.71`: Sentinel - Cache 1 +- `10.6.0.72`: Sentinel - Cache 2 +- `10.6.0.73`: Sentinel - Cache 3 +- `10.6.0.61`: Redis - Queues Primary +- `10.6.0.62`: Redis - Queues Replica 1 +- `10.6.0.63`: Redis - Queues Replica 2 +- `10.6.0.81`: Sentinel - Queues 1 +- `10.6.0.82`: Sentinel - Queues 2 +- `10.6.0.83`: Sentinel - Queues 3 +- `10.6.0.91`: Gitaly 1 +- `10.6.0.92`: Gitaly 2 +- `10.6.0.101`: Sidekiq 1 +- `10.6.0.102`: Sidekiq 2 +- `10.6.0.103`: Sidekiq 3 +- `10.6.0.104`: Sidekiq 4 +- `10.6.0.111`: GitLab application 1 +- `10.6.0.112`: GitLab application 2 +- `10.6.0.113`: GitLab application 3 +- `10.6.0.121`: Prometheus + +## Configure the external load balancer + +NOTE: **Note:** +This architecture has been tested and validated with [HAProxy](https://www.haproxy.org/) +as the load balancer. Although other load balancers with similar feature sets +could also be used, those load balancers have not been validated. + +In an active/active GitLab configuration, you will need a load balancer to route +traffic to the application servers. The specifics on which load balancer to use +or the exact configuration is beyond the scope of GitLab documentation. We hope +that if you're managing multi-node systems like GitLab you have a load balancer of +choice already. Some examples including HAProxy (open-source), F5 Big-IP LTM, +and Citrix Net Scaler. This documentation will outline what ports and protocols +you need to use with GitLab. + +The next question is how you will handle SSL in your environment. +There are several different options: + +- [The application node terminates SSL](#application-node-terminates-ssl). +- [The load balancer terminates SSL without backend SSL](#load-balancer-terminates-ssl-without-backend-ssl) + and communication is not secure between the load balancer and the application node. +- [The load balancer terminates SSL with backend SSL](#load-balancer-terminates-ssl-with-backend-ssl) + and communication is *secure* between the load balancer and the application node. + +### Application node terminates SSL + +Configure your load balancer to pass connections on port 443 as `TCP` rather +than `HTTP(S)` protocol. This will pass the connection to the application node's +NGINX service untouched. NGINX will have the SSL certificate and listen on port 443. + +See the [NGINX HTTPS documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https) +for details on managing SSL certificates and configuring NGINX. + +### Load balancer terminates SSL without backend SSL + +Configure your load balancer to use the `HTTP(S)` protocol rather than `TCP`. +The load balancer will then be responsible for managing SSL certificates and +terminating SSL. + +Since communication between the load balancer and GitLab will not be secure, +there is some additional configuration needed. See the +[NGINX proxied SSL documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#supporting-proxied-ssl) +for details. + +### Load balancer terminates SSL with backend SSL + +Configure your load balancer(s) to use the 'HTTP(S)' protocol rather than 'TCP'. +The load balancer(s) will be responsible for managing SSL certificates that +end users will see. + +Traffic will also be secure between the load balancer(s) and NGINX in this +scenario. There is no need to add configuration for proxied SSL since the +connection will be secure all the way. However, configuration will need to be +added to GitLab to configure SSL certificates. See +[NGINX HTTPS documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https) +for details on managing SSL certificates and configuring NGINX. + +### Ports + +The basic ports to be used are shown in the table below. + +| LB Port | Backend Port | Protocol | +| ------- | ------------ | ------------------------ | +| 80 | 80 | HTTP (*1*) | +| 443 | 443 | TCP or HTTPS (*1*) (*2*) | +| 22 | 22 | TCP | + +- (*1*): [Web terminal](../../ci/environments/index.md#web-terminals) support requires + your load balancer to correctly handle WebSocket connections. When using + HTTP or HTTPS proxying, this means your load balancer must be configured + to pass through the `Connection` and `Upgrade` hop-by-hop headers. See the + [web terminal](../integration/terminal.md) integration guide for + more details. +- (*2*): When using HTTPS protocol for port 443, you will need to add an SSL + certificate to the load balancers. If you wish to terminate SSL at the + GitLab application server instead, use TCP protocol. + +If you're using GitLab Pages with custom domain support you will need some +additional port configurations. +GitLab Pages requires a separate virtual IP address. Configure DNS to point the +`pages_external_url` from `/etc/gitlab/gitlab.rb` at the new virtual IP address. See the +[GitLab Pages documentation](../pages/index.md) for more information. + +| LB Port | Backend Port | Protocol | +| ------- | ------------- | --------- | +| 80 | Varies (*1*) | HTTP | +| 443 | Varies (*1*) | TCP (*2*) | + +- (*1*): The backend port for GitLab Pages depends on the + `gitlab_pages['external_http']` and `gitlab_pages['external_https']` + setting. See [GitLab Pages documentation](../pages/index.md) for more details. +- (*2*): Port 443 for GitLab Pages should always use the TCP protocol. Users can + configure custom domains with custom SSL, which would not be possible + if SSL was terminated at the load balancer. + +#### Alternate SSH Port + +Some organizations have policies against opening SSH port 22. In this case, +it may be helpful to configure an alternate SSH hostname that allows users +to use SSH on port 443. An alternate SSH hostname will require a new virtual IP address +compared to the other GitLab HTTP configuration above. + +Configure DNS for an alternate SSH hostname such as `altssh.gitlab.example.com`. + +| LB Port | Backend Port | Protocol | +| ------- | ------------ | -------- | +| 443 | 22 | TCP | + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure Consul + +The following IPs will be used as an example: + +- `10.6.0.11`: Consul 1 +- `10.6.0.12`: Consul 2 +- `10.6.0.13`: Consul 3 + +NOTE: **Note:** +The configuration processes for the other servers in your reference architecture will +use the `/etc/gitlab/gitlab-secrets.json` file from your Consul server to connect +with the other servers. + +To configure Consul: + +1. SSH into the server that will host Consul. +1. [Download/install](https://about.gitlab.com/install/) the + Omnibus GitLab Enterprise Edition package using **steps 1 and 2** from the + GitLab downloads page. + - Make sure you select the correct Omnibus package, with the same version + the GitLab application is running. + - Do not complete any other steps on the download page. + +1. Edit `/etc/gitlab/gitlab.rb` and add the contents: + + ```ruby + roles ['consul_role'] + + ## Enable service discovery for Prometheus + consul['enable'] = true + consul['monitoring_service_discovery'] = true + + ## The IPs of the Consul server nodes + ## You can also use FQDNs and intermix them with IPs + consul['configuration'] = { + server: true, + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), + } + + # Set the network addresses that the exporters will listen on + node_exporter['listen_address'] = '0.0.0.0:9100' + + # Disable auto migrations + gitlab_rails['auto_migrate'] = false + ``` + +1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. +1. Go through the steps again for all the other Consul nodes, and + make sure you set up the correct IPs. + +NOTE: **Note:** +A Consul leader will be elected when the provisioning of the third Consul server is completed. +Viewing the Consul logs `sudo gitlab-ctl tail consul` will display +`...[INFO] consul: New leader elected: ...` + +You can list the current Consul members (server, client): + +```shell +sudo /opt/gitlab/embedded/bin/consul members +``` + +You can verify the GitLab services are running: + +```shell +sudo gitlab-ctl status +``` + +The output should be similar to the following: + +```plaintext +run: consul: (pid 30074) 76834s; run: log: (pid 29740) 76844s +run: logrotate: (pid 30925) 3041s; run: log: (pid 29649) 76861s +run: node-exporter: (pid 30093) 76833s; run: log: (pid 29663) 76855s +``` + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure PostgreSQL + +In this section, you'll be guided through configuring an external PostgreSQL database +to be used with GitLab. + +### Provide your own PostgreSQL instance + +If you're hosting GitLab on a cloud provider, you can optionally use a +managed service for PostgreSQL. For example, AWS offers a managed Relational +Database Service (RDS) that runs PostgreSQL. + +If you use a cloud-managed service, or provide your own PostgreSQL: + +1. Set up PostgreSQL according to the + [database requirements document](../../install/requirements.md#database). +1. Set up a `gitlab` username with a password of your choice. The `gitlab` user + needs privileges to create the `gitlabhq_production` database. +1. Configure the GitLab application servers with the appropriate details. + This step is covered in [Configuring the GitLab Rails application](#configure-gitlab-rails). + +### Standalone PostgreSQL using Omnibus GitLab + +The following IPs will be used as an example: + +- `10.6.0.21`: PostgreSQL primary +- `10.6.0.22`: PostgreSQL secondary 1 +- `10.6.0.23`: PostgreSQL secondary 2 + +First, make sure to [install](https://about.gitlab.com/install/) +the Linux GitLab package **on each node**. Following the steps, +install the necessary dependencies from step 1, and add the +GitLab package repository from step 2. When installing GitLab +in the second step, do not supply the `EXTERNAL_URL` value. + +#### PostgreSQL primary node + +1. SSH into the PostgreSQL primary node. +1. Generate a password hash for the PostgreSQL username/password pair. This assumes you will use the default + username of `gitlab` (recommended). The command will request a password + and confirmation. Use the value that is output by this command in the next + step as the value of `<postgresql_password_hash>`: + + ```shell + sudo gitlab-ctl pg-password-md5 gitlab + ``` + +1. Generate a password hash for the PgBouncer username/password pair. This assumes you will use the default + username of `pgbouncer` (recommended). The command will request a password + and confirmation. Use the value that is output by this command in the next + step as the value of `<pgbouncer_password_hash>`: + + ```shell + sudo gitlab-ctl pg-password-md5 pgbouncer + ``` + +1. Generate a password hash for the Consul database username/password pair. This assumes you will use the default + username of `gitlab-consul` (recommended). The command will request a password + and confirmation. Use the value that is output by this command in the next + step as the value of `<consul_password_hash>`: + + ```shell + sudo gitlab-ctl pg-password-md5 gitlab-consul + ``` + +1. On the primary database node, edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: + + ```ruby + # Disable all components except PostgreSQL and Repmgr and Consul + roles ['postgres_role'] + + # PostgreSQL configuration + postgresql['listen_address'] = '0.0.0.0' + postgresql['hot_standby'] = 'on' + postgresql['wal_level'] = 'replica' + postgresql['shared_preload_libraries'] = 'repmgr_funcs' + + # Disable automatic database migrations + gitlab_rails['auto_migrate'] = false + + # Configure the Consul agent + consul['services'] = %w(postgresql) + + # START user configuration + # Please set the real values as explained in Required Information section + # + # Replace PGBOUNCER_PASSWORD_HASH with a generated md5 value + postgresql['pgbouncer_user_password'] = '<pgbouncer_password_hash>' + # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value + postgresql['sql_user_password'] = '<postgresql_password_hash>' + # Set `max_wal_senders` to one more than the number of database nodes in the cluster. + # This is used to prevent replication from using up all of the + # available database connections. + postgresql['max_wal_senders'] = 4 + postgresql['max_replication_slots'] = 4 + + # Replace XXX.XXX.XXX.XXX/YY with Network Address + postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24) + repmgr['trust_auth_cidr_addresses'] = %w(127.0.0.1/32 10.6.0.0/24) + + ## Enable service discovery for Prometheus + consul['monitoring_service_discovery'] = true + + # Set the network addresses that the exporters will listen on for monitoring + node_exporter['listen_address'] = '0.0.0.0:9100' + postgres_exporter['listen_address'] = '0.0.0.0:9187' + + ## The IPs of the Consul server nodes + ## You can also use FQDNs and intermix them with IPs + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), + } + # + # END user configuration + ``` + +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace + the file of the same name on this server. If that file is not on this server, + add the file from your Consul server to this server. + +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +#### PostgreSQL secondary nodes + +1. On both the secondary nodes, add the same configuration specified above for the primary node + with an additional setting (`repmgr['master_on_initialization'] = false`) that will inform `gitlab-ctl` that they are standby nodes initially + and there's no need to attempt to register them as a primary node: + + ```ruby + # Disable all components except PostgreSQL and Repmgr and Consul + roles ['postgres_role'] + + # PostgreSQL configuration + postgresql['listen_address'] = '0.0.0.0' + postgresql['hot_standby'] = 'on' + postgresql['wal_level'] = 'replica' + postgresql['shared_preload_libraries'] = 'repmgr_funcs' + + # Disable automatic database migrations + gitlab_rails['auto_migrate'] = false + + # Configure the Consul agent + consul['services'] = %w(postgresql) + + # Specify if a node should attempt to be primary on initialization. + repmgr['master_on_initialization'] = false + + # Replace PGBOUNCER_PASSWORD_HASH with a generated md5 value + postgresql['pgbouncer_user_password'] = '<pgbouncer_password_hash>' + # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value + postgresql['sql_user_password'] = '<postgresql_password_hash>' + # Set `max_wal_senders` to one more than the number of database nodes in the cluster. + # This is used to prevent replication from using up all of the + # available database connections. + postgresql['max_wal_senders'] = 4 + postgresql['max_replication_slots'] = 4 + + # Replace with your network addresses + postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24) + repmgr['trust_auth_cidr_addresses'] = %w(127.0.0.1/32 10.6.0.0/24) + + ## Enable service discovery for Prometheus + consul['monitoring_service_discovery'] = true + + # Set the network addresses that the exporters will listen on for monitoring + node_exporter['listen_address'] = '0.0.0.0:9100' + postgres_exporter['listen_address'] = '0.0.0.0:9187' + + ## The IPs of the Consul server nodes + ## You can also use FQDNs and intermix them with IPs + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), + } + ``` + +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace + the file of the same name on this server. If that file is not on this server, + add the file from your Consul server to this server. + +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + +Advanced [configuration options](https://docs.gitlab.com/omnibus/settings/database.html) +are supported and can be added if needed. + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +#### PostgreSQL post-configuration + +SSH into the **primary node**: + +1. Open a database prompt: + + ```shell + gitlab-psql -d gitlabhq_production + ``` + +1. Make sure the `pg_trgm` extension is enabled (it might already be): + + ```shell + CREATE EXTENSION pg_trgm; + ``` + +1. Exit the database prompt by typing `\q` and Enter. + +1. Verify the cluster is initialized with one node: + + ```shell + gitlab-ctl repmgr cluster show + ``` + + The output should be similar to the following: + + ```plaintext + Role | Name | Upstream | Connection String + ----------+----------|----------|---------------------------------------- + * master | HOSTNAME | | host=HOSTNAME user=gitlab_repmgr dbname=gitlab_repmgr + ``` + +1. Note down the hostname or IP address in the connection string: `host=HOSTNAME`. We will + refer to the hostname in the next section as `<primary_node_name>`. If the value + is not an IP address, it will need to be a resolvable name (via DNS or + `/etc/hosts`) + +SSH into the **secondary node**: + +1. Set up the repmgr standby: + + ```shell + gitlab-ctl repmgr standby setup <primary_node_name> + ``` + + Do note that this will remove the existing data on the node. The command + has a wait time. + + The output should be similar to the following: + + ```console + Doing this will delete the entire contents of /var/opt/gitlab/postgresql/data + If this is not what you want, hit Ctrl-C now to exit + To skip waiting, rerun with the -w option + Sleeping for 30 seconds + Stopping the database + Removing the data + Cloning the data + Starting the database + Registering the node with the cluster + ok: run: repmgrd: (pid 19068) 0s + ``` + +Before moving on, make sure the databases are configured correctly. Run the +following command on the **primary** node to verify that replication is working +properly and the secondary nodes appear in the cluster: + +```shell +gitlab-ctl repmgr cluster show +``` + +The output should be similar to the following: + +```plaintext +Role | Name | Upstream | Connection String +----------+---------|-----------|------------------------------------------------ +* master | MASTER | | host=<primary_node_name> user=gitlab_repmgr dbname=gitlab_repmgr + standby | STANDBY | MASTER | host=<secondary_node_name> user=gitlab_repmgr dbname=gitlab_repmgr + standby | STANDBY | MASTER | host=<secondary_node_name> user=gitlab_repmgr dbname=gitlab_repmgr +``` + +If the 'Role' column for any node says "FAILED", check the +[Troubleshooting section](troubleshooting.md) before proceeding. + +Also, check that the `repmgr-check-master` command works successfully on each node: + +```shell +su - gitlab-consul +gitlab-ctl repmgr-check-master || echo 'This node is a standby repmgr node' +``` + +This command relies on exit codes to tell Consul whether a particular node is a master +or secondary. The most important thing here is that this command does not produce errors. +If there are errors it's most likely due to incorrect `gitlab-consul` database user permissions. +Check the [Troubleshooting section](troubleshooting.md) before proceeding. + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure PgBouncer + +Now that the PostgreSQL servers are all set up, let's configure PgBouncer. +The following IPs will be used as an example: + +- `10.6.0.31`: PgBouncer 1 +- `10.6.0.32`: PgBouncer 2 +- `10.6.0.33`: PgBouncer 3 + +1. On each PgBouncer node, edit `/etc/gitlab/gitlab.rb`, and replace + `<consul_password_hash>` and `<pgbouncer_password_hash>` with the + password hashes you [set up previously](#postgresql-primary-node): + + ```ruby + # Disable all components except Pgbouncer and Consul agent + roles ['pgbouncer_role'] + + # Configure PgBouncer + pgbouncer['admin_users'] = %w(pgbouncer gitlab-consul) + + pgbouncer['users'] = { + 'gitlab-consul': { + password: '<consul_password_hash>' + }, + 'pgbouncer': { + password: '<pgbouncer_password_hash>' + } + } + + # Configure Consul agent + consul['watchers'] = %w(postgresql) + consul['enable'] = true + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13) + } + + # Enable service discovery for Prometheus + consul['monitoring_service_discovery'] = true + + # Set the network addresses that the exporters will listen on + node_exporter['listen_address'] = '0.0.0.0:9100' + ``` + +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace + the file of the same name on this server. If that file is not on this server, + add the file from your Consul server to this server. + +1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + + NOTE: **Note:** + If an error `execute[generate databases.ini]` occurs, this is due to an existing + [known issue](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/4713). + It will be resolved when you run a second `reconfigure` after the next step. + +1. Create a `.pgpass` file so Consul is able to + reload PgBouncer. Enter the PgBouncer password twice when asked: + + ```shell + gitlab-ctl write-pgpass --host 127.0.0.1 --database pgbouncer --user pgbouncer --hostuser gitlab-consul + ``` + +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) once again + to resolve any potential errors from the previous steps. +1. Ensure each node is talking to the current primary: + + ```shell + gitlab-ctl pgb-console # You will be prompted for PGBOUNCER_PASSWORD + ``` + +1. Once the console prompt is available, run the following queries: + + ```shell + show databases ; show clients ; + ``` + + The output should be similar to the following: + + ```plaintext + name | host | port | database | force_user | pool_size | reserve_pool | pool_mode | max_connections | current_connections + ---------------------+-------------+------+---------------------+------------+-----------+--------------+-----------+-----------------+--------------------- + gitlabhq_production | MASTER_HOST | 5432 | gitlabhq_production | | 20 | 0 | | 0 | 0 + pgbouncer | | 6432 | pgbouncer | pgbouncer | 2 | 0 | statement | 0 | 0 + (2 rows) + + type | user | database | state | addr | port | local_addr | local_port | connect_time | request_time | ptr | link | remote_pid | tls + ------+-----------+---------------------+---------+----------------+-------+------------+------------+---------------------+---------------------+-----------+------+------------+----- + C | pgbouncer | pgbouncer | active | 127.0.0.1 | 56846 | 127.0.0.1 | 6432 | 2017-08-21 18:09:59 | 2017-08-21 18:10:48 | 0x22b3880 | | 0 | + (2 rows) + ``` + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +### Configure the internal load balancer + +If you're running more than one PgBouncer node as recommended, then at this time you'll need to set +up a TCP internal load balancer to serve each correctly. + +The following IP will be used as an example: + +- `10.6.0.40`: Internal Load Balancer + +Here's how you could do it with [HAProxy](https://www.haproxy.org/): + +```plaintext +global + log /dev/log local0 + log localhost local1 notice + log stdout format raw local0 + +defaults + log global + default-server inter 10s fall 3 rise 2 + balance leastconn + +frontend internal-pgbouncer-tcp-in + bind *:6432 + mode tcp + option tcplog + + default_backend pgbouncer + +backend pgbouncer + mode tcp + option tcp-check + + server pgbouncer1 10.6.0.21:6432 check + server pgbouncer2 10.6.0.22:6432 check + server pgbouncer3 10.6.0.23:6432 check +``` + +Refer to your preferred Load Balancer's documentation for further guidance. + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure Redis + +Using [Redis](https://redis.io/) in scalable environment is possible using a **Primary** x **Replica** +topology with a [Redis Sentinel](https://redis.io/topics/sentinel) service to watch and automatically +start the failover procedure. + +Redis requires authentication if used with Sentinel. See +[Redis Security](https://redis.io/topics/security) documentation for more +information. We recommend using a combination of a Redis password and tight +firewall rules to secure your Redis service. +You are highly encouraged to read the [Redis Sentinel](https://redis.io/topics/sentinel) documentation +before configuring Redis with GitLab to fully understand the topology and +architecture. + +The requirements for a Redis setup are the following: + +1. All Redis nodes must be able to talk to each other and accept incoming + connections over Redis (`6379`) and Sentinel (`26379`) ports (unless you + change the default ones). +1. The server that hosts the GitLab application must be able to access the + Redis nodes. +1. Protect the nodes from access from external networks + ([Internet](https://gitlab.com/gitlab-org/gitlab-foss/uploads/c4cc8cd353604bd80315f9384035ff9e/The_Internet_IT_Crowd.png)), + using a firewall. + +In this section, you'll be guided through configuring two external Redis clusters +to be used with GitLab. The following IPs will be used as an example: + +- `10.6.0.51`: Redis - Cache Primary +- `10.6.0.52`: Redis - Cache Replica 1 +- `10.6.0.53`: Redis - Cache Replica 2 +- `10.6.0.71`: Sentinel - Cache 1 +- `10.6.0.72`: Sentinel - Cache 2 +- `10.6.0.73`: Sentinel - Cache 3 +- `10.6.0.61`: Redis - Queues Primary +- `10.6.0.62`: Redis - Queues Replica 1 +- `10.6.0.63`: Redis - Queues Replica 2 +- `10.6.0.81`: Sentinel - Queues 1 +- `10.6.0.82`: Sentinel - Queues 2 +- `10.6.0.83`: Sentinel - Queues 3 + +NOTE: **Providing your own Redis instance:** +Managed Redis from cloud providers such as AWS ElastiCache will work. If these +services support high availability, be sure it is **not** the Redis Cluster type. +Redis version 5.0 or higher is required, as this is what ships with +Omnibus GitLab packages starting with GitLab 13.0. Older Redis versions +do not support an optional count argument to SPOP which is now required for +[Merge Trains](../../ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md). +Note the Redis node's IP address or hostname, port, and password (if required). +These will be necessary when configuring the +[GitLab application servers](#configure-gitlab-rails) later. + +### Configure the Redis and Sentinel Cache cluster + +This is the section where we install and set up the new Redis Cache instances. + +NOTE: **Note:** +Redis nodes (both primary and replica) will need the same password defined in +`redis['password']`. At any time during a failover the Sentinels can +reconfigure a node and change its status from primary to replica and vice versa. + +#### Configure the primary Redis Cache node + +1. SSH into the **Primary** Redis server. +1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab + package you want using **steps 1 and 2** from the GitLab downloads page. + - Make sure you select the correct Omnibus package, with the same version + and type (Community, Enterprise editions) of your current install. + - Do not complete any other steps on the download page. + +1. Edit `/etc/gitlab/gitlab.rb` and add the contents: + + ```ruby + # Specify server role as 'redis_master_role' + roles ['redis_master_role'] + + # IP address pointing to a local IP that the other machines can reach to. + # You can also set bind to '0.0.0.0' which listen in all interfaces. + # If you really need to bind to an external accessible IP, make + # sure you add extra firewall rules to prevent unauthorized access. + redis['bind'] = '10.6.0.51' + + # Define a port so Redis can listen for TCP requests which will allow other + # machines to connect to it. + redis['port'] = 6379 + + # Set up password authentication for Redis (use the same password in all nodes). + redis['password'] = 'REDIS_PRIMARY_PASSWORD_OF_FIRST_CLUSTER' + + ## Enable service discovery for Prometheus + consul['enable'] = true + consul['monitoring_service_discovery'] = true + + ## The IPs of the Consul server nodes + ## You can also use FQDNs and intermix them with IPs + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), + } + + # Set the network addresses that the exporters will listen on + node_exporter['listen_address'] = '0.0.0.0:9100' + redis_exporter['listen_address'] = '0.0.0.0:9121' + + # Prevent database migrations from running on upgrade + gitlab_rails['auto_migrate'] = false + ``` + +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace + the file of the same name on this server. If that file is not on this server, + add the file from your Consul server to this server. + +1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + +NOTE: **Note:** +You can specify multiple roles like sentinel and Redis as: +`roles ['redis_sentinel_role', 'redis_master_role']`. +Read more about [roles](https://docs.gitlab.com/omnibus/roles/). + +#### Configure the replica Redis Cache nodes + +1. SSH into the **replica** Redis server. +1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab + package you want using **steps 1 and 2** from the GitLab downloads page. + - Make sure you select the correct Omnibus package, with the same version + and type (Community, Enterprise editions) of your current install. + - Do not complete any other steps on the download page. + +1. Edit `/etc/gitlab/gitlab.rb` and add the contents: + + ```ruby + # Specify server role as 'redis_replica_role' + roles ['redis_replica_role'] + + # IP address pointing to a local IP that the other machines can reach to. + # You can also set bind to '0.0.0.0' which listen in all interfaces. + # If you really need to bind to an external accessible IP, make + # sure you add extra firewall rules to prevent unauthorized access. + redis['bind'] = '10.6.0.52' + + # Define a port so Redis can listen for TCP requests which will allow other + # machines to connect to it. + redis['port'] = 6379 + + # The same password for Redis authentication you set up for the primary node. + redis['password'] = 'REDIS_PRIMARY_PASSWORD_OF_FIRST_CLUSTER' + + # The IP of the primary Redis node. + redis['master_ip'] = '10.6.0.51' + + # Port of primary Redis server, uncomment to change to non default. Defaults + # to `6379`. + #redis['master_port'] = 6379 + + ## Enable service discovery for Prometheus + consul['enable'] = true + consul['monitoring_service_discovery'] = true + + ## The IPs of the Consul server nodes + ## You can also use FQDNs and intermix them with IPs + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), + } + + # Set the network addresses that the exporters will listen on + node_exporter['listen_address'] = '0.0.0.0:9100' + redis_exporter['listen_address'] = '0.0.0.0:9121' + + # Prevent database migrations from running on upgrade + gitlab_rails['auto_migrate'] = false + ``` + +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace + the file of the same name on this server. If that file is not on this server, + add the file from your Consul server to this server. + +1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. +1. Go through the steps again for all the other replica nodes, and + make sure to set up the IPs correctly. + +NOTE: **Note:** +You can specify multiple roles like sentinel and Redis as: +`roles ['redis_sentinel_role', 'redis_master_role']`. +Read more about [roles](https://docs.gitlab.com/omnibus/roles/). + +These values don't have to be changed again in `/etc/gitlab/gitlab.rb` after +a failover, as the nodes will be managed by the [Sentinels](#configure-the-sentinel-cache-nodes), and even after a +`gitlab-ctl reconfigure`, they will get their configuration restored by +the same Sentinels. + +Advanced [configuration options](https://docs.gitlab.com/omnibus/settings/redis.html) +are supported and can be added if needed. + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +#### Configure the Sentinel Cache nodes + +NOTE: **Note:** +If you are using an external Redis Sentinel instance, be sure +to exclude the `requirepass` parameter from the Sentinel +configuration. This parameter will cause clients to report `NOAUTH +Authentication required.`. [Redis Sentinel 3.2.x does not support +password authentication](https://github.com/antirez/redis/issues/3279). + +Now that the Redis servers are all set up, let's configure the Sentinel +servers. The following IPs will be used as an example: + +- `10.6.0.71`: Sentinel - Cache 1 +- `10.6.0.72`: Sentinel - Cache 2 +- `10.6.0.73`: Sentinel - Cache 3 + +To configure the Sentinel Cache server: + +1. SSH into the server that will host Consul/Sentinel. +1. [Download/install](https://about.gitlab.com/install/) the + Omnibus GitLab Enterprise Edition package using **steps 1 and 2** from the + GitLab downloads page. + - Make sure you select the correct Omnibus package, with the same version + the GitLab application is running. + - Do not complete any other steps on the download page. + +1. Edit `/etc/gitlab/gitlab.rb` and add the contents: + + ```ruby + roles ['redis_sentinel_role'] + + ## Must be the same in every sentinel node + redis['master_name'] = 'gitlab-redis-cache' + + ## The same password for Redis authentication you set up for the primary node. + redis['master_password'] = 'REDIS_PRIMARY_PASSWORD_OF_FIRST_CLUSTER' + + ## The IP of the primary Redis node. + redis['master_ip'] = '10.6.0.51' + + ## Define a port so Redis can listen for TCP requests which will allow other + ## machines to connect to it. + redis['port'] = 6379 + + ## Port of primary Redis server, uncomment to change to non default. Defaults + ## to `6379`. + #redis['master_port'] = 6379 + + ## Configure Sentinel's IP + sentinel['bind'] = '10.6.0.71' + + ## Port that Sentinel listens on, uncomment to change to non default. Defaults + ## to `26379`. + #sentinel['port'] = 26379 + + ## Quorum must reflect the amount of voting sentinels it take to start a failover. + ## Value must NOT be greater then the amount of sentinels. + ## + ## The quorum can be used to tune Sentinel in two ways: + ## 1. If a the quorum is set to a value smaller than the majority of Sentinels + ## we deploy, we are basically making Sentinel more sensible to primary failures, + ## triggering a failover as soon as even just a minority of Sentinels is no longer + ## able to talk with the primary. + ## 1. If a quorum is set to a value greater than the majority of Sentinels, we are + ## making Sentinel able to failover only when there are a very large number (larger + ## than majority) of well connected Sentinels which agree about the primary being down.s + sentinel['quorum'] = 2 + + ## Consider unresponsive server down after x amount of ms. + #sentinel['down_after_milliseconds'] = 10000 + + ## Specifies the failover timeout in milliseconds. It is used in many ways: + ## + ## - The time needed to re-start a failover after a previous failover was + ## already tried against the same primary by a given Sentinel, is two + ## times the failover timeout. + ## + ## - The time needed for a replica replicating to a wrong primary according + ## to a Sentinel current configuration, to be forced to replicate + ## with the right primary, is exactly the failover timeout (counting since + ## the moment a Sentinel detected the misconfiguration). + ## + ## - The time needed to cancel a failover that is already in progress but + ## did not produced any configuration change (REPLICAOF NO ONE yet not + ## acknowledged by the promoted replica). + ## + ## - The maximum time a failover in progress waits for all the replica to be + ## reconfigured as replicas of the new primary. However even after this time + ## the replicas will be reconfigured by the Sentinels anyway, but not with + ## the exact parallel-syncs progression as specified. + #sentinel['failover_timeout'] = 60000 + + ## Enable service discovery for Prometheus + consul['enable'] = true + consul['monitoring_service_discovery'] = true + + ## The IPs of the Consul server nodes + ## You can also use FQDNs and intermix them with IPs + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), + } + + # Set the network addresses that the exporters will listen on + node_exporter['listen_address'] = '0.0.0.0:9100' + redis_exporter['listen_address'] = '0.0.0.0:9121' + + # Disable auto migrations + gitlab_rails['auto_migrate'] = false + ``` + +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace + the file of the same name on this server. If that file is not on this server, + add the file from your Consul server to this server. + +1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. +1. Go through the steps again for all the other Consul/Sentinel nodes, and + make sure you set up the correct IPs. + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +### Configure the Redis and Sentinel Queues cluster + +This is the section where we install and set up the new Redis Queues instances. + +NOTE: **Note:** +Redis nodes (both primary and replica) will need the same password defined in +`redis['password']`. At any time during a failover the Sentinels can +reconfigure a node and change its status from primary to replica and vice versa. + +#### Configure the primary Redis Queues node + +1. SSH into the **Primary** Redis server. +1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab + package you want using **steps 1 and 2** from the GitLab downloads page. + - Make sure you select the correct Omnibus package, with the same version + and type (Community, Enterprise editions) of your current install. + - Do not complete any other steps on the download page. + +1. Edit `/etc/gitlab/gitlab.rb` and add the contents: + + ```ruby + # Specify server role as 'redis_master_role' + roles ['redis_master_role'] + + # IP address pointing to a local IP that the other machines can reach to. + # You can also set bind to '0.0.0.0' which listen in all interfaces. + # If you really need to bind to an external accessible IP, make + # sure you add extra firewall rules to prevent unauthorized access. + redis['bind'] = '10.6.0.61' + + # Define a port so Redis can listen for TCP requests which will allow other + # machines to connect to it. + redis['port'] = 6379 + + # Set up password authentication for Redis (use the same password in all nodes). + redis['password'] = 'REDIS_PRIMARY_PASSWORD_OF_SECOND_CLUSTER' + + ## Enable service discovery for Prometheus + consul['enable'] = true + consul['monitoring_service_discovery'] = true + + ## The IPs of the Consul server nodes + ## You can also use FQDNs and intermix them with IPs + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), + } + + # Set the network addresses that the exporters will listen on + node_exporter['listen_address'] = '0.0.0.0:9100' + redis_exporter['listen_address'] = '0.0.0.0:9121' + ``` + +1. Only the primary GitLab application server should handle migrations. To + prevent database migrations from running on upgrade, add the following + configuration to your `/etc/gitlab/gitlab.rb` file: + + ```ruby + gitlab_rails['auto_migrate'] = false + ``` + +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace + the file of the same name on this server. If that file is not on this server, + add the file from your Consul server to this server. + +1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + +NOTE: **Note:** +You can specify multiple roles like sentinel and Redis as: +`roles ['redis_sentinel_role', 'redis_master_role']`. +Read more about [roles](https://docs.gitlab.com/omnibus/roles/). + +#### Configure the replica Redis Queues nodes + +1. SSH into the **replica** Redis Queue server. +1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab + package you want using **steps 1 and 2** from the GitLab downloads page. + - Make sure you select the correct Omnibus package, with the same version + and type (Community, Enterprise editions) of your current install. + - Do not complete any other steps on the download page. + +1. Edit `/etc/gitlab/gitlab.rb` and add the contents: + + ```ruby + # Specify server role as 'redis_replica_role' + roles ['redis_replica_role'] + + # IP address pointing to a local IP that the other machines can reach to. + # You can also set bind to '0.0.0.0' which listen in all interfaces. + # If you really need to bind to an external accessible IP, make + # sure you add extra firewall rules to prevent unauthorized access. + redis['bind'] = '10.6.0.62' + + # Define a port so Redis can listen for TCP requests which will allow other + # machines to connect to it. + redis['port'] = 6379 + + # The same password for Redis authentication you set up for the primary node. + redis['password'] = 'REDIS_PRIMARY_PASSWORD_OF_SECOND_CLUSTER' + + # The IP of the primary Redis node. + redis['master_ip'] = '10.6.0.61' + + # Port of primary Redis server, uncomment to change to non default. Defaults + # to `6379`. + #redis['master_port'] = 6379 + + ## Enable service discovery for Prometheus + consul['enable'] = true + consul['monitoring_service_discovery'] = true + + ## The IPs of the Consul server nodes + ## You can also use FQDNs and intermix them with IPs + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), + } + + # Set the network addresses that the exporters will listen on + node_exporter['listen_address'] = '0.0.0.0:9100' + redis_exporter['listen_address'] = '0.0.0.0:9121' + + # Disable auto migrations + gitlab_rails['auto_migrate'] = false + ``` + +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace + the file of the same name on this server. If that file is not on this server, + add the file from your Consul server to this server. + +1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. +1. Go through the steps again for all the other replica nodes, and + make sure to set up the IPs correctly. + +NOTE: **Note:** +You can specify multiple roles like sentinel and Redis as: +`roles ['redis_sentinel_role', 'redis_master_role']`. +Read more about [roles](https://docs.gitlab.com/omnibus/roles/). + +These values don't have to be changed again in `/etc/gitlab/gitlab.rb` after +a failover, as the nodes will be managed by the [Sentinels](#configure-the-sentinel-queues-nodes), and even after a +`gitlab-ctl reconfigure`, they will get their configuration restored by +the same Sentinels. + +Advanced [configuration options](https://docs.gitlab.com/omnibus/settings/redis.html) +are supported and can be added if needed. + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +#### Configure the Sentinel Queues nodes + +NOTE: **Note:** +If you are using an external Redis Sentinel instance, be sure +to exclude the `requirepass` parameter from the Sentinel +configuration. This parameter will cause clients to report `NOAUTH +Authentication required.`. [Redis Sentinel 3.2.x does not support +password authentication](https://github.com/antirez/redis/issues/3279). + +Now that the Redis servers are all set up, let's configure the Sentinel +servers. The following IPs will be used as an example: + +- `10.6.0.81`: Sentinel - Queues 1 +- `10.6.0.82`: Sentinel - Queues 2 +- `10.6.0.83`: Sentinel - Queues 3 + +To configure the Sentinel Queues server: + +1. SSH into the server that will host Sentinel. +1. [Download/install](https://about.gitlab.com/install/) the + Omnibus GitLab Enterprise Edition package using **steps 1 and 2** from the + GitLab downloads page. + - Make sure you select the correct Omnibus package, with the same version + the GitLab application is running. + - Do not complete any other steps on the download page. + +1. Edit `/etc/gitlab/gitlab.rb` and add the contents: + + ```ruby + roles ['redis_sentinel_role'] + + ## Must be the same in every sentinel node + redis['master_name'] = 'gitlab-redis-persistent' + + ## The same password for Redis authentication you set up for the primary node. + redis['master_password'] = 'REDIS_PRIMARY_PASSWORD_OF_SECOND_CLUSTER' + + ## The IP of the primary Redis node. + redis['master_ip'] = '10.6.0.61' + + ## Define a port so Redis can listen for TCP requests which will allow other + ## machines to connect to it. + redis['port'] = 6379 + + ## Port of primary Redis server, uncomment to change to non default. Defaults + ## to `6379`. + #redis['master_port'] = 6379 + + ## Configure Sentinel's IP + sentinel['bind'] = '10.6.0.81' + + ## Port that Sentinel listens on, uncomment to change to non default. Defaults + ## to `26379`. + #sentinel['port'] = 26379 + + ## Quorum must reflect the amount of voting sentinels it take to start a failover. + ## Value must NOT be greater then the amount of sentinels. + ## + ## The quorum can be used to tune Sentinel in two ways: + ## 1. If a the quorum is set to a value smaller than the majority of Sentinels + ## we deploy, we are basically making Sentinel more sensible to primary failures, + ## triggering a failover as soon as even just a minority of Sentinels is no longer + ## able to talk with the primary. + ## 1. If a quorum is set to a value greater than the majority of Sentinels, we are + ## making Sentinel able to failover only when there are a very large number (larger + ## than majority) of well connected Sentinels which agree about the primary being down.s + sentinel['quorum'] = 2 + + ## Consider unresponsive server down after x amount of ms. + #sentinel['down_after_milliseconds'] = 10000 + + ## Specifies the failover timeout in milliseconds. It is used in many ways: + ## + ## - The time needed to re-start a failover after a previous failover was + ## already tried against the same primary by a given Sentinel, is two + ## times the failover timeout. + ## + ## - The time needed for a replica replicating to a wrong primary according + ## to a Sentinel current configuration, to be forced to replicate + ## with the right primary, is exactly the failover timeout (counting since + ## the moment a Sentinel detected the misconfiguration). + ## + ## - The time needed to cancel a failover that is already in progress but + ## did not produced any configuration change (REPLICAOF NO ONE yet not + ## acknowledged by the promoted replica). + ## + ## - The maximum time a failover in progress waits for all the replica to be + ## reconfigured as replicas of the new primary. However even after this time + ## the replicas will be reconfigured by the Sentinels anyway, but not with + ## the exact parallel-syncs progression as specified. + #sentinel['failover_timeout'] = 60000 + + ## Enable service discovery for Prometheus + consul['enable'] = true + consul['monitoring_service_discovery'] = true + + ## The IPs of the Consul server nodes + ## You can also use FQDNs and intermix them with IPs + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), + } + + # Set the network addresses that the exporters will listen on + node_exporter['listen_address'] = '0.0.0.0:9100' + redis_exporter['listen_address'] = '0.0.0.0:9121' + + # Disable auto migrations + gitlab_rails['auto_migrate'] = false + ``` + +1. To prevent database migrations from running on upgrade, run: + + ```shell + sudo touch /etc/gitlab/skip-auto-reconfigure + ``` + + Only the primary GitLab application server should handle migrations. + +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace + the file of the same name on this server. If that file is not on this server, + add the file from your Consul server to this server. + +1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. +1. Go through the steps again for all the other Sentinel nodes, and + make sure you set up the correct IPs. + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure Gitaly + +Deploying Gitaly in its own server can benefit GitLab installations that are +larger than a single machine. + +The Gitaly node requirements are dependent on customer data, specifically the number of +projects and their repository sizes. Two nodes are recommended as an absolute minimum. +Each Gitaly node should store no more than 5TB of data and have the number of +[`gitaly-ruby` workers](../gitaly/index.md#gitaly-ruby) set to 20% of available CPUs. +Additional nodes should be considered in conjunction with a review of expected +data size and spread based on the recommendations above. + +It is also strongly recommended that all Gitaly nodes be set up with SSD disks with +a throughput of at least 8,000 IOPS for read operations and 2,000 IOPS for write, +as Gitaly has heavy I/O. These IOPS values are recommended only as a starter as with +time they may be adjusted higher or lower depending on the scale of your environment's workload. +If you're running the environment on a Cloud provider, you may need to refer to +their documentation on how to configure IOPS correctly. + +Some things to note: + +- The GitLab Rails application shards repositories into [repository storages](../repository_storage_paths.md). +- A Gitaly server can host one or more storages. +- A GitLab server can use one or more Gitaly servers. +- Gitaly addresses must be specified in such a way that they resolve + correctly for ALL Gitaly clients. +- Gitaly servers must not be exposed to the public internet, as Gitaly's network + traffic is unencrypted by default. The use of a firewall is highly recommended + to restrict access to the Gitaly server. Another option is to + [use TLS](#gitaly-tls-support). + +TIP: **Tip:** +For more information about Gitaly's history and network architecture see the +[standalone Gitaly documentation](../gitaly/index.md). + +Note: **Note:** +The token referred to throughout the Gitaly documentation is +just an arbitrary password selected by the administrator. It is unrelated to +tokens created for the GitLab API or other similar web API tokens. + +Below we describe how to configure two Gitaly servers, with IPs and +domain names: + +- `10.6.0.91`: Gitaly 1 (`gitaly1.internal`) +- `10.6.0.92`: Gitaly 2 (`gitaly2.internal`) + +The secret token is assumed to be `gitalysecret` and that +your GitLab installation has three repository storages: + +- `default` on Gitaly 1 +- `storage1` on Gitaly 1 +- `storage2` on Gitaly 2 + +On each node: + +1. [Download/Install](https://about.gitlab.com/install/) the Omnibus GitLab + package you want using **steps 1 and 2** from the GitLab downloads page but + **without** providing the `EXTERNAL_URL` value. +1. Edit `/etc/gitlab/gitlab.rb` to configure storage paths, enable + the network listener and configure the token: + + <!-- + updates to following example must also be made at + https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab + --> + + ```ruby + # /etc/gitlab/gitlab.rb + + # Gitaly and GitLab use two shared secrets for authentication, one to authenticate gRPC requests + # to Gitaly, and a second for authentication callbacks from GitLab-Shell to the GitLab internal API. + # The following two values must be the same as their respective values + # of the GitLab Rails application setup + gitaly['auth_token'] = 'gitalysecret' + gitlab_shell['secret_token'] = 'shellsecret' + + # Avoid running unnecessary services on the Gitaly server + postgresql['enable'] = false + redis['enable'] = false + nginx['enable'] = false + puma['enable'] = false + unicorn['enable'] = false + sidekiq['enable'] = false + gitlab_workhorse['enable'] = false + grafana['enable'] = false + + # If you run a seperate monitoring node you can disable these services + alertmanager['enable'] = false + prometheus['enable'] = false + + # Prevent database connections during 'gitlab-ctl reconfigure' + gitlab_rails['rake_cache_clear'] = false + gitlab_rails['auto_migrate'] = false + + # Configure the gitlab-shell API callback URL. Without this, `git push` will + # fail. This can be your 'front door' GitLab URL or an internal load + # balancer. + # Don't forget to copy `/etc/gitlab/gitlab-secrets.json` from web server to Gitaly server. + gitlab_rails['internal_api_url'] = 'https://gitlab.example.com' + + # Make Gitaly accept connections on all network interfaces. You must use + # firewalls to restrict access to this address/port. + # Comment out following line if you only want to support TLS connections + gitaly['listen_addr'] = "0.0.0.0:8075" + ``` + +1. Append the following to `/etc/gitlab/gitlab.rb` for each respective server: + 1. On `gitaly1.internal`: + + ```ruby + git_data_dirs({ + 'default' => { + 'path' => '/var/opt/gitlab/git-data' + }, + 'storage1' => { + 'path' => '/mnt/gitlab/git-data' + }, + }) + ``` + + 1. On `gitaly2.internal`: + + ```ruby + git_data_dirs({ + 'storage2' => { + 'path' => '/mnt/gitlab/git-data' + }, + }) + ``` + + <!-- + updates to following example must also be made at + https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab + --> + +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace + the file of the same name on this server. If that file is not on this server, + add the file from your Consul server to this server. + +1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). + +### Gitaly TLS support + +Gitaly supports TLS encryption. To be able to communicate +with a Gitaly instance that listens for secure connections you will need to use `tls://` URL +scheme in the `gitaly_address` of the corresponding storage entry in the GitLab configuration. + +You will need to bring your own certificates as this isn't provided automatically. +The certificate, or its certificate authority, must be installed on all Gitaly +nodes (including the Gitaly node using the certificate) and on all client nodes +that communicate with it following the procedure described in +[GitLab custom certificate configuration](https://docs.gitlab.com/omnibus/settings/ssl.html#install-custom-public-certificates). + +NOTE: **Note:** +The self-signed certificate must specify the address you use to access the +Gitaly server. If you are addressing the Gitaly server by a hostname, you can +either use the Common Name field for this, or add it as a Subject Alternative +Name. If you are addressing the Gitaly server by its IP address, you must add it +as a Subject Alternative Name to the certificate. +[gRPC does not support using an IP address as Common Name in a certificate](https://github.com/grpc/grpc/issues/2691). + +NOTE: **Note:** +It is possible to configure Gitaly servers with both an +unencrypted listening address `listen_addr` and an encrypted listening +address `tls_listen_addr` at the same time. This allows you to do a +gradual transition from unencrypted to encrypted traffic, if necessary. + +To configure Gitaly with TLS: + +1. Create the `/etc/gitlab/ssl` directory and copy your key and certificate there: + + ```shell + sudo mkdir -p /etc/gitlab/ssl + sudo chmod 755 /etc/gitlab/ssl + sudo cp key.pem cert.pem /etc/gitlab/ssl/ + sudo chmod 644 key.pem cert.pem + ``` + +1. Copy the cert to `/etc/gitlab/trusted-certs` so Gitaly will trust the cert when + calling into itself: + + ```shell + sudo cp /etc/gitlab/ssl/cert.pem /etc/gitlab/trusted-certs/ + ``` + +1. Edit `/etc/gitlab/gitlab.rb` and add: + + <!-- + updates to following example must also be made at + https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab + --> + + ```ruby + gitaly['tls_listen_addr'] = "0.0.0.0:9999" + gitaly['certificate_path'] = "/etc/gitlab/ssl/cert.pem" + gitaly['key_path'] = "/etc/gitlab/ssl/key.pem" + ``` + +1. Delete `gitaly['listen_addr']` to allow only encrypted connections. + +1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure Sidekiq + +Sidekiq requires connections to the Redis, PostgreSQL and Gitaly instances. +The following IPs will be used as an example: + +- `10.6.0.101`: Sidekiq 1 +- `10.6.0.102`: Sidekiq 2 +- `10.6.0.103`: Sidekiq 3 +- `10.6.0.104`: Sidekiq 4 + +To configure the Sidekiq nodes, on each one: + +1. SSH into the Sidekiq server. +1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab package +you want using steps 1 and 2 from the GitLab downloads page. +**Do not complete any other steps on the download page.** +1. Open `/etc/gitlab/gitlab.rb` with your editor: + + ```ruby + ######################################## + ##### Services Disabled ### + ######################################## + + nginx['enable'] = false + grafana['enable'] = false + prometheus['enable'] = false + alertmanager['enable'] = false + gitaly['enable'] = false + gitlab_workhorse['enable'] = false + nginx['enable'] = false + puma['enable'] = false + postgres_exporter['enable'] = false + postgresql['enable'] = false + redis['enable'] = false + redis_exporter['enable'] = false + gitlab_exporter['enable'] = false + + ######################################## + #### Redis ### + ######################################## + + ## Redis connection details + ## First cluster that will host the cache + gitlab_rails['redis_cache_instance'] = 'redis://:<REDIS_PRIMARY_PASSWORD_OF_FIRST_CLUSTER>@gitlab-redis-cache' + + gitlab_rails['redis_cache_sentinels'] = [ + {host: '10.6.0.71', port: 26379}, + {host: '10.6.0.72', port: 26379}, + {host: '10.6.0.73', port: 26379}, + ] + + ## Second cluster that will host the queues, shared state, and actioncable + gitlab_rails['redis_queues_instance'] = 'redis://:<REDIS_PRIMARY_PASSWORD_OF_SECOND_CLUSTER>@gitlab-redis-persistent' + gitlab_rails['redis_shared_state_instance'] = 'redis://:<REDIS_PRIMARY_PASSWORD_OF_SECOND_CLUSTER>@gitlab-redis-persistent' + gitlab_rails['redis_actioncable_instance'] = 'redis://:<REDIS_PRIMARY_PASSWORD_OF_SECOND_CLUSTER>@gitlab-redis-persistent' + + gitlab_rails['redis_queues_sentinels'] = [ + {host: '10.6.0.81', port: 26379}, + {host: '10.6.0.82', port: 26379}, + {host: '10.6.0.83', port: 26379}, + ] + gitlab_rails['redis_shared_state_sentinels'] = [ + {host: '10.6.0.81', port: 26379}, + {host: '10.6.0.82', port: 26379}, + {host: '10.6.0.83', port: 26379}, + ] + gitlab_rails['redis_actioncable_sentinels'] = [ + {host: '10.6.0.81', port: 26379}, + {host: '10.6.0.82', port: 26379}, + {host: '10.6.0.83', port: 26379}, + ] + + ####################################### + ### Gitaly ### + ####################################### + + git_data_dirs({ + 'default' => { 'gitaly_address' => 'tcp://gitaly1.internal:8075' }, + 'storage1' => { 'gitaly_address' => 'tcp://gitaly1.internal:8075' }, + 'storage2' => { 'gitaly_address' => 'tcp://gitaly2.internal:8075' }, + }) + gitlab_rails['gitaly_token'] = 'YOUR_TOKEN' + + ####################################### + ### Postgres ### + ####################################### + gitlab_rails['db_host'] = '10.6.0.20' # internal load balancer IP + gitlab_rails['db_port'] = 6432 + gitlab_rails['db_password'] = '<postgresql_user_password>' + gitlab_rails['db_adapter'] = 'postgresql' + gitlab_rails['db_encoding'] = 'unicode' + gitlab_rails['auto_migrate'] = false + + ####################################### + ### Sidekiq configuration ### + ####################################### + sidekiq['listen_address'] = "0.0.0.0" + sidekiq['cluster'] = true # no need to set this after GitLab 13.0 + + ####################################### + ### Monitoring configuration ### + ####################################### + consul['enable'] = true + consul['monitoring_service_discovery'] = true + + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13) + } + + # Set the network addresses that the exporters will listen on + node_exporter['listen_address'] = '0.0.0.0:9100' + + # Rails Status for prometheus + gitlab_rails['monitoring_whitelist'] = ['10.6.0.121/32', '127.0.0.0/8'] + ``` + +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace + the file of the same name on this server. If that file is not on this server, + add the file from your Consul server to this server. + +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + +TIP: **Tip:** +You can also run [multiple Sidekiq processes](../operations/extra_sidekiq_processes.md). + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure GitLab Rails + +NOTE: **Note:** +In our architectures we run each GitLab Rails node using the Puma webserver +and have its number of workers set to 90% of available CPUs along with four threads. For +nodes that are running Rails with other components the worker value should be reduced +accordingly where we've found 50% achieves a good balance but this is dependent +on workload. + +This section describes how to configure the GitLab application (Rails) component. + +The following IPs will be used as an example: + +- `10.6.0.111`: GitLab application 1 +- `10.6.0.112`: GitLab application 2 +- `10.6.0.113`: GitLab application 3 + +On each node perform the following: + +1. Download/install Omnibus GitLab using **steps 1 and 2** from + [GitLab downloads](https://about.gitlab.com/install/). Do not complete other + steps on the download page. + +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace + the file of the same name on this server. If that file is not on this server, + add the file from your Consul server to this server. + +1. Edit `/etc/gitlab/gitlab.rb` and use the following configuration. + To maintain uniformity of links across nodes, the `external_url` + on the application server should point to the external URL that users will use + to access GitLab. This would be the URL of the [external load balancer](#configure-the-external-load-balancer) + which will route traffic to the GitLab application server: + + ```ruby + external_url 'https://gitlab.example.com' + + # Gitaly and GitLab use two shared secrets for authentication, one to authenticate gRPC requests + # to Gitaly, and a second for authentication callbacks from GitLab-Shell to the GitLab internal API. + # The following two values must be the same as their respective values + # of the Gitaly setup + gitlab_rails['gitaly_token'] = 'gitalysecret' + gitlab_shell['secret_token'] = 'shellsecret' + + git_data_dirs({ + 'default' => { 'gitaly_address' => 'tcp://gitaly1.internal:8075' }, + 'storage1' => { 'gitaly_address' => 'tcp://gitaly1.internal:8075' }, + 'storage2' => { 'gitaly_address' => 'tcp://gitaly2.internal:8075' }, + }) + + ## Disable components that will not be on the GitLab application server + roles ['application_role'] + gitaly['enable'] = false + nginx['enable'] = true + + ## PostgreSQL connection details + # Disable PostgreSQL on the application node + postgresql['enable'] = false + gitlab_rails['db_host'] = '10.6.0.20' # internal load balancer IP + gitlab_rails['db_port'] = 6432 + gitlab_rails['db_password'] = '<postgresql_user_password>' + gitlab_rails['auto_migrate'] = false + + ## Redis connection details + ## First cluster that will host the cache + gitlab_rails['redis_cache_instance'] = 'redis://:<REDIS_PRIMARY_PASSWORD_OF_FIRST_CLUSTER>@gitlab-redis-cache' + + gitlab_rails['redis_cache_sentinels'] = [ + {host: '10.6.0.71', port: 26379}, + {host: '10.6.0.72', port: 26379}, + {host: '10.6.0.73', port: 26379}, + ] + + ## Second cluster that will host the queues, shared state, and actionable + gitlab_rails['redis_queues_instance'] = 'redis://:<REDIS_PRIMARY_PASSWORD_OF_SECOND_CLUSTER>@gitlab-redis-persistent' + gitlab_rails['redis_shared_state_instance'] = 'redis://:<REDIS_PRIMARY_PASSWORD_OF_SECOND_CLUSTER>@gitlab-redis-persistent' + gitlab_rails['redis_actioncable_instance'] = 'redis://:<REDIS_PRIMARY_PASSWORD_OF_SECOND_CLUSTER>@gitlab-redis-persistent' + + gitlab_rails['redis_queues_sentinels'] = [ + {host: '10.6.0.81', port: 26379}, + {host: '10.6.0.82', port: 26379}, + {host: '10.6.0.83', port: 26379}, + ] + gitlab_rails['redis_shared_state_sentinels'] = [ + {host: '10.6.0.81', port: 26379}, + {host: '10.6.0.82', port: 26379}, + {host: '10.6.0.83', port: 26379}, + ] + gitlab_rails['redis_actioncable_sentinels'] = [ + {host: '10.6.0.81', port: 26379}, + {host: '10.6.0.82', port: 26379}, + {host: '10.6.0.83', port: 26379}, + ] + + # Set the network addresses that the exporters used for monitoring will listen on + node_exporter['listen_address'] = '0.0.0.0:9100' + gitlab_workhorse['prometheus_listen_addr'] = '0.0.0.0:9229' + sidekiq['listen_address'] = "0.0.0.0" + puma['listen'] = '0.0.0.0' + + # Add the monitoring node's IP address to the monitoring whitelist and allow it to + # scrape the NGINX metrics + gitlab_rails['monitoring_whitelist'] = ['10.6.0.121/32', '127.0.0.0/8'] + nginx['status']['options']['allow'] = ['10.6.0.121/32', '127.0.0.0/8'] + ``` + +1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). +1. If you're using [Gitaly with TLS support](#gitaly-tls-support), make sure the + `git_data_dirs` entry is configured with `tls` instead of `tcp`: + + ```ruby + git_data_dirs({ + 'default' => { 'gitaly_address' => 'tls://gitaly1.internal:9999' }, + 'storage1' => { 'gitaly_address' => 'tls://gitaly1.internal:9999' }, + 'storage2' => { 'gitaly_address' => 'tls://gitaly2.internal:9999' }, + }) + ``` + + 1. Copy the cert into `/etc/gitlab/trusted-certs`: + + ```shell + sudo cp cert.pem /etc/gitlab/trusted-certs/ + ``` + +1. If you're [using NFS](#configure-nfs-optional): + 1. If necessary, install the NFS client utility packages using the following + commands: + + ```shell + # Ubuntu/Debian + apt-get install nfs-common + + # CentOS/Red Hat + yum install nfs-utils nfs-utils-lib + ``` + + 1. Specify the necessary NFS mounts in `/etc/fstab`. + The exact contents of `/etc/fstab` will depend on how you chose + to configure your NFS server. See the [NFS documentation](../high_availability/nfs.md) + for examples and the various options. + + 1. Create the shared directories. These may be different depending on your NFS + mount locations. + + ```shell + mkdir -p /var/opt/gitlab/.ssh /var/opt/gitlab/gitlab-rails/uploads /var/opt/gitlab/gitlab-rails/shared /var/opt/gitlab/gitlab-ci/builds /var/opt/gitlab/git-data + ``` + + 1. Edit `/etc/gitlab/gitlab.rb` and use the following configuration: + + ```ruby + ## Prevent GitLab from starting if NFS data mounts are not available + high_availability['mountpoint'] = '/var/opt/gitlab/git-data' + + ## Ensure UIDs and GIDs match between servers for permissions via NFS + user['uid'] = 9000 + user['gid'] = 9000 + web_server['uid'] = 9001 + web_server['gid'] = 9001 + registry['uid'] = 9002 + registry['gid'] = 9002 + ``` + +1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). +1. Confirm the node can connect to Gitaly: + + ```shell + sudo gitlab-rake gitlab:gitaly:check + ``` + + Then, tail the logs to see the requests: + + ```shell + sudo gitlab-ctl tail gitaly + ``` + +1. Optionally, from the Gitaly servers, confirm that Gitaly can perform callbacks to the internal API: + + ```shell + sudo /opt/gitlab/embedded/service/gitlab-shell/bin/check -config /opt/gitlab/embedded/service/gitlab-shell/config.yml + ``` + +NOTE: **Note:** +When you specify `https` in the `external_url`, as in the example +above, GitLab assumes you have SSL certificates in `/etc/gitlab/ssl/`. If +certificates are not present, NGINX will fail to start. See the +[NGINX documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https) +for more information. + +### GitLab Rails post-configuration + +Initialize the GitLab database, by running the following in one of the Rails nodes: + +```shell +sudo gitlab-rake gitlab:db:configure +``` + +NOTE: **Note:** +If you encounter a `rake aborted!` error stating that PgBouncer is failing to connect to +PostgreSQL it may be that your PgBouncer node's IP address is missing from +PostgreSQL's `trust_auth_cidr_addresses` in `gitlab.rb` on your database nodes. See +[PgBouncer error `ERROR: pgbouncer cannot connect to server`](troubleshooting.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server) +in the Troubleshooting section before proceeding. + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure Prometheus + +The Omnibus GitLab package can be used to configure a standalone Monitoring node +running [Prometheus](../monitoring/prometheus/index.md) and +[Grafana](../monitoring/performance/grafana_configuration.md). + +The following IP will be used as an example: + +- `10.6.0.121`: Prometheus + +To configure the Monitoring node: + +1. SSH into the Monitoring node. +1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab + package you want using **steps 1 and 2** from the GitLab downloads page. + Do not complete any other steps on the download page. + +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace + the file of the same name on this server. If that file is not on this server, + add the file from your Consul server to this server. + +1. Edit `/etc/gitlab/gitlab.rb` and add the contents: + + ```ruby + external_url 'http://gitlab.example.com' + + # Disable all other services + gitlab_rails['auto_migrate'] = false + alertmanager['enable'] = false + gitaly['enable'] = false + gitlab_exporter['enable'] = false + gitlab_workhorse['enable'] = false + nginx['enable'] = true + postgres_exporter['enable'] = false + postgresql['enable'] = false + redis['enable'] = false + redis_exporter['enable'] = false + sidekiq['enable'] = false + puma['enable'] = false + unicorn['enable'] = false + node_exporter['enable'] = false + gitlab_exporter['enable'] = false + + # Enable Prometheus + prometheus['enable'] = true + prometheus['listen_address'] = '0.0.0.0:9090' + prometheus['monitor_kubernetes'] = false + + # Enable Login form + grafana['disable_login_form'] = false + + # Enable Grafana + grafana['enable'] = true + grafana['admin_password'] = '<grafana_password>' + + # Enable service discovery for Prometheus + consul['enable'] = true + consul['monitoring_service_discovery'] = true + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13) + } + ``` + +1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). +1. In the GitLab UI, set `admin/application_settings/metrics_and_profiling` > Metrics - Grafana to `/-/grafana` to +`http[s]://<MONITOR NODE>/-/grafana` + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure the object storage + +GitLab supports using an object storage service for holding numerous types of data. +It's recommended over [NFS](#configure-nfs-optional) and in general it's better +in larger setups as object storage is typically much more performant, reliable, +and scalable. + +Object storage options that GitLab has tested, or is aware of customers using include: + +- SaaS/Cloud solutions such as [Amazon S3](https://aws.amazon.com/s3/), [Google cloud storage](https://cloud.google.com/storage). +- On-premises hardware and appliances from various storage vendors. +- MinIO. There is [a guide to deploying this](https://docs.gitlab.com/charts/advanced/external-object-storage/minio.html) within our Helm Chart documentation. + +For configuring GitLab to use Object Storage refer to the following guides +based on what features you intend to use: + +1. Configure [object storage for backups](../../raketasks/backup_restore.md#uploading-backups-to-a-remote-cloud-storage). +1. Configure [object storage for job artifacts](../job_artifacts.md#using-object-storage) + including [incremental logging](../job_logs.md#new-incremental-logging-architecture). +1. Configure [object storage for LFS objects](../lfs/index.md#storing-lfs-objects-in-remote-object-storage). +1. Configure [object storage for uploads](../uploads.md#using-object-storage-core-only). +1. Configure [object storage for merge request diffs](../merge_request_diffs.md#using-object-storage). +1. Configure [object storage for Container Registry](../packages/container_registry.md#use-object-storage) (optional feature). +1. Configure [object storage for Mattermost](https://docs.mattermost.com/administration/config-settings.html#file-storage) (optional feature). +1. Configure [object storage for packages](../packages/index.md#using-object-storage) (optional feature). **(PREMIUM ONLY)** +1. Configure [object storage for Dependency Proxy](../packages/dependency_proxy.md#using-object-storage) (optional feature). **(PREMIUM ONLY)** +1. Configure [object storage for Pseudonymizer](../pseudonymizer.md#configuration) (optional feature). **(ULTIMATE ONLY)** +1. Configure [object storage for autoscale Runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional - for improved performance). +1. Configure [object storage for Terraform state files](../terraform_state.md#using-object-storage-core-only). + +Using separate buckets for each data type is the recommended approach for GitLab. + +A limitation of our configuration is that each use of object storage is separately configured. +[We have an issue for improving this](https://gitlab.com/gitlab-org/gitlab/-/issues/23345) +and easily using one bucket with separate folders is one improvement that this might bring. + +There is at least one specific issue with using the same bucket: +when GitLab is deployed with the Helm chart restore from backup +[will not properly function](https://docs.gitlab.com/charts/advanced/external-object-storage/#lfs-artifacts-uploads-packages-external-diffs-pseudonymizer) +unless separate buckets are used. + +One risk of using a single bucket would be if your organization decided to +migrate GitLab to the Helm deployment in the future. GitLab would run, but the situation with +backups might not be realized until the organization had a critical requirement for the backups to +work. + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure NFS (optional) + +[Object storage](#configure-the-object-storage), along with [Gitaly](#configure-gitaly) +are recommended over NFS wherever possible for improved performance. If you intend +to use GitLab Pages, this currently [requires NFS](troubleshooting.md#gitlab-pages-requires-nfs). + +See how to [configure NFS](../high_availability/nfs.md). + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Troubleshooting + +See the [troubleshooting documentation](troubleshooting.md). + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> diff --git a/doc/administration/reference_architectures/1k_users.md b/doc/administration/reference_architectures/1k_users.md index def23619a5c..d3cf5f49413 100644 --- a/doc/administration/reference_architectures/1k_users.md +++ b/doc/administration/reference_architectures/1k_users.md @@ -1,86 +1,49 @@ -# Reference architecture: up to 1,000 users +--- +stage: Enablement +group: Distribution +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- -This page describes GitLab reference architecture for up to 1,000 users. -For a full list of reference architectures, see -[Available reference architectures](index.md#available-reference-architectures). - -> - **Supported users (approximate):** 1,000 -> - **High Availability:** False +# Reference architecture: up to 1,000 users **(CORE ONLY)** -| Users | Configuration([8](#footnotes)) | GCP | AWS | Azure | -|-------|------------------------------------|----------------|---------------------|------------------------| -| 500 | 4 vCPU, 3.6GB Memory | `n1-highcpu-4` | `c5.xlarge` | F4s v2 | -| 1000 | 8 vCPU, 7.2GB Memory | `n1-highcpu-8` | `c5.2xlarge` | F8s v2 | +This page describes GitLab reference architecture for up to 1,000 users. For a +full list of reference architectures, see +[Available reference architectures](index.md#available-reference-architectures). -In addition to the above, we recommend having at least -2GB of swap on your server, even if you currently have -enough available RAM. Having swap will help reduce the chance of errors occurring -if your available memory changes. We also recommend -configuring the kernel's swappiness setting -to a low value like `10` to make the most of your RAM while still having the swap -available when needed. +If you need to serve up to 1,000 users and you don't have strict availability +requirements, a single-node solution with +[frequent backups](index.md#automated-backups-core-only) is appropriate for +many organizations . -For situations where you need to serve up to 1,000 users, a single-node -solution with [frequent backups](index.md#automated-backups-core-only) is appropriate -for many organizations. With automatic backup of the GitLab repositories, -configuration, and the database, if you don't have strict availability -requirements, this is the ideal solution. +> - **Supported users (approximate):** 1,000 +> - **High Availability:** No + +| Users | Configuration | GCP | AWS | Azure | +|--------------|-------------------------|----------------|-----------------|----------------| +| Up to 500 | 4 vCPU, 3.6GB memory | n1-highcpu-4 | c5.xlarge | F4s v2 | +| Up to 1,000 | 8 vCPU, 7.2GB memory | n1-highcpu-8 | c5.2xlarge | F8s v2 | + +The Google Cloud Platform (GCP) architectures were built and tested using the +[Intel Xeon E5 v3 (Haswell)](https://cloud.google.com/compute/docs/cpu-platforms) +CPU platform. On different hardware you may find that adjustments, either lower +or higher, are required for your CPU or node counts. For more information, see +our [Sysbench](https://github.com/akopytov/sysbench)-based +[CPU benchmark](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Reference-Architectures/GCP-CPU-Benchmarks). + +In addition to the stated configurations, we recommend having at least 2GB of +swap on your server, even if you currently have enough available memory. Having +swap will help reduce the chance of errors occurring if your available memory +changes. We also recommend configuring the kernel's swappiness setting to a +lower value (such as `10`) to make the most of your memory, while still having +the swap available when needed. ## Setup instructions -- For this default reference architecture, use the standard [installation instructions](../../install/README.md) to install GitLab. +For this default reference architecture, to install GitLab use the standard +[installation instructions](../../install/README.md). NOTE: **Note:** You can also optionally configure GitLab to use an [external PostgreSQL service](../postgresql/external.md) or an [external object storage service](../high_availability/object_storage.md) for added performance and reliability at a reduced complexity cost. - -## Footnotes - -1. In our architectures we run each GitLab Rails node using the Puma webserver - and have its number of workers set to 90% of available CPUs along with four threads. For - nodes that are running Rails with other components the worker value should be reduced - accordingly where we've found 50% achieves a good balance but this is dependent - on workload. - -1. Gitaly node requirements are dependent on customer data, specifically the number of - projects and their sizes. We recommend two nodes as an absolute minimum for HA environments - and at least four nodes should be used when supporting 50,000 or more users. - We also recommend that each Gitaly node should store no more than 5TB of data - and have the number of [`gitaly-ruby` workers](../gitaly/index.md#gitaly-ruby) - set to 20% of available CPUs. Additional nodes should be considered in conjunction - with a review of expected data size and spread based on the recommendations above. - -1. Recommended Redis setup differs depending on the size of the architecture. - For smaller architectures (less than 3,000 users) a single instance should suffice. - For medium sized installs (3,000 - 5,000) we suggest one Redis cluster for all - classes and that Redis Sentinel is hosted alongside Consul. - For larger architectures (10,000 users or more) we suggest running a separate - [Redis Cluster](../redis/replication_and_failover.md#running-multiple-redis-clusters) for the Cache class - and another for the Queues and Shared State classes respectively. We also recommend - that you run the Redis Sentinel clusters separately for each Redis Cluster. - -1. For data objects such as LFS, Uploads, Artifacts, etc. We recommend an [Object Storage service](../object_storage.md) - over NFS where possible, due to better performance and availability. - -1. NFS can be used as an alternative for both repository data (replacing Gitaly) and - object storage but this isn't typically recommended for performance reasons. Note however it is required for - [GitLab Pages](https://gitlab.com/gitlab-org/gitlab-pages/-/issues/196). - -1. Our architectures have been tested and validated with [HAProxy](https://www.haproxy.org/) - as the load balancer. Although other load balancers with similar feature sets - could also be used, those load balancers have not been validated. - -1. We strongly recommend that any Gitaly or NFS nodes be set up with SSD disks over - HDD with a throughput of at least 8,000 IOPS for read operations and 2,000 IOPS for write - as these components have heavy I/O. These IOPS values are recommended only as a starter - as with time they may be adjusted higher or lower depending on the scale of your - environment's workload. If you're running the environment on a Cloud provider - you may need to refer to their documentation on how configure IOPS correctly. - -1. The architectures were built and tested with the [Intel Xeon E5 v3 (Haswell)](https://cloud.google.com/compute/docs/cpu-platforms) - CPU platform on GCP. On different hardware you may find that adjustments, either lower - or higher, are required for your CPU or Node counts accordingly. For more information, a - [Sysbench](https://github.com/akopytov/sysbench) benchmark of the CPU can be found - [here](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Reference-Architectures/GCP-CPU-Benchmarks). diff --git a/doc/administration/reference_architectures/25k_users.md b/doc/administration/reference_architectures/25k_users.md index 17f4300eb03..1cfa2565893 100644 --- a/doc/administration/reference_architectures/25k_users.md +++ b/doc/administration/reference_architectures/25k_users.md @@ -1,76 +1,2047 @@ -# Reference architecture: up to 25,000 users +--- +reading_time: true +stage: Enablement +group: Distribution +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- -This page describes GitLab reference architecture for up to 25,000 users. -For a full list of reference architectures, see +# Reference architecture: up to 25,000 users **(PREMIUM ONLY)** + +This page describes GitLab reference architecture for up to 25,000 users. For a +full list of reference architectures, see [Available reference architectures](index.md#available-reference-architectures). > - **Supported users (approximate):** 25,000 -> - **High Availability:** True -> - **Test RPS rates:** API: 500 RPS, Web: 50 RPS, Git: 50 RPS - -| Service | Nodes | Configuration ([8](#footnotes)) | GCP | AWS | Azure | -|--------------------------------------------------------------|-------|---------------------------------|------------------|-----------------------|----------------| -| GitLab Rails ([1](#footnotes)) | 5 | 32 vCPU, 28.8GB Memory | `n1-highcpu-32` | `c5.9xlarge` | F32s v2 | -| PostgreSQL | 3 | 8 vCPU, 30GB Memory | `n1-standard-8` | `m5.2xlarge` | D8s v3 | -| PgBouncer | 3 | 2 vCPU, 1.8GB Memory | `n1-highcpu-2` | `c5.large` | F2s v2 | -| Gitaly ([2](#footnotes)) ([5](#footnotes)) ([7](#footnotes)) | X | 32 vCPU, 120GB Memory | `n1-standard-32` | `m5.8xlarge` | D32s v3 | -| Redis ([3](#footnotes)) - Cache | 3 | 4 vCPU, 15GB Memory | `n1-standard-4` | `m5.xlarge` | D4s v3 | -| Redis ([3](#footnotes)) - Queues / Shared State | 3 | 4 vCPU, 15GB Memory | `n1-standard-4` | `m5.xlarge` | D4s v3 | -| Redis Sentinel ([3](#footnotes)) - Cache | 3 | 1 vCPU, 1.7GB Memory | `g1-small` | `t2.small` | B1MS | -| Redis Sentinel ([3](#footnotes)) - Queues / Shared State | 3 | 1 vCPU, 1.7GB Memory | `g1-small` | `t2.small` | B1MS | -| Consul | 3 | 2 vCPU, 1.8GB Memory | `n1-highcpu-2` | `c5.large` | F2s v2 | -| Sidekiq | 4 | 4 vCPU, 15GB Memory | `n1-standard-4` | `m5.xlarge` | D4s v3 | -| Object Storage ([4](#footnotes)) | - | - | - | - | - | -| NFS Server ([5](#footnotes)) ([7](#footnotes)) | 1 | 4 vCPU, 3.6GB Memory | `n1-highcpu-4` | `c5.xlarge` | F4s v2 | -| Monitoring node | 1 | 4 vCPU, 3.6GB Memory | `n1-highcpu-4` | `c5.xlarge` | F4s v2 | -| External load balancing node ([6](#footnotes)) | 1 | 4 vCPU, 3.6GB Memory | `n1-highcpu-4` | `c5.xlarge` | F4s v2 | -| Internal load balancing node ([6](#footnotes)) | 1 | 4 vCPU, 3.6GB Memory | `n1-highcpu-4` | `c5.xlarge` | F4s v2 | - -## Footnotes - -1. In our architectures we run each GitLab Rails node using the Puma webserver - and have its number of workers set to 90% of available CPUs along with four threads. For - nodes that are running Rails with other components the worker value should be reduced - accordingly where we've found 50% achieves a good balance but this is dependent - on workload. - -1. Gitaly node requirements are dependent on customer data, specifically the number of - projects and their sizes. We recommend two nodes as an absolute minimum for HA environments - and at least four nodes should be used when supporting 50,000 or more users. - We also recommend that each Gitaly node should store no more than 5TB of data - and have the number of [`gitaly-ruby` workers](../gitaly/index.md#gitaly-ruby) - set to 20% of available CPUs. Additional nodes should be considered in conjunction - with a review of expected data size and spread based on the recommendations above. - -1. Recommended Redis setup differs depending on the size of the architecture. - For smaller architectures (less than 3,000 users) a single instance should suffice. - For medium sized installs (3,000 - 5,000) we suggest one Redis cluster for all - classes and that Redis Sentinel is hosted alongside Consul. - For larger architectures (10,000 users or more) we suggest running a separate - [Redis Cluster](../redis/replication_and_failover.md#running-multiple-redis-clusters) for the Cache class - and another for the Queues and Shared State classes respectively. We also recommend - that you run the Redis Sentinel clusters separately for each Redis Cluster. - -1. For data objects such as LFS, Uploads, Artifacts, etc. We recommend an [Object Storage service](../object_storage.md) - over NFS where possible, due to better performance and availability. - -1. NFS can be used as an alternative for both repository data (replacing Gitaly) and - object storage but this isn't typically recommended for performance reasons. Note however it is required for - [GitLab Pages](https://gitlab.com/gitlab-org/gitlab-pages/-/issues/196). - -1. Our architectures have been tested and validated with [HAProxy](https://www.haproxy.org/) - as the load balancer. Although other load balancers with similar feature sets - could also be used, those load balancers have not been validated. - -1. We strongly recommend that any Gitaly or NFS nodes be set up with SSD disks over - HDD with a throughput of at least 8,000 IOPS for read operations and 2,000 IOPS for write - as these components have heavy I/O. These IOPS values are recommended only as a starter - as with time they may be adjusted higher or lower depending on the scale of your - environment's workload. If you're running the environment on a Cloud provider - you may need to refer to their documentation on how configure IOPS correctly. - -1. The architectures were built and tested with the [Intel Xeon E5 v3 (Haswell)](https://cloud.google.com/compute/docs/cpu-platforms) - CPU platform on GCP. On different hardware you may find that adjustments, either lower - or higher, are required for your CPU or Node counts accordingly. For more information, a - [Sysbench](https://github.com/akopytov/sysbench) benchmark of the CPU can be found - [here](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Reference-Architectures/GCP-CPU-Benchmarks). +> - **High Availability:** Yes +> - **Test requests per second (RPS) rates:** API: 500 RPS, Web: 50 RPS, Git: 50 RPS + +| Service | Nodes | Configuration | GCP | AWS | Azure | +|-----------------------------------------|-------------|-------------------------|-----------------|-------------|----------| +| External load balancing node | 1 | 4 vCPU, 3.6GB memory | n1-highcpu-4 | c5.xlarge | F4s v2 | +| Consul | 3 | 2 vCPU, 1.8GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| PostgreSQL | 3 | 8 vCPU, 30GB memory | n1-standard-8 | m5.2xlarge | D8s v3 | +| PgBouncer | 3 | 2 vCPU, 1.8GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| Internal load balancing node | 1 | 2 vCPU, 1.8GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| Redis - Cache | 3 | 4 vCPU, 15GB memory | n1-standard-4 | m5.xlarge | D4s v3 | +| Redis - Queues / Shared State | 3 | 4 vCPU, 15GB memory | n1-standard-4 | m5.xlarge | D4s v3 | +| Redis Sentinel - Cache | 3 | 1 vCPU, 1.7GB memory | g1-small | t2.small | B1MS | +| Redis Sentinel - Queues / Shared State | 3 | 1 vCPU, 1.7GB memory | g1-small | t2.small | B1MS | +| Gitaly | 2 (minimum) | 32 vCPU, 120GB memory | n1-standard-32 | m5.8xlarge | D32s v3 | +| Sidekiq | 4 | 4 vCPU, 15GB memory | n1-standard-4 | m5.xlarge | D4s v3 | +| GitLab Rails | 5 | 32 vCPU, 28.8GB memory | n1-highcpu-32 | c5.9xlarge | F32s v2 | +| Monitoring node | 1 | 4 vCPU, 3.6GB memory | n1-highcpu-4 | c5.xlarge | F4s v2 | +| Object Storage | n/a | n/a | n/a | n/a | n/a | +| NFS Server | 1 | 4 vCPU, 3.6GB memory | n1-highcpu-4 | c5.xlarge | F4s v2 | + +The Google Cloud Platform (GCP) architectures were built and tested using the +[Intel Xeon E5 v3 (Haswell)](https://cloud.google.com/compute/docs/cpu-platforms) +CPU platform. On different hardware you may find that adjustments, either lower +or higher, are required for your CPU or node counts. For more information, see +our [Sysbench](https://github.com/akopytov/sysbench)-based +[CPU benchmark](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Reference-Architectures/GCP-CPU-Benchmarks). + +For data objects (such as LFS, Uploads, or Artifacts), an +[object storage service](#configure-the-object-storage) is recommended instead +of NFS where possible, due to better performance and availability. Since this +doesn't require a node to be set up, *Object Storage* is noted as not +applicable (n/a) in the previous table. + +## Setup components + +To set up GitLab and its components to accommodate up to 25,000 users: + +1. [Configure the external load balancing node](#configure-the-external-load-balancer) + that will handle the load balancing of the three GitLab application services nodes. +1. [Configure Consul](#configure-consul). +1. [Configure PostgreSQL](#configure-postgresql), the database for GitLab. +1. [Configure PgBouncer](#configure-pgbouncer). +1. [Configure the internal load balancing node](#configure-the-internal-load-balancer) +1. [Configure Redis](#configure-redis). +1. [Configure Gitaly](#configure-gitaly), + which provides access to the Git repositories. +1. [Configure Sidekiq](#configure-sidekiq). +1. [Configure the main GitLab Rails application](#configure-gitlab-rails) + to run Puma/Unicorn, Workhorse, GitLab Shell, and to serve all frontend requests (UI, API, Git + over HTTP/SSH). +1. [Configure Prometheus](#configure-prometheus) to monitor your GitLab environment. +1. [Configure the Object Storage](#configure-the-object-storage) + used for shared data objects. +1. [Configure NFS (Optional)](#configure-nfs-optional) + to have shared disk storage service as an alternative to Gitaly and/or Object Storage (although + not recommended). NFS is required for GitLab Pages, you can skip this step if you're not using + that feature. + +We start with all servers on the same 10.6.0.0/24 private network range, they +can connect to each other freely on those addresses. + +Here is a list and description of each machine and the assigned IP: + +- `10.6.0.10`: External Load Balancer +- `10.6.0.11`: Consul 1 +- `10.6.0.12`: Consul 2 +- `10.6.0.13`: Consul 3 +- `10.6.0.21`: PostgreSQL primary +- `10.6.0.22`: PostgreSQL secondary 1 +- `10.6.0.23`: PostgreSQL secondary 2 +- `10.6.0.31`: PgBouncer 1 +- `10.6.0.32`: PgBouncer 2 +- `10.6.0.33`: PgBouncer 3 +- `10.6.0.40`: Internal Load Balancer +- `10.6.0.51`: Redis - Cache Primary +- `10.6.0.52`: Redis - Cache Replica 1 +- `10.6.0.53`: Redis - Cache Replica 2 +- `10.6.0.71`: Sentinel - Cache 1 +- `10.6.0.72`: Sentinel - Cache 2 +- `10.6.0.73`: Sentinel - Cache 3 +- `10.6.0.61`: Redis - Queues Primary +- `10.6.0.62`: Redis - Queues Replica 1 +- `10.6.0.63`: Redis - Queues Replica 2 +- `10.6.0.81`: Sentinel - Queues 1 +- `10.6.0.82`: Sentinel - Queues 2 +- `10.6.0.83`: Sentinel - Queues 3 +- `10.6.0.91`: Gitaly 1 +- `10.6.0.92`: Gitaly 2 +- `10.6.0.101`: Sidekiq 1 +- `10.6.0.102`: Sidekiq 2 +- `10.6.0.103`: Sidekiq 3 +- `10.6.0.104`: Sidekiq 4 +- `10.6.0.111`: GitLab application 1 +- `10.6.0.112`: GitLab application 2 +- `10.6.0.113`: GitLab application 3 +- `10.6.0.121`: Prometheus + +## Configure the external load balancer + +NOTE: **Note:** +This architecture has been tested and validated with [HAProxy](https://www.haproxy.org/) +as the load balancer. Although other load balancers with similar feature sets +could also be used, those load balancers have not been validated. + +In an active/active GitLab configuration, you will need a load balancer to route +traffic to the application servers. The specifics on which load balancer to use +or the exact configuration is beyond the scope of GitLab documentation. We hope +that if you're managing multi-node systems like GitLab you have a load balancer of +choice already. Some examples including HAProxy (open-source), F5 Big-IP LTM, +and Citrix Net Scaler. This documentation will outline what ports and protocols +you need to use with GitLab. + +The next question is how you will handle SSL in your environment. +There are several different options: + +- [The application node terminates SSL](#application-node-terminates-ssl). +- [The load balancer terminates SSL without backend SSL](#load-balancer-terminates-ssl-without-backend-ssl) + and communication is not secure between the load balancer and the application node. +- [The load balancer terminates SSL with backend SSL](#load-balancer-terminates-ssl-with-backend-ssl) + and communication is *secure* between the load balancer and the application node. + +### Application node terminates SSL + +Configure your load balancer to pass connections on port 443 as `TCP` rather +than `HTTP(S)` protocol. This will pass the connection to the application node's +NGINX service untouched. NGINX will have the SSL certificate and listen on port 443. + +See the [NGINX HTTPS documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https) +for details on managing SSL certificates and configuring NGINX. + +### Load balancer terminates SSL without backend SSL + +Configure your load balancer to use the `HTTP(S)` protocol rather than `TCP`. +The load balancer will then be responsible for managing SSL certificates and +terminating SSL. + +Since communication between the load balancer and GitLab will not be secure, +there is some additional configuration needed. See the +[NGINX proxied SSL documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#supporting-proxied-ssl) +for details. + +### Load balancer terminates SSL with backend SSL + +Configure your load balancer(s) to use the 'HTTP(S)' protocol rather than 'TCP'. +The load balancer(s) will be responsible for managing SSL certificates that +end users will see. + +Traffic will also be secure between the load balancer(s) and NGINX in this +scenario. There is no need to add configuration for proxied SSL since the +connection will be secure all the way. However, configuration will need to be +added to GitLab to configure SSL certificates. See +[NGINX HTTPS documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https) +for details on managing SSL certificates and configuring NGINX. + +### Ports + +The basic ports to be used are shown in the table below. + +| LB Port | Backend Port | Protocol | +| ------- | ------------ | ------------------------ | +| 80 | 80 | HTTP (*1*) | +| 443 | 443 | TCP or HTTPS (*1*) (*2*) | +| 22 | 22 | TCP | + +- (*1*): [Web terminal](../../ci/environments/index.md#web-terminals) support requires + your load balancer to correctly handle WebSocket connections. When using + HTTP or HTTPS proxying, this means your load balancer must be configured + to pass through the `Connection` and `Upgrade` hop-by-hop headers. See the + [web terminal](../integration/terminal.md) integration guide for + more details. +- (*2*): When using HTTPS protocol for port 443, you will need to add an SSL + certificate to the load balancers. If you wish to terminate SSL at the + GitLab application server instead, use TCP protocol. + +If you're using GitLab Pages with custom domain support you will need some +additional port configurations. +GitLab Pages requires a separate virtual IP address. Configure DNS to point the +`pages_external_url` from `/etc/gitlab/gitlab.rb` at the new virtual IP address. See the +[GitLab Pages documentation](../pages/index.md) for more information. + +| LB Port | Backend Port | Protocol | +| ------- | ------------- | --------- | +| 80 | Varies (*1*) | HTTP | +| 443 | Varies (*1*) | TCP (*2*) | + +- (*1*): The backend port for GitLab Pages depends on the + `gitlab_pages['external_http']` and `gitlab_pages['external_https']` + setting. See [GitLab Pages documentation](../pages/index.md) for more details. +- (*2*): Port 443 for GitLab Pages should always use the TCP protocol. Users can + configure custom domains with custom SSL, which would not be possible + if SSL was terminated at the load balancer. + +#### Alternate SSH Port + +Some organizations have policies against opening SSH port 22. In this case, +it may be helpful to configure an alternate SSH hostname that allows users +to use SSH on port 443. An alternate SSH hostname will require a new virtual IP address +compared to the other GitLab HTTP configuration above. + +Configure DNS for an alternate SSH hostname such as `altssh.gitlab.example.com`. + +| LB Port | Backend Port | Protocol | +| ------- | ------------ | -------- | +| 443 | 22 | TCP | + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure Consul + +The following IPs will be used as an example: + +- `10.6.0.11`: Consul 1 +- `10.6.0.12`: Consul 2 +- `10.6.0.13`: Consul 3 + +NOTE: **Note:** +The configuration processes for the other servers in your reference architecture will +use the `/etc/gitlab/gitlab-secrets.json` file from your Consul server to connect +with the other servers. + +To configure Consul: + +1. SSH into the server that will host Consul. +1. [Download/install](https://about.gitlab.com/install/) the + Omnibus GitLab Enterprise Edition package using **steps 1 and 2** from the + GitLab downloads page. + - Make sure you select the correct Omnibus package, with the same version + the GitLab application is running. + - Do not complete any other steps on the download page. + +1. Edit `/etc/gitlab/gitlab.rb` and add the contents: + + ```ruby + roles ['consul_role'] + + ## Enable service discovery for Prometheus + consul['enable'] = true + consul['monitoring_service_discovery'] = true + + ## The IPs of the Consul server nodes + ## You can also use FQDNs and intermix them with IPs + consul['configuration'] = { + server: true, + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), + } + + # Set the network addresses that the exporters will listen on + node_exporter['listen_address'] = '0.0.0.0:9100' + + # Disable auto migrations + gitlab_rails['auto_migrate'] = false + ``` + +1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. +1. Go through the steps again for all the other Consul nodes, and + make sure you set up the correct IPs. + +NOTE: **Note:** +A Consul leader will be elected when the provisioning of the third Consul server is completed. +Viewing the Consul logs `sudo gitlab-ctl tail consul` will display +`...[INFO] consul: New leader elected: ...` + +You can list the current Consul members (server, client): + +```shell +sudo /opt/gitlab/embedded/bin/consul members +``` + +You can verify the GitLab services are running: + +```shell +sudo gitlab-ctl status +``` + +The output should be similar to the following: + +```plaintext +run: consul: (pid 30074) 76834s; run: log: (pid 29740) 76844s +run: logrotate: (pid 30925) 3041s; run: log: (pid 29649) 76861s +run: node-exporter: (pid 30093) 76833s; run: log: (pid 29663) 76855s +``` + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure PostgreSQL + +In this section, you'll be guided through configuring an external PostgreSQL database +to be used with GitLab. + +### Provide your own PostgreSQL instance + +If you're hosting GitLab on a cloud provider, you can optionally use a +managed service for PostgreSQL. For example, AWS offers a managed Relational +Database Service (RDS) that runs PostgreSQL. + +If you use a cloud-managed service, or provide your own PostgreSQL: + +1. Set up PostgreSQL according to the + [database requirements document](../../install/requirements.md#database). +1. Set up a `gitlab` username with a password of your choice. The `gitlab` user + needs privileges to create the `gitlabhq_production` database. +1. Configure the GitLab application servers with the appropriate details. + This step is covered in [Configuring the GitLab Rails application](#configure-gitlab-rails). + +### Standalone PostgreSQL using Omnibus GitLab + +The following IPs will be used as an example: + +- `10.6.0.21`: PostgreSQL primary +- `10.6.0.22`: PostgreSQL secondary 1 +- `10.6.0.23`: PostgreSQL secondary 2 + +First, make sure to [install](https://about.gitlab.com/install/) +the Linux GitLab package **on each node**. Following the steps, +install the necessary dependencies from step 1, and add the +GitLab package repository from step 2. When installing GitLab +in the second step, do not supply the `EXTERNAL_URL` value. + +#### PostgreSQL primary node + +1. SSH into the PostgreSQL primary node. +1. Generate a password hash for the PostgreSQL username/password pair. This assumes you will use the default + username of `gitlab` (recommended). The command will request a password + and confirmation. Use the value that is output by this command in the next + step as the value of `<postgresql_password_hash>`: + + ```shell + sudo gitlab-ctl pg-password-md5 gitlab + ``` + +1. Generate a password hash for the PgBouncer username/password pair. This assumes you will use the default + username of `pgbouncer` (recommended). The command will request a password + and confirmation. Use the value that is output by this command in the next + step as the value of `<pgbouncer_password_hash>`: + + ```shell + sudo gitlab-ctl pg-password-md5 pgbouncer + ``` + +1. Generate a password hash for the Consul database username/password pair. This assumes you will use the default + username of `gitlab-consul` (recommended). The command will request a password + and confirmation. Use the value that is output by this command in the next + step as the value of `<consul_password_hash>`: + + ```shell + sudo gitlab-ctl pg-password-md5 gitlab-consul + ``` + +1. On the primary database node, edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: + + ```ruby + # Disable all components except PostgreSQL and Repmgr and Consul + roles ['postgres_role'] + + # PostgreSQL configuration + postgresql['listen_address'] = '0.0.0.0' + postgresql['hot_standby'] = 'on' + postgresql['wal_level'] = 'replica' + postgresql['shared_preload_libraries'] = 'repmgr_funcs' + + # Disable automatic database migrations + gitlab_rails['auto_migrate'] = false + + # Configure the Consul agent + consul['services'] = %w(postgresql) + + # START user configuration + # Please set the real values as explained in Required Information section + # + # Replace PGBOUNCER_PASSWORD_HASH with a generated md5 value + postgresql['pgbouncer_user_password'] = '<pgbouncer_password_hash>' + # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value + postgresql['sql_user_password'] = '<postgresql_password_hash>' + # Set `max_wal_senders` to one more than the number of database nodes in the cluster. + # This is used to prevent replication from using up all of the + # available database connections. + postgresql['max_wal_senders'] = 4 + postgresql['max_replication_slots'] = 4 + + # Replace XXX.XXX.XXX.XXX/YY with Network Address + postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24) + repmgr['trust_auth_cidr_addresses'] = %w(127.0.0.1/32 10.6.0.0/24) + + ## Enable service discovery for Prometheus + consul['monitoring_service_discovery'] = true + + # Set the network addresses that the exporters will listen on for monitoring + node_exporter['listen_address'] = '0.0.0.0:9100' + postgres_exporter['listen_address'] = '0.0.0.0:9187' + + ## The IPs of the Consul server nodes + ## You can also use FQDNs and intermix them with IPs + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), + } + # + # END user configuration + ``` + +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace + the file of the same name on this server. If that file is not on this server, + add the file from your Consul server to this server. + +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +#### PostgreSQL secondary nodes + +1. On both the secondary nodes, add the same configuration specified above for the primary node + with an additional setting (`repmgr['master_on_initialization'] = false`) that will inform `gitlab-ctl` that they are standby nodes initially + and there's no need to attempt to register them as a primary node: + + ```ruby + # Disable all components except PostgreSQL and Repmgr and Consul + roles ['postgres_role'] + + # PostgreSQL configuration + postgresql['listen_address'] = '0.0.0.0' + postgresql['hot_standby'] = 'on' + postgresql['wal_level'] = 'replica' + postgresql['shared_preload_libraries'] = 'repmgr_funcs' + + # Disable automatic database migrations + gitlab_rails['auto_migrate'] = false + + # Configure the Consul agent + consul['services'] = %w(postgresql) + + # Specify if a node should attempt to be primary on initialization. + repmgr['master_on_initialization'] = false + + # Replace PGBOUNCER_PASSWORD_HASH with a generated md5 value + postgresql['pgbouncer_user_password'] = '<pgbouncer_password_hash>' + # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value + postgresql['sql_user_password'] = '<postgresql_password_hash>' + # Set `max_wal_senders` to one more than the number of database nodes in the cluster. + # This is used to prevent replication from using up all of the + # available database connections. + postgresql['max_wal_senders'] = 4 + postgresql['max_replication_slots'] = 4 + + # Replace with your network addresses + postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24) + repmgr['trust_auth_cidr_addresses'] = %w(127.0.0.1/32 10.6.0.0/24) + + ## Enable service discovery for Prometheus + consul['monitoring_service_discovery'] = true + + # Set the network addresses that the exporters will listen on for monitoring + node_exporter['listen_address'] = '0.0.0.0:9100' + postgres_exporter['listen_address'] = '0.0.0.0:9187' + + ## The IPs of the Consul server nodes + ## You can also use FQDNs and intermix them with IPs + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), + } + ``` + +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace + the file of the same name on this server. If that file is not on this server, + add the file from your Consul server to this server. + +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + +Advanced [configuration options](https://docs.gitlab.com/omnibus/settings/database.html) +are supported and can be added if needed. + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +#### PostgreSQL post-configuration + +SSH into the **primary node**: + +1. Open a database prompt: + + ```shell + gitlab-psql -d gitlabhq_production + ``` + +1. Make sure the `pg_trgm` extension is enabled (it might already be): + + ```shell + CREATE EXTENSION pg_trgm; + ``` + +1. Exit the database prompt by typing `\q` and Enter. + +1. Verify the cluster is initialized with one node: + + ```shell + gitlab-ctl repmgr cluster show + ``` + + The output should be similar to the following: + + ```plaintext + Role | Name | Upstream | Connection String + ----------+----------|----------|---------------------------------------- + * master | HOSTNAME | | host=HOSTNAME user=gitlab_repmgr dbname=gitlab_repmgr + ``` + +1. Note down the hostname or IP address in the connection string: `host=HOSTNAME`. We will + refer to the hostname in the next section as `<primary_node_name>`. If the value + is not an IP address, it will need to be a resolvable name (via DNS or + `/etc/hosts`) + +SSH into the **secondary node**: + +1. Set up the repmgr standby: + + ```shell + gitlab-ctl repmgr standby setup <primary_node_name> + ``` + + Do note that this will remove the existing data on the node. The command + has a wait time. + + The output should be similar to the following: + + ```console + Doing this will delete the entire contents of /var/opt/gitlab/postgresql/data + If this is not what you want, hit Ctrl-C now to exit + To skip waiting, rerun with the -w option + Sleeping for 30 seconds + Stopping the database + Removing the data + Cloning the data + Starting the database + Registering the node with the cluster + ok: run: repmgrd: (pid 19068) 0s + ``` + +Before moving on, make sure the databases are configured correctly. Run the +following command on the **primary** node to verify that replication is working +properly and the secondary nodes appear in the cluster: + +```shell +gitlab-ctl repmgr cluster show +``` + +The output should be similar to the following: + +```plaintext +Role | Name | Upstream | Connection String +----------+---------|-----------|------------------------------------------------ +* master | MASTER | | host=<primary_node_name> user=gitlab_repmgr dbname=gitlab_repmgr + standby | STANDBY | MASTER | host=<secondary_node_name> user=gitlab_repmgr dbname=gitlab_repmgr + standby | STANDBY | MASTER | host=<secondary_node_name> user=gitlab_repmgr dbname=gitlab_repmgr +``` + +If the 'Role' column for any node says "FAILED", check the +[Troubleshooting section](troubleshooting.md) before proceeding. + +Also, check that the `repmgr-check-master` command works successfully on each node: + +```shell +su - gitlab-consul +gitlab-ctl repmgr-check-master || echo 'This node is a standby repmgr node' +``` + +This command relies on exit codes to tell Consul whether a particular node is a master +or secondary. The most important thing here is that this command does not produce errors. +If there are errors it's most likely due to incorrect `gitlab-consul` database user permissions. +Check the [Troubleshooting section](troubleshooting.md) before proceeding. + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure PgBouncer + +Now that the PostgreSQL servers are all set up, let's configure PgBouncer. +The following IPs will be used as an example: + +- `10.6.0.31`: PgBouncer 1 +- `10.6.0.32`: PgBouncer 2 +- `10.6.0.33`: PgBouncer 3 + +1. On each PgBouncer node, edit `/etc/gitlab/gitlab.rb`, and replace + `<consul_password_hash>` and `<pgbouncer_password_hash>` with the + password hashes you [set up previously](#postgresql-primary-node): + + ```ruby + # Disable all components except Pgbouncer and Consul agent + roles ['pgbouncer_role'] + + # Configure PgBouncer + pgbouncer['admin_users'] = %w(pgbouncer gitlab-consul) + + pgbouncer['users'] = { + 'gitlab-consul': { + password: '<consul_password_hash>' + }, + 'pgbouncer': { + password: '<pgbouncer_password_hash>' + } + } + + # Configure Consul agent + consul['watchers'] = %w(postgresql) + consul['enable'] = true + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13) + } + + # Enable service discovery for Prometheus + consul['monitoring_service_discovery'] = true + + # Set the network addresses that the exporters will listen on + node_exporter['listen_address'] = '0.0.0.0:9100' + ``` + +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace + the file of the same name on this server. If that file is not on this server, + add the file from your Consul server to this server. + +1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + + NOTE: **Note:** + If an error `execute[generate databases.ini]` occurs, this is due to an existing + [known issue](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/4713). + It will be resolved when you run a second `reconfigure` after the next step. + +1. Create a `.pgpass` file so Consul is able to + reload PgBouncer. Enter the PgBouncer password twice when asked: + + ```shell + gitlab-ctl write-pgpass --host 127.0.0.1 --database pgbouncer --user pgbouncer --hostuser gitlab-consul + ``` + +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) once again + to resolve any potential errors from the previous steps. +1. Ensure each node is talking to the current primary: + + ```shell + gitlab-ctl pgb-console # You will be prompted for PGBOUNCER_PASSWORD + ``` + +1. Once the console prompt is available, run the following queries: + + ```shell + show databases ; show clients ; + ``` + + The output should be similar to the following: + + ```plaintext + name | host | port | database | force_user | pool_size | reserve_pool | pool_mode | max_connections | current_connections + ---------------------+-------------+------+---------------------+------------+-----------+--------------+-----------+-----------------+--------------------- + gitlabhq_production | MASTER_HOST | 5432 | gitlabhq_production | | 20 | 0 | | 0 | 0 + pgbouncer | | 6432 | pgbouncer | pgbouncer | 2 | 0 | statement | 0 | 0 + (2 rows) + + type | user | database | state | addr | port | local_addr | local_port | connect_time | request_time | ptr | link | remote_pid | tls + ------+-----------+---------------------+---------+----------------+-------+------------+------------+---------------------+---------------------+-----------+------+------------+----- + C | pgbouncer | pgbouncer | active | 127.0.0.1 | 56846 | 127.0.0.1 | 6432 | 2017-08-21 18:09:59 | 2017-08-21 18:10:48 | 0x22b3880 | | 0 | + (2 rows) + ``` + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +### Configure the internal load balancer + +If you're running more than one PgBouncer node as recommended, then at this time you'll need to set +up a TCP internal load balancer to serve each correctly. + +The following IP will be used as an example: + +- `10.6.0.40`: Internal Load Balancer + +Here's how you could do it with [HAProxy](https://www.haproxy.org/): + +```plaintext +global + log /dev/log local0 + log localhost local1 notice + log stdout format raw local0 + +defaults + log global + default-server inter 10s fall 3 rise 2 + balance leastconn + +frontend internal-pgbouncer-tcp-in + bind *:6432 + mode tcp + option tcplog + + default_backend pgbouncer + +backend pgbouncer + mode tcp + option tcp-check + + server pgbouncer1 10.6.0.21:6432 check + server pgbouncer2 10.6.0.22:6432 check + server pgbouncer3 10.6.0.23:6432 check +``` + +Refer to your preferred Load Balancer's documentation for further guidance. + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure Redis + +Using [Redis](https://redis.io/) in scalable environment is possible using a **Primary** x **Replica** +topology with a [Redis Sentinel](https://redis.io/topics/sentinel) service to watch and automatically +start the failover procedure. + +Redis requires authentication if used with Sentinel. See +[Redis Security](https://redis.io/topics/security) documentation for more +information. We recommend using a combination of a Redis password and tight +firewall rules to secure your Redis service. +You are highly encouraged to read the [Redis Sentinel](https://redis.io/topics/sentinel) documentation +before configuring Redis with GitLab to fully understand the topology and +architecture. + +The requirements for a Redis setup are the following: + +1. All Redis nodes must be able to talk to each other and accept incoming + connections over Redis (`6379`) and Sentinel (`26379`) ports (unless you + change the default ones). +1. The server that hosts the GitLab application must be able to access the + Redis nodes. +1. Protect the nodes from access from external networks + ([Internet](https://gitlab.com/gitlab-org/gitlab-foss/uploads/c4cc8cd353604bd80315f9384035ff9e/The_Internet_IT_Crowd.png)), + using a firewall. + +In this section, you'll be guided through configuring two external Redis clusters +to be used with GitLab. The following IPs will be used as an example: + +- `10.6.0.51`: Redis - Cache Primary +- `10.6.0.52`: Redis - Cache Replica 1 +- `10.6.0.53`: Redis - Cache Replica 2 +- `10.6.0.71`: Sentinel - Cache 1 +- `10.6.0.72`: Sentinel - Cache 2 +- `10.6.0.73`: Sentinel - Cache 3 +- `10.6.0.61`: Redis - Queues Primary +- `10.6.0.62`: Redis - Queues Replica 1 +- `10.6.0.63`: Redis - Queues Replica 2 +- `10.6.0.81`: Sentinel - Queues 1 +- `10.6.0.82`: Sentinel - Queues 2 +- `10.6.0.83`: Sentinel - Queues 3 + +NOTE: **Providing your own Redis instance:** +Managed Redis from cloud providers such as AWS ElastiCache will work. If these +services support high availability, be sure it is **not** the Redis Cluster type. +Redis version 5.0 or higher is required, as this is what ships with +Omnibus GitLab packages starting with GitLab 13.0. Older Redis versions +do not support an optional count argument to SPOP which is now required for +[Merge Trains](../../ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md). +Note the Redis node's IP address or hostname, port, and password (if required). +These will be necessary when configuring the +[GitLab application servers](#configure-gitlab-rails) later. + +### Configure the Redis and Sentinel Cache cluster + +This is the section where we install and set up the new Redis Cache instances. + +NOTE: **Note:** +Redis nodes (both primary and replica) will need the same password defined in +`redis['password']`. At any time during a failover the Sentinels can +reconfigure a node and change its status from primary to replica and vice versa. + +#### Configure the primary Redis Cache node + +1. SSH into the **Primary** Redis server. +1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab + package you want using **steps 1 and 2** from the GitLab downloads page. + - Make sure you select the correct Omnibus package, with the same version + and type (Community, Enterprise editions) of your current install. + - Do not complete any other steps on the download page. + +1. Edit `/etc/gitlab/gitlab.rb` and add the contents: + + ```ruby + # Specify server role as 'redis_master_role' + roles ['redis_master_role'] + + # IP address pointing to a local IP that the other machines can reach to. + # You can also set bind to '0.0.0.0' which listen in all interfaces. + # If you really need to bind to an external accessible IP, make + # sure you add extra firewall rules to prevent unauthorized access. + redis['bind'] = '10.6.0.51' + + # Define a port so Redis can listen for TCP requests which will allow other + # machines to connect to it. + redis['port'] = 6379 + + # Set up password authentication for Redis (use the same password in all nodes). + redis['password'] = 'REDIS_PRIMARY_PASSWORD_OF_FIRST_CLUSTER' + + ## Enable service discovery for Prometheus + consul['enable'] = true + consul['monitoring_service_discovery'] = true + + ## The IPs of the Consul server nodes + ## You can also use FQDNs and intermix them with IPs + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), + } + + # Set the network addresses that the exporters will listen on + node_exporter['listen_address'] = '0.0.0.0:9100' + redis_exporter['listen_address'] = '0.0.0.0:9121' + + # Prevent database migrations from running on upgrade + gitlab_rails['auto_migrate'] = false + ``` + +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace + the file of the same name on this server. If that file is not on this server, + add the file from your Consul server to this server. + +1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + +NOTE: **Note:** +You can specify multiple roles like sentinel and Redis as: +`roles ['redis_sentinel_role', 'redis_master_role']`. +Read more about [roles](https://docs.gitlab.com/omnibus/roles/). + +#### Configure the replica Redis Cache nodes + +1. SSH into the **replica** Redis server. +1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab + package you want using **steps 1 and 2** from the GitLab downloads page. + - Make sure you select the correct Omnibus package, with the same version + and type (Community, Enterprise editions) of your current install. + - Do not complete any other steps on the download page. + +1. Edit `/etc/gitlab/gitlab.rb` and add the contents: + + ```ruby + # Specify server role as 'redis_replica_role' + roles ['redis_replica_role'] + + # IP address pointing to a local IP that the other machines can reach to. + # You can also set bind to '0.0.0.0' which listen in all interfaces. + # If you really need to bind to an external accessible IP, make + # sure you add extra firewall rules to prevent unauthorized access. + redis['bind'] = '10.6.0.52' + + # Define a port so Redis can listen for TCP requests which will allow other + # machines to connect to it. + redis['port'] = 6379 + + # The same password for Redis authentication you set up for the primary node. + redis['password'] = 'REDIS_PRIMARY_PASSWORD_OF_FIRST_CLUSTER' + + # The IP of the primary Redis node. + redis['master_ip'] = '10.6.0.51' + + # Port of primary Redis server, uncomment to change to non default. Defaults + # to `6379`. + #redis['master_port'] = 6379 + + ## Enable service discovery for Prometheus + consul['enable'] = true + consul['monitoring_service_discovery'] = true + + ## The IPs of the Consul server nodes + ## You can also use FQDNs and intermix them with IPs + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), + } + + # Set the network addresses that the exporters will listen on + node_exporter['listen_address'] = '0.0.0.0:9100' + redis_exporter['listen_address'] = '0.0.0.0:9121' + + # Prevent database migrations from running on upgrade + gitlab_rails['auto_migrate'] = false + ``` + +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace + the file of the same name on this server. If that file is not on this server, + add the file from your Consul server to this server. + +1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. +1. Go through the steps again for all the other replica nodes, and + make sure to set up the IPs correctly. + +NOTE: **Note:** +You can specify multiple roles like sentinel and Redis as: +`roles ['redis_sentinel_role', 'redis_master_role']`. +Read more about [roles](https://docs.gitlab.com/omnibus/roles/). + +These values don't have to be changed again in `/etc/gitlab/gitlab.rb` after +a failover, as the nodes will be managed by the [Sentinels](#configure-the-sentinel-cache-nodes), and even after a +`gitlab-ctl reconfigure`, they will get their configuration restored by +the same Sentinels. + +Advanced [configuration options](https://docs.gitlab.com/omnibus/settings/redis.html) +are supported and can be added if needed. + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +#### Configure the Sentinel Cache nodes + +NOTE: **Note:** +If you are using an external Redis Sentinel instance, be sure +to exclude the `requirepass` parameter from the Sentinel +configuration. This parameter will cause clients to report `NOAUTH +Authentication required.`. [Redis Sentinel 3.2.x does not support +password authentication](https://github.com/antirez/redis/issues/3279). + +Now that the Redis servers are all set up, let's configure the Sentinel +servers. The following IPs will be used as an example: + +- `10.6.0.71`: Sentinel - Cache 1 +- `10.6.0.72`: Sentinel - Cache 2 +- `10.6.0.73`: Sentinel - Cache 3 + +To configure the Sentinel Cache server: + +1. SSH into the server that will host Consul/Sentinel. +1. [Download/install](https://about.gitlab.com/install/) the + Omnibus GitLab Enterprise Edition package using **steps 1 and 2** from the + GitLab downloads page. + - Make sure you select the correct Omnibus package, with the same version + the GitLab application is running. + - Do not complete any other steps on the download page. + +1. Edit `/etc/gitlab/gitlab.rb` and add the contents: + + ```ruby + roles ['redis_sentinel_role'] + + ## Must be the same in every sentinel node + redis['master_name'] = 'gitlab-redis-cache' + + ## The same password for Redis authentication you set up for the primary node. + redis['master_password'] = 'REDIS_PRIMARY_PASSWORD_OF_FIRST_CLUSTER' + + ## The IP of the primary Redis node. + redis['master_ip'] = '10.6.0.51' + + ## Define a port so Redis can listen for TCP requests which will allow other + ## machines to connect to it. + redis['port'] = 6379 + + ## Port of primary Redis server, uncomment to change to non default. Defaults + ## to `6379`. + #redis['master_port'] = 6379 + + ## Configure Sentinel's IP + sentinel['bind'] = '10.6.0.71' + + ## Port that Sentinel listens on, uncomment to change to non default. Defaults + ## to `26379`. + #sentinel['port'] = 26379 + + ## Quorum must reflect the amount of voting sentinels it take to start a failover. + ## Value must NOT be greater then the amount of sentinels. + ## + ## The quorum can be used to tune Sentinel in two ways: + ## 1. If a the quorum is set to a value smaller than the majority of Sentinels + ## we deploy, we are basically making Sentinel more sensible to primary failures, + ## triggering a failover as soon as even just a minority of Sentinels is no longer + ## able to talk with the primary. + ## 1. If a quorum is set to a value greater than the majority of Sentinels, we are + ## making Sentinel able to failover only when there are a very large number (larger + ## than majority) of well connected Sentinels which agree about the primary being down.s + sentinel['quorum'] = 2 + + ## Consider unresponsive server down after x amount of ms. + #sentinel['down_after_milliseconds'] = 10000 + + ## Specifies the failover timeout in milliseconds. It is used in many ways: + ## + ## - The time needed to re-start a failover after a previous failover was + ## already tried against the same primary by a given Sentinel, is two + ## times the failover timeout. + ## + ## - The time needed for a replica replicating to a wrong primary according + ## to a Sentinel current configuration, to be forced to replicate + ## with the right primary, is exactly the failover timeout (counting since + ## the moment a Sentinel detected the misconfiguration). + ## + ## - The time needed to cancel a failover that is already in progress but + ## did not produced any configuration change (REPLICAOF NO ONE yet not + ## acknowledged by the promoted replica). + ## + ## - The maximum time a failover in progress waits for all the replica to be + ## reconfigured as replicas of the new primary. However even after this time + ## the replicas will be reconfigured by the Sentinels anyway, but not with + ## the exact parallel-syncs progression as specified. + #sentinel['failover_timeout'] = 60000 + + ## Enable service discovery for Prometheus + consul['enable'] = true + consul['monitoring_service_discovery'] = true + + ## The IPs of the Consul server nodes + ## You can also use FQDNs and intermix them with IPs + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), + } + + # Set the network addresses that the exporters will listen on + node_exporter['listen_address'] = '0.0.0.0:9100' + redis_exporter['listen_address'] = '0.0.0.0:9121' + + # Disable auto migrations + gitlab_rails['auto_migrate'] = false + ``` + +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace + the file of the same name on this server. If that file is not on this server, + add the file from your Consul server to this server. + +1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. +1. Go through the steps again for all the other Consul/Sentinel nodes, and + make sure you set up the correct IPs. + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +### Configure the Redis and Sentinel Queues cluster + +This is the section where we install and set up the new Redis Queues instances. + +NOTE: **Note:** +Redis nodes (both primary and replica) will need the same password defined in +`redis['password']`. At any time during a failover the Sentinels can +reconfigure a node and change its status from primary to replica and vice versa. + +#### Configure the primary Redis Queues node + +1. SSH into the **Primary** Redis server. +1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab + package you want using **steps 1 and 2** from the GitLab downloads page. + - Make sure you select the correct Omnibus package, with the same version + and type (Community, Enterprise editions) of your current install. + - Do not complete any other steps on the download page. + +1. Edit `/etc/gitlab/gitlab.rb` and add the contents: + + ```ruby + # Specify server role as 'redis_master_role' + roles ['redis_master_role'] + + # IP address pointing to a local IP that the other machines can reach to. + # You can also set bind to '0.0.0.0' which listen in all interfaces. + # If you really need to bind to an external accessible IP, make + # sure you add extra firewall rules to prevent unauthorized access. + redis['bind'] = '10.6.0.61' + + # Define a port so Redis can listen for TCP requests which will allow other + # machines to connect to it. + redis['port'] = 6379 + + # Set up password authentication for Redis (use the same password in all nodes). + redis['password'] = 'REDIS_PRIMARY_PASSWORD_OF_SECOND_CLUSTER' + + ## Enable service discovery for Prometheus + consul['enable'] = true + consul['monitoring_service_discovery'] = true + + ## The IPs of the Consul server nodes + ## You can also use FQDNs and intermix them with IPs + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), + } + + # Set the network addresses that the exporters will listen on + node_exporter['listen_address'] = '0.0.0.0:9100' + redis_exporter['listen_address'] = '0.0.0.0:9121' + ``` + +1. Only the primary GitLab application server should handle migrations. To + prevent database migrations from running on upgrade, add the following + configuration to your `/etc/gitlab/gitlab.rb` file: + + ```ruby + gitlab_rails['auto_migrate'] = false + ``` + +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace + the file of the same name on this server. If that file is not on this server, + add the file from your Consul server to this server. + +1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + +NOTE: **Note:** +You can specify multiple roles like sentinel and Redis as: +`roles ['redis_sentinel_role', 'redis_master_role']`. +Read more about [roles](https://docs.gitlab.com/omnibus/roles/). + +#### Configure the replica Redis Queues nodes + +1. SSH into the **replica** Redis Queue server. +1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab + package you want using **steps 1 and 2** from the GitLab downloads page. + - Make sure you select the correct Omnibus package, with the same version + and type (Community, Enterprise editions) of your current install. + - Do not complete any other steps on the download page. + +1. Edit `/etc/gitlab/gitlab.rb` and add the contents: + + ```ruby + # Specify server role as 'redis_replica_role' + roles ['redis_replica_role'] + + # IP address pointing to a local IP that the other machines can reach to. + # You can also set bind to '0.0.0.0' which listen in all interfaces. + # If you really need to bind to an external accessible IP, make + # sure you add extra firewall rules to prevent unauthorized access. + redis['bind'] = '10.6.0.62' + + # Define a port so Redis can listen for TCP requests which will allow other + # machines to connect to it. + redis['port'] = 6379 + + # The same password for Redis authentication you set up for the primary node. + redis['password'] = 'REDIS_PRIMARY_PASSWORD_OF_SECOND_CLUSTER' + + # The IP of the primary Redis node. + redis['master_ip'] = '10.6.0.61' + + # Port of primary Redis server, uncomment to change to non default. Defaults + # to `6379`. + #redis['master_port'] = 6379 + + ## Enable service discovery for Prometheus + consul['enable'] = true + consul['monitoring_service_discovery'] = true + + ## The IPs of the Consul server nodes + ## You can also use FQDNs and intermix them with IPs + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), + } + + # Set the network addresses that the exporters will listen on + node_exporter['listen_address'] = '0.0.0.0:9100' + redis_exporter['listen_address'] = '0.0.0.0:9121' + + # Disable auto migrations + gitlab_rails['auto_migrate'] = false + ``` + +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace + the file of the same name on this server. If that file is not on this server, + add the file from your Consul server to this server. + +1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. +1. Go through the steps again for all the other replica nodes, and + make sure to set up the IPs correctly. + +NOTE: **Note:** +You can specify multiple roles like sentinel and Redis as: +`roles ['redis_sentinel_role', 'redis_master_role']`. +Read more about [roles](https://docs.gitlab.com/omnibus/roles/). + +These values don't have to be changed again in `/etc/gitlab/gitlab.rb` after +a failover, as the nodes will be managed by the [Sentinels](#configure-the-sentinel-queues-nodes), and even after a +`gitlab-ctl reconfigure`, they will get their configuration restored by +the same Sentinels. + +Advanced [configuration options](https://docs.gitlab.com/omnibus/settings/redis.html) +are supported and can be added if needed. + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +#### Configure the Sentinel Queues nodes + +NOTE: **Note:** +If you are using an external Redis Sentinel instance, be sure +to exclude the `requirepass` parameter from the Sentinel +configuration. This parameter will cause clients to report `NOAUTH +Authentication required.`. [Redis Sentinel 3.2.x does not support +password authentication](https://github.com/antirez/redis/issues/3279). + +Now that the Redis servers are all set up, let's configure the Sentinel +servers. The following IPs will be used as an example: + +- `10.6.0.81`: Sentinel - Queues 1 +- `10.6.0.82`: Sentinel - Queues 2 +- `10.6.0.83`: Sentinel - Queues 3 + +To configure the Sentinel Queues server: + +1. SSH into the server that will host Sentinel. +1. [Download/install](https://about.gitlab.com/install/) the + Omnibus GitLab Enterprise Edition package using **steps 1 and 2** from the + GitLab downloads page. + - Make sure you select the correct Omnibus package, with the same version + the GitLab application is running. + - Do not complete any other steps on the download page. + +1. Edit `/etc/gitlab/gitlab.rb` and add the contents: + + ```ruby + roles ['redis_sentinel_role'] + + ## Must be the same in every sentinel node + redis['master_name'] = 'gitlab-redis-persistent' + + ## The same password for Redis authentication you set up for the primary node. + redis['master_password'] = 'REDIS_PRIMARY_PASSWORD_OF_SECOND_CLUSTER' + + ## The IP of the primary Redis node. + redis['master_ip'] = '10.6.0.61' + + ## Define a port so Redis can listen for TCP requests which will allow other + ## machines to connect to it. + redis['port'] = 6379 + + ## Port of primary Redis server, uncomment to change to non default. Defaults + ## to `6379`. + #redis['master_port'] = 6379 + + ## Configure Sentinel's IP + sentinel['bind'] = '10.6.0.81' + + ## Port that Sentinel listens on, uncomment to change to non default. Defaults + ## to `26379`. + #sentinel['port'] = 26379 + + ## Quorum must reflect the amount of voting sentinels it take to start a failover. + ## Value must NOT be greater then the amount of sentinels. + ## + ## The quorum can be used to tune Sentinel in two ways: + ## 1. If a the quorum is set to a value smaller than the majority of Sentinels + ## we deploy, we are basically making Sentinel more sensible to primary failures, + ## triggering a failover as soon as even just a minority of Sentinels is no longer + ## able to talk with the primary. + ## 1. If a quorum is set to a value greater than the majority of Sentinels, we are + ## making Sentinel able to failover only when there are a very large number (larger + ## than majority) of well connected Sentinels which agree about the primary being down.s + sentinel['quorum'] = 2 + + ## Consider unresponsive server down after x amount of ms. + #sentinel['down_after_milliseconds'] = 10000 + + ## Specifies the failover timeout in milliseconds. It is used in many ways: + ## + ## - The time needed to re-start a failover after a previous failover was + ## already tried against the same primary by a given Sentinel, is two + ## times the failover timeout. + ## + ## - The time needed for a replica replicating to a wrong primary according + ## to a Sentinel current configuration, to be forced to replicate + ## with the right primary, is exactly the failover timeout (counting since + ## the moment a Sentinel detected the misconfiguration). + ## + ## - The time needed to cancel a failover that is already in progress but + ## did not produced any configuration change (REPLICAOF NO ONE yet not + ## acknowledged by the promoted replica). + ## + ## - The maximum time a failover in progress waits for all the replica to be + ## reconfigured as replicas of the new primary. However even after this time + ## the replicas will be reconfigured by the Sentinels anyway, but not with + ## the exact parallel-syncs progression as specified. + #sentinel['failover_timeout'] = 60000 + + ## Enable service discovery for Prometheus + consul['enable'] = true + consul['monitoring_service_discovery'] = true + + ## The IPs of the Consul server nodes + ## You can also use FQDNs and intermix them with IPs + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), + } + + # Set the network addresses that the exporters will listen on + node_exporter['listen_address'] = '0.0.0.0:9100' + redis_exporter['listen_address'] = '0.0.0.0:9121' + + # Disable auto migrations + gitlab_rails['auto_migrate'] = false + ``` + +1. To prevent database migrations from running on upgrade, run: + + ```shell + sudo touch /etc/gitlab/skip-auto-reconfigure + ``` + + Only the primary GitLab application server should handle migrations. + +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace + the file of the same name on this server. If that file is not on this server, + add the file from your Consul server to this server. + +1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. +1. Go through the steps again for all the other Sentinel nodes, and + make sure you set up the correct IPs. + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure Gitaly + +Deploying Gitaly in its own server can benefit GitLab installations that are +larger than a single machine. + +The Gitaly node requirements are dependent on customer data, specifically the number of +projects and their repository sizes. Two nodes are recommended as an absolute minimum. +Each Gitaly node should store no more than 5TB of data and have the number of +[`gitaly-ruby` workers](../gitaly/index.md#gitaly-ruby) set to 20% of available CPUs. +Additional nodes should be considered in conjunction with a review of expected +data size and spread based on the recommendations above. + +It is also strongly recommended that all Gitaly nodes be set up with SSD disks with +a throughput of at least 8,000 IOPS for read operations and 2,000 IOPS for write, +as Gitaly has heavy I/O. These IOPS values are recommended only as a starter as with +time they may be adjusted higher or lower depending on the scale of your environment's workload. +If you're running the environment on a Cloud provider, you may need to refer to +their documentation on how to configure IOPS correctly. + +Some things to note: + +- The GitLab Rails application shards repositories into [repository storages](../repository_storage_paths.md). +- A Gitaly server can host one or more storages. +- A GitLab server can use one or more Gitaly servers. +- Gitaly addresses must be specified in such a way that they resolve + correctly for ALL Gitaly clients. +- Gitaly servers must not be exposed to the public internet, as Gitaly's network + traffic is unencrypted by default. The use of a firewall is highly recommended + to restrict access to the Gitaly server. Another option is to + [use TLS](#gitaly-tls-support). + +TIP: **Tip:** +For more information about Gitaly's history and network architecture see the +[standalone Gitaly documentation](../gitaly/index.md). + +Note: **Note:** +The token referred to throughout the Gitaly documentation is +just an arbitrary password selected by the administrator. It is unrelated to +tokens created for the GitLab API or other similar web API tokens. + +Below we describe how to configure two Gitaly servers, with IPs and +domain names: + +- `10.6.0.91`: Gitaly 1 (`gitaly1.internal`) +- `10.6.0.92`: Gitaly 2 (`gitaly2.internal`) + +The secret token is assumed to be `gitalysecret` and that +your GitLab installation has three repository storages: + +- `default` on Gitaly 1 +- `storage1` on Gitaly 1 +- `storage2` on Gitaly 2 + +On each node: + +1. [Download/Install](https://about.gitlab.com/install/) the Omnibus GitLab + package you want using **steps 1 and 2** from the GitLab downloads page but + **without** providing the `EXTERNAL_URL` value. +1. Edit `/etc/gitlab/gitlab.rb` to configure storage paths, enable + the network listener and configure the token: + + <!-- + updates to following example must also be made at + https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab + --> + + ```ruby + # /etc/gitlab/gitlab.rb + + # Gitaly and GitLab use two shared secrets for authentication, one to authenticate gRPC requests + # to Gitaly, and a second for authentication callbacks from GitLab-Shell to the GitLab internal API. + # The following two values must be the same as their respective values + # of the GitLab Rails application setup + gitaly['auth_token'] = 'gitalysecret' + gitlab_shell['secret_token'] = 'shellsecret' + + # Avoid running unnecessary services on the Gitaly server + postgresql['enable'] = false + redis['enable'] = false + nginx['enable'] = false + puma['enable'] = false + unicorn['enable'] = false + sidekiq['enable'] = false + gitlab_workhorse['enable'] = false + grafana['enable'] = false + + # If you run a seperate monitoring node you can disable these services + alertmanager['enable'] = false + prometheus['enable'] = false + + # Prevent database connections during 'gitlab-ctl reconfigure' + gitlab_rails['rake_cache_clear'] = false + gitlab_rails['auto_migrate'] = false + + # Configure the gitlab-shell API callback URL. Without this, `git push` will + # fail. This can be your 'front door' GitLab URL or an internal load + # balancer. + # Don't forget to copy `/etc/gitlab/gitlab-secrets.json` from web server to Gitaly server. + gitlab_rails['internal_api_url'] = 'https://gitlab.example.com' + + # Make Gitaly accept connections on all network interfaces. You must use + # firewalls to restrict access to this address/port. + # Comment out following line if you only want to support TLS connections + gitaly['listen_addr'] = "0.0.0.0:8075" + ``` + +1. Append the following to `/etc/gitlab/gitlab.rb` for each respective server: + 1. On `gitaly1.internal`: + + ```ruby + git_data_dirs({ + 'default' => { + 'path' => '/var/opt/gitlab/git-data' + }, + 'storage1' => { + 'path' => '/mnt/gitlab/git-data' + }, + }) + ``` + + 1. On `gitaly2.internal`: + + ```ruby + git_data_dirs({ + 'storage2' => { + 'path' => '/mnt/gitlab/git-data' + }, + }) + ``` + + <!-- + updates to following example must also be made at + https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab + --> + +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace + the file of the same name on this server. If that file is not on this server, + add the file from your Consul server to this server. + +1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). + +### Gitaly TLS support + +Gitaly supports TLS encryption. To be able to communicate +with a Gitaly instance that listens for secure connections you will need to use `tls://` URL +scheme in the `gitaly_address` of the corresponding storage entry in the GitLab configuration. + +You will need to bring your own certificates as this isn't provided automatically. +The certificate, or its certificate authority, must be installed on all Gitaly +nodes (including the Gitaly node using the certificate) and on all client nodes +that communicate with it following the procedure described in +[GitLab custom certificate configuration](https://docs.gitlab.com/omnibus/settings/ssl.html#install-custom-public-certificates). + +NOTE: **Note:** +The self-signed certificate must specify the address you use to access the +Gitaly server. If you are addressing the Gitaly server by a hostname, you can +either use the Common Name field for this, or add it as a Subject Alternative +Name. If you are addressing the Gitaly server by its IP address, you must add it +as a Subject Alternative Name to the certificate. +[gRPC does not support using an IP address as Common Name in a certificate](https://github.com/grpc/grpc/issues/2691). + +NOTE: **Note:** +It is possible to configure Gitaly servers with both an +unencrypted listening address `listen_addr` and an encrypted listening +address `tls_listen_addr` at the same time. This allows you to do a +gradual transition from unencrypted to encrypted traffic, if necessary. + +To configure Gitaly with TLS: + +1. Create the `/etc/gitlab/ssl` directory and copy your key and certificate there: + + ```shell + sudo mkdir -p /etc/gitlab/ssl + sudo chmod 755 /etc/gitlab/ssl + sudo cp key.pem cert.pem /etc/gitlab/ssl/ + sudo chmod 644 key.pem cert.pem + ``` + +1. Copy the cert to `/etc/gitlab/trusted-certs` so Gitaly will trust the cert when + calling into itself: + + ```shell + sudo cp /etc/gitlab/ssl/cert.pem /etc/gitlab/trusted-certs/ + ``` + +1. Edit `/etc/gitlab/gitlab.rb` and add: + + <!-- + updates to following example must also be made at + https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab + --> + + ```ruby + gitaly['tls_listen_addr'] = "0.0.0.0:9999" + gitaly['certificate_path'] = "/etc/gitlab/ssl/cert.pem" + gitaly['key_path'] = "/etc/gitlab/ssl/key.pem" + ``` + +1. Delete `gitaly['listen_addr']` to allow only encrypted connections. + +1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure Sidekiq + +Sidekiq requires connections to the Redis, PostgreSQL and Gitaly instances. +The following IPs will be used as an example: + +- `10.6.0.101`: Sidekiq 1 +- `10.6.0.102`: Sidekiq 2 +- `10.6.0.103`: Sidekiq 3 +- `10.6.0.104`: Sidekiq 4 + +To configure the Sidekiq nodes, on each one: + +1. SSH into the Sidekiq server. +1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab package +you want using steps 1 and 2 from the GitLab downloads page. +**Do not complete any other steps on the download page.** +1. Open `/etc/gitlab/gitlab.rb` with your editor: + + ```ruby + ######################################## + ##### Services Disabled ### + ######################################## + + nginx['enable'] = false + grafana['enable'] = false + prometheus['enable'] = false + alertmanager['enable'] = false + gitaly['enable'] = false + gitlab_workhorse['enable'] = false + nginx['enable'] = false + puma['enable'] = false + postgres_exporter['enable'] = false + postgresql['enable'] = false + redis['enable'] = false + redis_exporter['enable'] = false + gitlab_exporter['enable'] = false + + ######################################## + #### Redis ### + ######################################## + + ## Redis connection details + ## First cluster that will host the cache + gitlab_rails['redis_cache_instance'] = 'redis://:<REDIS_PRIMARY_PASSWORD_OF_FIRST_CLUSTER>@gitlab-redis-cache' + + gitlab_rails['redis_cache_sentinels'] = [ + {host: '10.6.0.71', port: 26379}, + {host: '10.6.0.72', port: 26379}, + {host: '10.6.0.73', port: 26379}, + ] + + ## Second cluster that will host the queues, shared state, and actioncable + gitlab_rails['redis_queues_instance'] = 'redis://:<REDIS_PRIMARY_PASSWORD_OF_SECOND_CLUSTER>@gitlab-redis-persistent' + gitlab_rails['redis_shared_state_instance'] = 'redis://:<REDIS_PRIMARY_PASSWORD_OF_SECOND_CLUSTER>@gitlab-redis-persistent' + gitlab_rails['redis_actioncable_instance'] = 'redis://:<REDIS_PRIMARY_PASSWORD_OF_SECOND_CLUSTER>@gitlab-redis-persistent' + + gitlab_rails['redis_queues_sentinels'] = [ + {host: '10.6.0.81', port: 26379}, + {host: '10.6.0.82', port: 26379}, + {host: '10.6.0.83', port: 26379}, + ] + gitlab_rails['redis_shared_state_sentinels'] = [ + {host: '10.6.0.81', port: 26379}, + {host: '10.6.0.82', port: 26379}, + {host: '10.6.0.83', port: 26379}, + ] + gitlab_rails['redis_actioncable_sentinels'] = [ + {host: '10.6.0.81', port: 26379}, + {host: '10.6.0.82', port: 26379}, + {host: '10.6.0.83', port: 26379}, + ] + + ####################################### + ### Gitaly ### + ####################################### + + git_data_dirs({ + 'default' => { 'gitaly_address' => 'tcp://gitaly1.internal:8075' }, + 'storage1' => { 'gitaly_address' => 'tcp://gitaly1.internal:8075' }, + 'storage2' => { 'gitaly_address' => 'tcp://gitaly2.internal:8075' }, + }) + gitlab_rails['gitaly_token'] = 'YOUR_TOKEN' + + ####################################### + ### Postgres ### + ####################################### + gitlab_rails['db_host'] = '10.6.0.20' # internal load balancer IP + gitlab_rails['db_port'] = 6432 + gitlab_rails['db_password'] = '<postgresql_user_password>' + gitlab_rails['db_adapter'] = 'postgresql' + gitlab_rails['db_encoding'] = 'unicode' + gitlab_rails['auto_migrate'] = false + + ####################################### + ### Sidekiq configuration ### + ####################################### + sidekiq['listen_address'] = "0.0.0.0" + sidekiq['cluster'] = true # no need to set this after GitLab 13.0 + + ####################################### + ### Monitoring configuration ### + ####################################### + consul['enable'] = true + consul['monitoring_service_discovery'] = true + + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13) + } + + # Set the network addresses that the exporters will listen on + node_exporter['listen_address'] = '0.0.0.0:9100' + + # Rails Status for prometheus + gitlab_rails['monitoring_whitelist'] = ['10.6.0.121/32', '127.0.0.0/8'] + ``` + +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace + the file of the same name on this server. If that file is not on this server, + add the file from your Consul server to this server. + +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + +TIP: **Tip:** +You can also run [multiple Sidekiq processes](../operations/extra_sidekiq_processes.md). + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure GitLab Rails + +NOTE: **Note:** +In our architectures we run each GitLab Rails node using the Puma webserver +and have its number of workers set to 90% of available CPUs along with four threads. For +nodes that are running Rails with other components the worker value should be reduced +accordingly where we've found 50% achieves a good balance but this is dependent +on workload. + +This section describes how to configure the GitLab application (Rails) component. + +The following IPs will be used as an example: + +- `10.6.0.111`: GitLab application 1 +- `10.6.0.112`: GitLab application 2 +- `10.6.0.113`: GitLab application 3 + +On each node perform the following: + +1. Download/install Omnibus GitLab using **steps 1 and 2** from + [GitLab downloads](https://about.gitlab.com/install/). Do not complete other + steps on the download page. + +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace + the file of the same name on this server. If that file is not on this server, + add the file from your Consul server to this server. + +1. Edit `/etc/gitlab/gitlab.rb` and use the following configuration. + To maintain uniformity of links across nodes, the `external_url` + on the application server should point to the external URL that users will use + to access GitLab. This would be the URL of the [external load balancer](#configure-the-external-load-balancer) + which will route traffic to the GitLab application server: + + ```ruby + external_url 'https://gitlab.example.com' + + # Gitaly and GitLab use two shared secrets for authentication, one to authenticate gRPC requests + # to Gitaly, and a second for authentication callbacks from GitLab-Shell to the GitLab internal API. + # The following two values must be the same as their respective values + # of the Gitaly setup + gitlab_rails['gitaly_token'] = 'gitalysecret' + gitlab_shell['secret_token'] = 'shellsecret' + + git_data_dirs({ + 'default' => { 'gitaly_address' => 'tcp://gitaly1.internal:8075' }, + 'storage1' => { 'gitaly_address' => 'tcp://gitaly1.internal:8075' }, + 'storage2' => { 'gitaly_address' => 'tcp://gitaly2.internal:8075' }, + }) + + ## Disable components that will not be on the GitLab application server + roles ['application_role'] + gitaly['enable'] = false + nginx['enable'] = true + + ## PostgreSQL connection details + # Disable PostgreSQL on the application node + postgresql['enable'] = false + gitlab_rails['db_host'] = '10.6.0.20' # internal load balancer IP + gitlab_rails['db_port'] = 6432 + gitlab_rails['db_password'] = '<postgresql_user_password>' + gitlab_rails['auto_migrate'] = false + + ## Redis connection details + ## First cluster that will host the cache + gitlab_rails['redis_cache_instance'] = 'redis://:<REDIS_PRIMARY_PASSWORD_OF_FIRST_CLUSTER>@gitlab-redis-cache' + + gitlab_rails['redis_cache_sentinels'] = [ + {host: '10.6.0.71', port: 26379}, + {host: '10.6.0.72', port: 26379}, + {host: '10.6.0.73', port: 26379}, + ] + + ## Second cluster that will host the queues, shared state, and actionable + gitlab_rails['redis_queues_instance'] = 'redis://:<REDIS_PRIMARY_PASSWORD_OF_SECOND_CLUSTER>@gitlab-redis-persistent' + gitlab_rails['redis_shared_state_instance'] = 'redis://:<REDIS_PRIMARY_PASSWORD_OF_SECOND_CLUSTER>@gitlab-redis-persistent' + gitlab_rails['redis_actioncable_instance'] = 'redis://:<REDIS_PRIMARY_PASSWORD_OF_SECOND_CLUSTER>@gitlab-redis-persistent' + + gitlab_rails['redis_queues_sentinels'] = [ + {host: '10.6.0.81', port: 26379}, + {host: '10.6.0.82', port: 26379}, + {host: '10.6.0.83', port: 26379}, + ] + gitlab_rails['redis_shared_state_sentinels'] = [ + {host: '10.6.0.81', port: 26379}, + {host: '10.6.0.82', port: 26379}, + {host: '10.6.0.83', port: 26379}, + ] + gitlab_rails['redis_actioncable_sentinels'] = [ + {host: '10.6.0.81', port: 26379}, + {host: '10.6.0.82', port: 26379}, + {host: '10.6.0.83', port: 26379}, + ] + + # Set the network addresses that the exporters used for monitoring will listen on + node_exporter['listen_address'] = '0.0.0.0:9100' + gitlab_workhorse['prometheus_listen_addr'] = '0.0.0.0:9229' + sidekiq['listen_address'] = "0.0.0.0" + puma['listen'] = '0.0.0.0' + + # Add the monitoring node's IP address to the monitoring whitelist and allow it to + # scrape the NGINX metrics + gitlab_rails['monitoring_whitelist'] = ['10.6.0.121/32', '127.0.0.0/8'] + nginx['status']['options']['allow'] = ['10.6.0.121/32', '127.0.0.0/8'] + ``` + +1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). +1. If you're using [Gitaly with TLS support](#gitaly-tls-support), make sure the + `git_data_dirs` entry is configured with `tls` instead of `tcp`: + + ```ruby + git_data_dirs({ + 'default' => { 'gitaly_address' => 'tls://gitaly1.internal:9999' }, + 'storage1' => { 'gitaly_address' => 'tls://gitaly1.internal:9999' }, + 'storage2' => { 'gitaly_address' => 'tls://gitaly2.internal:9999' }, + }) + ``` + + 1. Copy the cert into `/etc/gitlab/trusted-certs`: + + ```shell + sudo cp cert.pem /etc/gitlab/trusted-certs/ + ``` + +1. If you're [using NFS](#configure-nfs-optional): + 1. If necessary, install the NFS client utility packages using the following + commands: + + ```shell + # Ubuntu/Debian + apt-get install nfs-common + + # CentOS/Red Hat + yum install nfs-utils nfs-utils-lib + ``` + + 1. Specify the necessary NFS mounts in `/etc/fstab`. + The exact contents of `/etc/fstab` will depend on how you chose + to configure your NFS server. See the [NFS documentation](../high_availability/nfs.md) + for examples and the various options. + + 1. Create the shared directories. These may be different depending on your NFS + mount locations. + + ```shell + mkdir -p /var/opt/gitlab/.ssh /var/opt/gitlab/gitlab-rails/uploads /var/opt/gitlab/gitlab-rails/shared /var/opt/gitlab/gitlab-ci/builds /var/opt/gitlab/git-data + ``` + + 1. Edit `/etc/gitlab/gitlab.rb` and use the following configuration: + + ```ruby + ## Prevent GitLab from starting if NFS data mounts are not available + high_availability['mountpoint'] = '/var/opt/gitlab/git-data' + + ## Ensure UIDs and GIDs match between servers for permissions via NFS + user['uid'] = 9000 + user['gid'] = 9000 + web_server['uid'] = 9001 + web_server['gid'] = 9001 + registry['uid'] = 9002 + registry['gid'] = 9002 + ``` + +1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). +1. Confirm the node can connect to Gitaly: + + ```shell + sudo gitlab-rake gitlab:gitaly:check + ``` + + Then, tail the logs to see the requests: + + ```shell + sudo gitlab-ctl tail gitaly + ``` + +1. Optionally, from the Gitaly servers, confirm that Gitaly can perform callbacks to the internal API: + + ```shell + sudo /opt/gitlab/embedded/service/gitlab-shell/bin/check -config /opt/gitlab/embedded/service/gitlab-shell/config.yml + ``` + +NOTE: **Note:** +When you specify `https` in the `external_url`, as in the example +above, GitLab assumes you have SSL certificates in `/etc/gitlab/ssl/`. If +certificates are not present, NGINX will fail to start. See the +[NGINX documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https) +for more information. + +### GitLab Rails post-configuration + +Initialize the GitLab database, by running the following in one of the Rails nodes: + +```shell +sudo gitlab-rake gitlab:db:configure +``` + +NOTE: **Note:** +If you encounter a `rake aborted!` error stating that PgBouncer is failing to connect to +PostgreSQL it may be that your PgBouncer node's IP address is missing from +PostgreSQL's `trust_auth_cidr_addresses` in `gitlab.rb` on your database nodes. See +[PgBouncer error `ERROR: pgbouncer cannot connect to server`](troubleshooting.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server) +in the Troubleshooting section before proceeding. + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure Prometheus + +The Omnibus GitLab package can be used to configure a standalone Monitoring node +running [Prometheus](../monitoring/prometheus/index.md) and +[Grafana](../monitoring/performance/grafana_configuration.md). + +The following IP will be used as an example: + +- `10.6.0.121`: Prometheus + +To configure the Monitoring node: + +1. SSH into the Monitoring node. +1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab + package you want using **steps 1 and 2** from the GitLab downloads page. + Do not complete any other steps on the download page. + +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace + the file of the same name on this server. If that file is not on this server, + add the file from your Consul server to this server. + +1. Edit `/etc/gitlab/gitlab.rb` and add the contents: + + ```ruby + external_url 'http://gitlab.example.com' + + # Disable all other services + gitlab_rails['auto_migrate'] = false + alertmanager['enable'] = false + gitaly['enable'] = false + gitlab_exporter['enable'] = false + gitlab_workhorse['enable'] = false + nginx['enable'] = true + postgres_exporter['enable'] = false + postgresql['enable'] = false + redis['enable'] = false + redis_exporter['enable'] = false + sidekiq['enable'] = false + puma['enable'] = false + unicorn['enable'] = false + node_exporter['enable'] = false + gitlab_exporter['enable'] = false + + # Enable Prometheus + prometheus['enable'] = true + prometheus['listen_address'] = '0.0.0.0:9090' + prometheus['monitor_kubernetes'] = false + + # Enable Login form + grafana['disable_login_form'] = false + + # Enable Grafana + grafana['enable'] = true + grafana['admin_password'] = '<grafana_password>' + + # Enable service discovery for Prometheus + consul['enable'] = true + consul['monitoring_service_discovery'] = true + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13) + } + ``` + +1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). +1. In the GitLab UI, set `admin/application_settings/metrics_and_profiling` > Metrics - Grafana to `/-/grafana` to +`http[s]://<MONITOR NODE>/-/grafana` + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure the object storage + +GitLab supports using an object storage service for holding numerous types of data. +It's recommended over [NFS](#configure-nfs-optional) and in general it's better +in larger setups as object storage is typically much more performant, reliable, +and scalable. + +Object storage options that GitLab has tested, or is aware of customers using include: + +- SaaS/Cloud solutions such as [Amazon S3](https://aws.amazon.com/s3/), [Google cloud storage](https://cloud.google.com/storage). +- On-premises hardware and appliances from various storage vendors. +- MinIO. There is [a guide to deploying this](https://docs.gitlab.com/charts/advanced/external-object-storage/minio.html) within our Helm Chart documentation. + +For configuring GitLab to use Object Storage refer to the following guides +based on what features you intend to use: + +1. Configure [object storage for backups](../../raketasks/backup_restore.md#uploading-backups-to-a-remote-cloud-storage). +1. Configure [object storage for job artifacts](../job_artifacts.md#using-object-storage) + including [incremental logging](../job_logs.md#new-incremental-logging-architecture). +1. Configure [object storage for LFS objects](../lfs/index.md#storing-lfs-objects-in-remote-object-storage). +1. Configure [object storage for uploads](../uploads.md#using-object-storage-core-only). +1. Configure [object storage for merge request diffs](../merge_request_diffs.md#using-object-storage). +1. Configure [object storage for Container Registry](../packages/container_registry.md#use-object-storage) (optional feature). +1. Configure [object storage for Mattermost](https://docs.mattermost.com/administration/config-settings.html#file-storage) (optional feature). +1. Configure [object storage for packages](../packages/index.md#using-object-storage) (optional feature). **(PREMIUM ONLY)** +1. Configure [object storage for Dependency Proxy](../packages/dependency_proxy.md#using-object-storage) (optional feature). **(PREMIUM ONLY)** +1. Configure [object storage for Pseudonymizer](../pseudonymizer.md#configuration) (optional feature). **(ULTIMATE ONLY)** +1. Configure [object storage for autoscale Runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional - for improved performance). +1. Configure [object storage for Terraform state files](../terraform_state.md#using-object-storage-core-only). + +Using separate buckets for each data type is the recommended approach for GitLab. + +A limitation of our configuration is that each use of object storage is separately configured. +[We have an issue for improving this](https://gitlab.com/gitlab-org/gitlab/-/issues/23345) +and easily using one bucket with separate folders is one improvement that this might bring. + +There is at least one specific issue with using the same bucket: +when GitLab is deployed with the Helm chart restore from backup +[will not properly function](https://docs.gitlab.com/charts/advanced/external-object-storage/#lfs-artifacts-uploads-packages-external-diffs-pseudonymizer) +unless separate buckets are used. + +One risk of using a single bucket would be if your organization decided to +migrate GitLab to the Helm deployment in the future. GitLab would run, but the situation with +backups might not be realized until the organization had a critical requirement for the backups to +work. + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure NFS (optional) + +[Object storage](#configure-the-object-storage), along with [Gitaly](#configure-gitaly) +are recommended over NFS wherever possible for improved performance. If you intend +to use GitLab Pages, this currently [requires NFS](troubleshooting.md#gitlab-pages-requires-nfs). + +See how to [configure NFS](../high_availability/nfs.md). + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Troubleshooting + +See the [troubleshooting documentation](troubleshooting.md). + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> diff --git a/doc/administration/reference_architectures/2k_users.md b/doc/administration/reference_architectures/2k_users.md index d182daf45b3..a7feb78a365 100644 --- a/doc/administration/reference_architectures/2k_users.md +++ b/doc/administration/reference_architectures/2k_users.md @@ -1,27 +1,30 @@ --- reading_time: true +stage: Enablement +group: Distribution +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# Reference architecture: up to 2,000 users +# Reference architecture: up to 2,000 users **(CORE ONLY)** This page describes GitLab reference architecture for up to 2,000 users. For a full list of reference architectures, see [Available reference architectures](index.md#available-reference-architectures). > - **Supported users (approximate):** 2,000 -> - **High Availability:** False +> - **High Availability:** No > - **Test requests per second (RPS) rates:** API: 40 RPS, Web: 4 RPS, Git: 4 RPS -| Service | Nodes | Configuration | GCP | AWS | Azure | -|------------------------------------------|--------|-------------------------|-----------------|----------------|-----------| -| Load balancer | 1 | 2 vCPU, 1.8GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| PostgreSQL | 1 | 2 vCPU, 7.5GB memory | `n1-standard-2` | `m5.large` | `D2s v3` | -| Redis | 1 | 1 vCPU, 3.75GB memory | `n1-standard-1` | `m5.large` | `D2s v3` | -| Gitaly | 1 | 4 vCPU, 15GB memory | `n1-standard-4` | `m5.xlarge` | `D4s v3` | -| GitLab Rails | 2 | 8 vCPU, 7.2GB memory | `n1-highcpu-8` | `c5.2xlarge` | `F8s v2` | -| Monitoring node | 1 | 2 vCPU, 1.8GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| Object storage | n/a | n/a | n/a | n/a | n/a | -| NFS server (optional, not recommended) | 1 | 4 vCPU, 3.6GB memory | `n1-highcpu-4` | `c5.xlarge` | `F4s v2` | +| Service | Nodes | Configuration | GCP | AWS | Azure | +|------------------------------------------|--------|-------------------------|----------------|--------------|---------| +| Load balancer | 1 | 2 vCPU, 1.8GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| PostgreSQL | 1 | 2 vCPU, 7.5GB memory | n1-standard-2 | m5.large | D2s v3 | +| Redis | 1 | 1 vCPU, 3.75GB memory | n1-standard-1 | m5.large | D2s v3 | +| Gitaly | 1 | 4 vCPU, 15GB memory | n1-standard-4 | m5.xlarge | D4s v3 | +| GitLab Rails | 2 | 8 vCPU, 7.2GB memory | n1-highcpu-8 | c5.2xlarge | F8s v2 | +| Monitoring node | 1 | 2 vCPU, 1.8GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| Object storage | n/a | n/a | n/a | n/a | n/a | +| NFS server (optional, not recommended) | 1 | 4 vCPU, 3.6GB memory | n1-highcpu-4 | c5.xlarge | F4s v2 | The Google Cloud Platform (GCP) architectures were built and tested using the [Intel Xeon E5 v3 (Haswell)](https://cloud.google.com/compute/docs/cpu-platforms) @@ -554,7 +557,7 @@ On each node perform the following: 1. Specify the necessary NFS mounts in `/etc/fstab`. The exact contents of `/etc/fstab` will depend on how you chose - to configure your NFS server. See the [NFS documentation](../high_availability/nfs.md) + to configure your NFS server. See the [NFS documentation](../nfs.md) for examples and the various options. 1. Create the shared directories. These may be different depending on your NFS @@ -852,7 +855,7 @@ along with [Gitaly](#configure-gitaly), are recommended over using NFS whenever possible. However, if you intend to use GitLab Pages, [you must use NFS](troubleshooting.md#gitlab-pages-requires-nfs). -For information about configuring NFS, see the [NFS documentation page](../high_availability/nfs.md). +For information about configuring NFS, see the [NFS documentation page](../nfs.md). <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> diff --git a/doc/administration/reference_architectures/3k_users.md b/doc/administration/reference_architectures/3k_users.md index 04cb9fa4769..2f88413de6f 100644 --- a/doc/administration/reference_architectures/3k_users.md +++ b/doc/administration/reference_architectures/3k_users.md @@ -1,49 +1,54 @@ --- reading_time: true +stage: Enablement +group: Distribution +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# Reference architecture: up to 3,000 users +# Reference architecture: up to 3,000 users **(PREMIUM ONLY)** -This page describes GitLab reference architecture for up to 3,000 users. -For a full list of reference architectures, see +This page describes GitLab reference architecture for up to 3,000 users. For a +full list of reference architectures, see [Available reference architectures](index.md#available-reference-architectures). NOTE: **Note:** -The 3,000-user reference architecture documented below is -designed to help your organization achieve a highly-available GitLab deployment. -If you do not have the expertise or need to maintain a highly-available -environment, you can have a simpler and less costly-to-operate environment by -following the [2,000-user reference architecture](2k_users.md). +This reference architecture is designed to help your organization achieve a +highly-available GitLab deployment. If you do not have the expertise or need to +maintain a highly-available environment, you can have a simpler and less +costly-to-operate environment by using the +[2,000-user reference architecture](2k_users.md). > - **Supported users (approximate):** 3,000 -> - **High Availability:** True -> - **Test RPS rates:** API: 60 RPS, Web: 6 RPS, Git: 6 RPS - -| Service | Nodes | Configuration | GCP | AWS | Azure | -|--------------------------------------------------------------|-------|---------------------------------|-----------------|-------------------------|----------------| -| External load balancing node | 1 | 2 vCPU, 1.8GB Memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| Redis | 3 | 2 vCPU, 7.5GB Memory | `n1-standard-2` | `m5.large` | `D2s v3` | -| Consul + Sentinel | 3 | 2 vCPU, 1.8GB Memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| PostgreSQL | 3 | 2 vCPU, 7.5GB Memory | `n1-standard-2` | `m5.large` | `D2s v3` | -| PgBouncer | 3 | 2 vCPU, 1.8GB Memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| Internal load balancing node | 1 | 2 vCPU, 1.8GB Memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| Gitaly | 2 minimum | 4 vCPU, 15GB Memory | `n1-standard-4` | `m5.xlarge` | `D4s v3` | -| Sidekiq | 4 | 2 vCPU, 7.5GB Memory | `n1-standard-2` | `m5.large` | `D2s v3` | -| GitLab Rails | 3 | 8 vCPU, 7.2GB Memory | `n1-highcpu-8` | `c5.2xlarge` | `F8s v2` | -| Monitoring node | 1 | 2 vCPU, 1.8GB Memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| Object Storage | n/a | n/a | n/a | n/a | n/a | -| NFS Server (optional, not recommended) | 1 | 4 vCPU, 3.6GB Memory | `n1-highcpu-4` | `c5.xlarge` | `F4s v2` | - -The architectures were built and tested with the [Intel Xeon E5 v3 (Haswell)](https://cloud.google.com/compute/docs/cpu-platforms) -CPU platform on GCP. On different hardware you may find that adjustments, either lower -or higher, are required for your CPU or Node counts accordingly. For more information, a -[Sysbench](https://github.com/akopytov/sysbench) benchmark of the CPU can be found -[here](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Reference-Architectures/GCP-CPU-Benchmarks). - -For data objects such as LFS, Uploads, Artifacts, etc, an [object storage service](#configure-the-object-storage) -is recommended over NFS where possible, due to better performance and availability. -Since this doesn't require a node to be set up, it's marked as not applicable (n/a) -in the table above. +> - **High Availability:** Yes +> - **Test requests per second (RPS) rates:** API: 60 RPS, Web: 6 RPS, Git: 6 RPS + +| Service | Nodes | Configuration | GCP | AWS | Azure | +|--------------------------------------------|-------------|-----------------------|----------------|-------------|---------| +| External load balancing node | 1 | 2 vCPU, 1.8GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| Redis | 3 | 2 vCPU, 7.5GB memory | n1-standard-2 | m5.large | D2s v3 | +| Consul + Sentinel | 3 | 2 vCPU, 1.8GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| PostgreSQL | 3 | 2 vCPU, 7.5GB memory | n1-standard-2 | m5.large | D2s v3 | +| PgBouncer | 3 | 2 vCPU, 1.8GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| Internal load balancing node | 1 | 2 vCPU, 1.8GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| Gitaly | 2 (minimum) | 4 vCPU, 15GB memory | n1-standard-4 | m5.xlarge | D4s v3 | +| Sidekiq | 4 | 2 vCPU, 7.5GB memory | n1-standard-2 | m5.large | D2s v3 | +| GitLab Rails | 3 | 8 vCPU, 7.2GB memory | n1-highcpu-8 | c5.2xlarge | F8s v2 | +| Monitoring node | 1 | 2 vCPU, 1.8GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| Object Storage | n/a | n/a | n/a | n/a | n/a | +| NFS Server (optional, not recommended) | 1 | 4 vCPU, 3.6GB memory | n1-highcpu-4 | c5.xlarge | F4s v2 | + +The Google Cloud Platform (GCP) architectures were built and tested using the +[Intel Xeon E5 v3 (Haswell)](https://cloud.google.com/compute/docs/cpu-platforms) +CPU platform. On different hardware you may find that adjustments, either lower +or higher, are required for your CPU or node counts. For more information, see +our [Sysbench](https://github.com/akopytov/sysbench)-based +[CPU benchmark](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Reference-Architectures/GCP-CPU-Benchmarks). + +For data objects (such as LFS, Uploads, or Artifacts), an +[object storage service](#configure-the-object-storage) is recommended instead +of NFS where possible, due to better performance and availability. Since this +doesn't require a node to be set up, *Object Storage* is noted as not +applicable (n/a) in the previous table. ## Setup components @@ -804,10 +809,11 @@ SSH into the **primary node**: gitlab-psql -d gitlabhq_production ``` -1. Enable the `pg_trgm` extension: +1. Enable the `pg_trgm` and `btree_gist` extensions: ```shell CREATE EXTENSION pg_trgm; + CREATE EXTENSION btree_gist; ``` 1. Exit the database prompt by typing `\q` and Enter. @@ -1439,7 +1445,7 @@ On each node perform the following: 1. Specify the necessary NFS mounts in `/etc/fstab`. The exact contents of `/etc/fstab` will depend on how you chose - to configure your NFS server. See the [NFS documentation](../high_availability/nfs.md) + to configure your NFS server. See the [NFS documentation](../nfs.md) for examples and the various options. 1. Create the shared directories. These may be different depending on your NFS @@ -1748,7 +1754,7 @@ work. are recommended over NFS wherever possible for improved performance. If you intend to use GitLab Pages, this currently [requires NFS](troubleshooting.md#gitlab-pages-requires-nfs). -See how to [configure NFS](../high_availability/nfs.md). +See how to [configure NFS](../nfs.md). <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> diff --git a/doc/administration/reference_architectures/50k_users.md b/doc/administration/reference_architectures/50k_users.md index 2540fe68dac..565845b4bf5 100644 --- a/doc/administration/reference_architectures/50k_users.md +++ b/doc/administration/reference_architectures/50k_users.md @@ -1,76 +1,2047 @@ -# Reference architecture: up to 50,000 users +--- +reading_time: true +stage: Enablement +group: Distribution +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- -This page describes GitLab reference architecture for up to 50,000 users. -For a full list of reference architectures, see +# Reference architecture: up to 50,000 users **(PREMIUM ONLY)** + +This page describes GitLab reference architecture for up to 50,000 users. For a +full list of reference architectures, see [Available reference architectures](index.md#available-reference-architectures). > - **Supported users (approximate):** 50,000 -> - **High Availability:** True -> - **Test RPS rates:** API: 1000 RPS, Web: 100 RPS, Git: 100 RPS - -| Service | Nodes | Configuration ([8](#footnotes)) | GCP | AWS | Azure | -|--------------------------------------------------------------|-------|---------------------------------|----------------|-----------------------|----------------| -| GitLab Rails ([1](#footnotes)) | 12 | 32 vCPU, 28.8GB Memory | `n1-highcpu-32` | `c5.9xlarge` | F32s v2 | -| PostgreSQL | 3 | 16 vCPU, 60GB Memory | `n1-standard-16` | `m5.4xlarge` | D16s v3 | -| PgBouncer | 3 | 2 vCPU, 1.8GB Memory | `n1-highcpu-2` | `c5.large` | F2s v2 | -| Gitaly ([2](#footnotes)) ([5](#footnotes)) ([7](#footnotes)) | X | 64 vCPU, 240GB Memory | `n1-standard-64` | `m5.16xlarge` | D64s v3 | -| Redis ([3](#footnotes)) - Cache | 3 | 4 vCPU, 15GB Memory | `n1-standard-4` | `m5.xlarge` | D4s v3 | -| Redis ([3](#footnotes)) - Queues / Shared State | 3 | 4 vCPU, 15GB Memory | `n1-standard-4` | `m5.xlarge` | D4s v3 | -| Redis Sentinel ([3](#footnotes)) - Cache | 3 | 1 vCPU, 1.7GB Memory | `g1-small` | `t2.small` | B1MS | -| Redis Sentinel ([3](#footnotes)) - Queues / Shared State | 3 | 1 vCPU, 1.7GB Memory | `g1-small` | `t2.small` | B1MS | -| Consul | 3 | 2 vCPU, 1.8GB Memory | `n1-highcpu-2` | `c5.large` | F2s v2 | -| Sidekiq | 4 | 4 vCPU, 15GB Memory | `n1-standard-4` | `m5.xlarge` | D4s v3 | -| NFS Server ([5](#footnotes)) ([7](#footnotes)) | 1 | 4 vCPU, 3.6GB Memory | `n1-highcpu-4` | `c5.xlarge` | F4s v2 | -| Object Storage ([4](#footnotes)) | - | - | - | - | - | -| Monitoring node | 1 | 4 vCPU, 3.6GB Memory | `n1-highcpu-4` | `c5.xlarge` | F4s v2 | -| External load balancing node ([6](#footnotes)) | 1 | 8 vCPU, 7.2GB Memory | `n1-highcpu-8` | `c5.2xlarge` | F8s v2 | -| Internal load balancing node ([6](#footnotes)) | 1 | 8 vCPU, 7.2GB Memory | `n1-highcpu-8` | `c5.2xlarge` | F8s v2 | - -## Footnotes - -1. In our architectures we run each GitLab Rails node using the Puma webserver - and have its number of workers set to 90% of available CPUs along with four threads. For - nodes that are running Rails with other components the worker value should be reduced - accordingly where we've found 50% achieves a good balance but this is dependent - on workload. - -1. Gitaly node requirements are dependent on customer data, specifically the number of - projects and their sizes. We recommend two nodes as an absolute minimum for HA environments - and at least four nodes should be used when supporting 50,000 or more users. - We also recommend that each Gitaly node should store no more than 5TB of data - and have the number of [`gitaly-ruby` workers](../gitaly/index.md#gitaly-ruby) - set to 20% of available CPUs. Additional nodes should be considered in conjunction - with a review of expected data size and spread based on the recommendations above. - -1. Recommended Redis setup differs depending on the size of the architecture. - For smaller architectures (less than 3,000 users) a single instance should suffice. - For medium sized installs (3,000 - 5,000) we suggest one Redis cluster for all - classes and that Redis Sentinel is hosted alongside Consul. - For larger architectures (10,000 users or more) we suggest running a separate - [Redis Cluster](../redis/replication_and_failover.md#running-multiple-redis-clusters) for the Cache class - and another for the Queues and Shared State classes respectively. We also recommend - that you run the Redis Sentinel clusters separately for each Redis Cluster. - -1. For data objects such as LFS, Uploads, Artifacts, etc. We recommend an [Object Storage service](../object_storage.md) - over NFS where possible, due to better performance and availability. - -1. NFS can be used as an alternative for both repository data (replacing Gitaly) and - object storage but this isn't typically recommended for performance reasons. Note however it is required for - [GitLab Pages](https://gitlab.com/gitlab-org/gitlab-pages/-/issues/196). - -1. Our architectures have been tested and validated with [HAProxy](https://www.haproxy.org/) - as the load balancer. Although other load balancers with similar feature sets - could also be used, those load balancers have not been validated. - -1. We strongly recommend that any Gitaly or NFS nodes be set up with SSD disks over - HDD with a throughput of at least 8,000 IOPS for read operations and 2,000 IOPS for write - as these components have heavy I/O. These IOPS values are recommended only as a starter - as with time they may be adjusted higher or lower depending on the scale of your - environment's workload. If you're running the environment on a Cloud provider - you may need to refer to their documentation on how configure IOPS correctly. - -1. The architectures were built and tested with the [Intel Xeon E5 v3 (Haswell)](https://cloud.google.com/compute/docs/cpu-platforms) - CPU platform on GCP. On different hardware you may find that adjustments, either lower - or higher, are required for your CPU or Node counts accordingly. For more information, a - [Sysbench](https://github.com/akopytov/sysbench) benchmark of the CPU can be found - [here](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Reference-Architectures/GCP-CPU-Benchmarks). +> - **High Availability:** Yes +> - **Test requests per second (RPS) rates:** API: 1000 RPS, Web: 100 RPS, Git: 100 RPS + +| Service | Nodes | Configuration | GCP | AWS | Azure | +|-----------------------------------------|-------------|-------------------------|-----------------|--------------|----------| +| External load balancing node | 1 | 8 vCPU, 7.2GB memory | n1-highcpu-8 | c5.2xlarge | F8s v2 | +| Consul | 3 | 2 vCPU, 1.8GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| PostgreSQL | 3 | 16 vCPU, 60GB memory | n1-standard-16 | m5.4xlarge | D16s v3 | +| PgBouncer | 3 | 2 vCPU, 1.8GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| Internal load balancing node | 1 | 8 vCPU, 7.2GB memory | n1-highcpu-8 | c5.2xlarge | F8s v2 | +| Redis - Cache | 3 | 4 vCPU, 15GB memory | n1-standard-4 | m5.xlarge | D4s v3 | +| Redis - Queues / Shared State | 3 | 4 vCPU, 15GB memory | n1-standard-4 | m5.xlarge | D4s v3 | +| Redis Sentinel - Cache | 3 | 1 vCPU, 1.7GB memory | g1-small | t2.small | B1MS | +| Redis Sentinel - Queues / Shared State | 3 | 1 vCPU, 1.7GB memory | g1-small | t2.small | B1MS | +| Gitaly | 2 (minimum) | 64 vCPU, 240GB memory | n1-standard-64 | m5.16xlarge | D64s v3 | +| Sidekiq | 4 | 4 vCPU, 15GB memory | n1-standard-4 | m5.xlarge | D4s v3 | +| GitLab Rails | 12 | 32 vCPU, 28.8GB memory | n1-highcpu-32 | c5.9xlarge | F32s v2 | +| Monitoring node | 1 | 4 vCPU, 3.6GB memory | n1-highcpu-4 | c5.xlarge | F4s v2 | +| Object Storage | n/a | n/a | n/a | n/a | n/a | +| NFS Server | 1 | 4 vCPU, 3.6GB memory | n1-highcpu-4 | c5.xlarge | F4s v2 | + +The Google Cloud Platform (GCP) architectures were built and tested using the +[Intel Xeon E5 v3 (Haswell)](https://cloud.google.com/compute/docs/cpu-platforms) +CPU platform. On different hardware you may find that adjustments, either lower +or higher, are required for your CPU or node counts. For more information, see +our [Sysbench](https://github.com/akopytov/sysbench)-based +[CPU benchmark](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Reference-Architectures/GCP-CPU-Benchmarks). + +For data objects (such as LFS, Uploads, or Artifacts), an +[object storage service](#configure-the-object-storage) is recommended instead +of NFS where possible, due to better performance and availability. Since this +doesn't require a node to be set up, *Object Storage* is noted as not +applicable (n/a) in the previous table. + +## Setup components + +To set up GitLab and its components to accommodate up to 50,000 users: + +1. [Configure the external load balancing node](#configure-the-external-load-balancer) + that will handle the load balancing of the three GitLab application services nodes. +1. [Configure Consul](#configure-consul). +1. [Configure PostgreSQL](#configure-postgresql), the database for GitLab. +1. [Configure PgBouncer](#configure-pgbouncer). +1. [Configure the internal load balancing node](#configure-the-internal-load-balancer) +1. [Configure Redis](#configure-redis). +1. [Configure Gitaly](#configure-gitaly), + which provides access to the Git repositories. +1. [Configure Sidekiq](#configure-sidekiq). +1. [Configure the main GitLab Rails application](#configure-gitlab-rails) + to run Puma/Unicorn, Workhorse, GitLab Shell, and to serve all frontend requests (UI, API, Git + over HTTP/SSH). +1. [Configure Prometheus](#configure-prometheus) to monitor your GitLab environment. +1. [Configure the Object Storage](#configure-the-object-storage) + used for shared data objects. +1. [Configure NFS (Optional)](#configure-nfs-optional) + to have shared disk storage service as an alternative to Gitaly and/or Object Storage (although + not recommended). NFS is required for GitLab Pages, you can skip this step if you're not using + that feature. + +We start with all servers on the same 10.6.0.0/24 private network range, they +can connect to each other freely on those addresses. + +Here is a list and description of each machine and the assigned IP: + +- `10.6.0.10`: External Load Balancer +- `10.6.0.11`: Consul 1 +- `10.6.0.12`: Consul 2 +- `10.6.0.13`: Consul 3 +- `10.6.0.21`: PostgreSQL primary +- `10.6.0.22`: PostgreSQL secondary 1 +- `10.6.0.23`: PostgreSQL secondary 2 +- `10.6.0.31`: PgBouncer 1 +- `10.6.0.32`: PgBouncer 2 +- `10.6.0.33`: PgBouncer 3 +- `10.6.0.40`: Internal Load Balancer +- `10.6.0.51`: Redis - Cache Primary +- `10.6.0.52`: Redis - Cache Replica 1 +- `10.6.0.53`: Redis - Cache Replica 2 +- `10.6.0.71`: Sentinel - Cache 1 +- `10.6.0.72`: Sentinel - Cache 2 +- `10.6.0.73`: Sentinel - Cache 3 +- `10.6.0.61`: Redis - Queues Primary +- `10.6.0.62`: Redis - Queues Replica 1 +- `10.6.0.63`: Redis - Queues Replica 2 +- `10.6.0.81`: Sentinel - Queues 1 +- `10.6.0.82`: Sentinel - Queues 2 +- `10.6.0.83`: Sentinel - Queues 3 +- `10.6.0.91`: Gitaly 1 +- `10.6.0.92`: Gitaly 2 +- `10.6.0.101`: Sidekiq 1 +- `10.6.0.102`: Sidekiq 2 +- `10.6.0.103`: Sidekiq 3 +- `10.6.0.104`: Sidekiq 4 +- `10.6.0.111`: GitLab application 1 +- `10.6.0.112`: GitLab application 2 +- `10.6.0.113`: GitLab application 3 +- `10.6.0.121`: Prometheus + +## Configure the external load balancer + +NOTE: **Note:** +This architecture has been tested and validated with [HAProxy](https://www.haproxy.org/) +as the load balancer. Although other load balancers with similar feature sets +could also be used, those load balancers have not been validated. + +In an active/active GitLab configuration, you will need a load balancer to route +traffic to the application servers. The specifics on which load balancer to use +or the exact configuration is beyond the scope of GitLab documentation. We hope +that if you're managing multi-node systems like GitLab you have a load balancer of +choice already. Some examples including HAProxy (open-source), F5 Big-IP LTM, +and Citrix Net Scaler. This documentation will outline what ports and protocols +you need to use with GitLab. + +The next question is how you will handle SSL in your environment. +There are several different options: + +- [The application node terminates SSL](#application-node-terminates-ssl). +- [The load balancer terminates SSL without backend SSL](#load-balancer-terminates-ssl-without-backend-ssl) + and communication is not secure between the load balancer and the application node. +- [The load balancer terminates SSL with backend SSL](#load-balancer-terminates-ssl-with-backend-ssl) + and communication is *secure* between the load balancer and the application node. + +### Application node terminates SSL + +Configure your load balancer to pass connections on port 443 as `TCP` rather +than `HTTP(S)` protocol. This will pass the connection to the application node's +NGINX service untouched. NGINX will have the SSL certificate and listen on port 443. + +See the [NGINX HTTPS documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https) +for details on managing SSL certificates and configuring NGINX. + +### Load balancer terminates SSL without backend SSL + +Configure your load balancer to use the `HTTP(S)` protocol rather than `TCP`. +The load balancer will then be responsible for managing SSL certificates and +terminating SSL. + +Since communication between the load balancer and GitLab will not be secure, +there is some additional configuration needed. See the +[NGINX proxied SSL documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#supporting-proxied-ssl) +for details. + +### Load balancer terminates SSL with backend SSL + +Configure your load balancer(s) to use the 'HTTP(S)' protocol rather than 'TCP'. +The load balancer(s) will be responsible for managing SSL certificates that +end users will see. + +Traffic will also be secure between the load balancer(s) and NGINX in this +scenario. There is no need to add configuration for proxied SSL since the +connection will be secure all the way. However, configuration will need to be +added to GitLab to configure SSL certificates. See +[NGINX HTTPS documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https) +for details on managing SSL certificates and configuring NGINX. + +### Ports + +The basic ports to be used are shown in the table below. + +| LB Port | Backend Port | Protocol | +| ------- | ------------ | ------------------------ | +| 80 | 80 | HTTP (*1*) | +| 443 | 443 | TCP or HTTPS (*1*) (*2*) | +| 22 | 22 | TCP | + +- (*1*): [Web terminal](../../ci/environments/index.md#web-terminals) support requires + your load balancer to correctly handle WebSocket connections. When using + HTTP or HTTPS proxying, this means your load balancer must be configured + to pass through the `Connection` and `Upgrade` hop-by-hop headers. See the + [web terminal](../integration/terminal.md) integration guide for + more details. +- (*2*): When using HTTPS protocol for port 443, you will need to add an SSL + certificate to the load balancers. If you wish to terminate SSL at the + GitLab application server instead, use TCP protocol. + +If you're using GitLab Pages with custom domain support you will need some +additional port configurations. +GitLab Pages requires a separate virtual IP address. Configure DNS to point the +`pages_external_url` from `/etc/gitlab/gitlab.rb` at the new virtual IP address. See the +[GitLab Pages documentation](../pages/index.md) for more information. + +| LB Port | Backend Port | Protocol | +| ------- | ------------- | --------- | +| 80 | Varies (*1*) | HTTP | +| 443 | Varies (*1*) | TCP (*2*) | + +- (*1*): The backend port for GitLab Pages depends on the + `gitlab_pages['external_http']` and `gitlab_pages['external_https']` + setting. See [GitLab Pages documentation](../pages/index.md) for more details. +- (*2*): Port 443 for GitLab Pages should always use the TCP protocol. Users can + configure custom domains with custom SSL, which would not be possible + if SSL was terminated at the load balancer. + +#### Alternate SSH Port + +Some organizations have policies against opening SSH port 22. In this case, +it may be helpful to configure an alternate SSH hostname that allows users +to use SSH on port 443. An alternate SSH hostname will require a new virtual IP address +compared to the other GitLab HTTP configuration above. + +Configure DNS for an alternate SSH hostname such as `altssh.gitlab.example.com`. + +| LB Port | Backend Port | Protocol | +| ------- | ------------ | -------- | +| 443 | 22 | TCP | + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure Consul + +The following IPs will be used as an example: + +- `10.6.0.11`: Consul 1 +- `10.6.0.12`: Consul 2 +- `10.6.0.13`: Consul 3 + +NOTE: **Note:** +The configuration processes for the other servers in your reference architecture will +use the `/etc/gitlab/gitlab-secrets.json` file from your Consul server to connect +with the other servers. + +To configure Consul: + +1. SSH into the server that will host Consul. +1. [Download/install](https://about.gitlab.com/install/) the + Omnibus GitLab Enterprise Edition package using **steps 1 and 2** from the + GitLab downloads page. + - Make sure you select the correct Omnibus package, with the same version + the GitLab application is running. + - Do not complete any other steps on the download page. + +1. Edit `/etc/gitlab/gitlab.rb` and add the contents: + + ```ruby + roles ['consul_role'] + + ## Enable service discovery for Prometheus + consul['enable'] = true + consul['monitoring_service_discovery'] = true + + ## The IPs of the Consul server nodes + ## You can also use FQDNs and intermix them with IPs + consul['configuration'] = { + server: true, + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), + } + + # Set the network addresses that the exporters will listen on + node_exporter['listen_address'] = '0.0.0.0:9100' + + # Disable auto migrations + gitlab_rails['auto_migrate'] = false + ``` + +1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. +1. Go through the steps again for all the other Consul nodes, and + make sure you set up the correct IPs. + +NOTE: **Note:** +A Consul leader will be elected when the provisioning of the third Consul server is completed. +Viewing the Consul logs `sudo gitlab-ctl tail consul` will display +`...[INFO] consul: New leader elected: ...` + +You can list the current Consul members (server, client): + +```shell +sudo /opt/gitlab/embedded/bin/consul members +``` + +You can verify the GitLab services are running: + +```shell +sudo gitlab-ctl status +``` + +The output should be similar to the following: + +```plaintext +run: consul: (pid 30074) 76834s; run: log: (pid 29740) 76844s +run: logrotate: (pid 30925) 3041s; run: log: (pid 29649) 76861s +run: node-exporter: (pid 30093) 76833s; run: log: (pid 29663) 76855s +``` + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure PostgreSQL + +In this section, you'll be guided through configuring an external PostgreSQL database +to be used with GitLab. + +### Provide your own PostgreSQL instance + +If you're hosting GitLab on a cloud provider, you can optionally use a +managed service for PostgreSQL. For example, AWS offers a managed Relational +Database Service (RDS) that runs PostgreSQL. + +If you use a cloud-managed service, or provide your own PostgreSQL: + +1. Set up PostgreSQL according to the + [database requirements document](../../install/requirements.md#database). +1. Set up a `gitlab` username with a password of your choice. The `gitlab` user + needs privileges to create the `gitlabhq_production` database. +1. Configure the GitLab application servers with the appropriate details. + This step is covered in [Configuring the GitLab Rails application](#configure-gitlab-rails). + +### Standalone PostgreSQL using Omnibus GitLab + +The following IPs will be used as an example: + +- `10.6.0.21`: PostgreSQL primary +- `10.6.0.22`: PostgreSQL secondary 1 +- `10.6.0.23`: PostgreSQL secondary 2 + +First, make sure to [install](https://about.gitlab.com/install/) +the Linux GitLab package **on each node**. Following the steps, +install the necessary dependencies from step 1, and add the +GitLab package repository from step 2. When installing GitLab +in the second step, do not supply the `EXTERNAL_URL` value. + +#### PostgreSQL primary node + +1. SSH into the PostgreSQL primary node. +1. Generate a password hash for the PostgreSQL username/password pair. This assumes you will use the default + username of `gitlab` (recommended). The command will request a password + and confirmation. Use the value that is output by this command in the next + step as the value of `<postgresql_password_hash>`: + + ```shell + sudo gitlab-ctl pg-password-md5 gitlab + ``` + +1. Generate a password hash for the PgBouncer username/password pair. This assumes you will use the default + username of `pgbouncer` (recommended). The command will request a password + and confirmation. Use the value that is output by this command in the next + step as the value of `<pgbouncer_password_hash>`: + + ```shell + sudo gitlab-ctl pg-password-md5 pgbouncer + ``` + +1. Generate a password hash for the Consul database username/password pair. This assumes you will use the default + username of `gitlab-consul` (recommended). The command will request a password + and confirmation. Use the value that is output by this command in the next + step as the value of `<consul_password_hash>`: + + ```shell + sudo gitlab-ctl pg-password-md5 gitlab-consul + ``` + +1. On the primary database node, edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: + + ```ruby + # Disable all components except PostgreSQL and Repmgr and Consul + roles ['postgres_role'] + + # PostgreSQL configuration + postgresql['listen_address'] = '0.0.0.0' + postgresql['hot_standby'] = 'on' + postgresql['wal_level'] = 'replica' + postgresql['shared_preload_libraries'] = 'repmgr_funcs' + + # Disable automatic database migrations + gitlab_rails['auto_migrate'] = false + + # Configure the Consul agent + consul['services'] = %w(postgresql) + + # START user configuration + # Please set the real values as explained in Required Information section + # + # Replace PGBOUNCER_PASSWORD_HASH with a generated md5 value + postgresql['pgbouncer_user_password'] = '<pgbouncer_password_hash>' + # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value + postgresql['sql_user_password'] = '<postgresql_password_hash>' + # Set `max_wal_senders` to one more than the number of database nodes in the cluster. + # This is used to prevent replication from using up all of the + # available database connections. + postgresql['max_wal_senders'] = 4 + postgresql['max_replication_slots'] = 4 + + # Replace XXX.XXX.XXX.XXX/YY with Network Address + postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24) + repmgr['trust_auth_cidr_addresses'] = %w(127.0.0.1/32 10.6.0.0/24) + + ## Enable service discovery for Prometheus + consul['monitoring_service_discovery'] = true + + # Set the network addresses that the exporters will listen on for monitoring + node_exporter['listen_address'] = '0.0.0.0:9100' + postgres_exporter['listen_address'] = '0.0.0.0:9187' + + ## The IPs of the Consul server nodes + ## You can also use FQDNs and intermix them with IPs + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), + } + # + # END user configuration + ``` + +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace + the file of the same name on this server. If that file is not on this server, + add the file from your Consul server to this server. + +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +#### PostgreSQL secondary nodes + +1. On both the secondary nodes, add the same configuration specified above for the primary node + with an additional setting (`repmgr['master_on_initialization'] = false`) that will inform `gitlab-ctl` that they are standby nodes initially + and there's no need to attempt to register them as a primary node: + + ```ruby + # Disable all components except PostgreSQL and Repmgr and Consul + roles ['postgres_role'] + + # PostgreSQL configuration + postgresql['listen_address'] = '0.0.0.0' + postgresql['hot_standby'] = 'on' + postgresql['wal_level'] = 'replica' + postgresql['shared_preload_libraries'] = 'repmgr_funcs' + + # Disable automatic database migrations + gitlab_rails['auto_migrate'] = false + + # Configure the Consul agent + consul['services'] = %w(postgresql) + + # Specify if a node should attempt to be primary on initialization. + repmgr['master_on_initialization'] = false + + # Replace PGBOUNCER_PASSWORD_HASH with a generated md5 value + postgresql['pgbouncer_user_password'] = '<pgbouncer_password_hash>' + # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value + postgresql['sql_user_password'] = '<postgresql_password_hash>' + # Set `max_wal_senders` to one more than the number of database nodes in the cluster. + # This is used to prevent replication from using up all of the + # available database connections. + postgresql['max_wal_senders'] = 4 + postgresql['max_replication_slots'] = 4 + + # Replace with your network addresses + postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24) + repmgr['trust_auth_cidr_addresses'] = %w(127.0.0.1/32 10.6.0.0/24) + + ## Enable service discovery for Prometheus + consul['monitoring_service_discovery'] = true + + # Set the network addresses that the exporters will listen on for monitoring + node_exporter['listen_address'] = '0.0.0.0:9100' + postgres_exporter['listen_address'] = '0.0.0.0:9187' + + ## The IPs of the Consul server nodes + ## You can also use FQDNs and intermix them with IPs + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), + } + ``` + +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace + the file of the same name on this server. If that file is not on this server, + add the file from your Consul server to this server. + +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + +Advanced [configuration options](https://docs.gitlab.com/omnibus/settings/database.html) +are supported and can be added if needed. + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +#### PostgreSQL post-configuration + +SSH into the **primary node**: + +1. Open a database prompt: + + ```shell + gitlab-psql -d gitlabhq_production + ``` + +1. Make sure the `pg_trgm` extension is enabled (it might already be): + + ```shell + CREATE EXTENSION pg_trgm; + ``` + +1. Exit the database prompt by typing `\q` and Enter. + +1. Verify the cluster is initialized with one node: + + ```shell + gitlab-ctl repmgr cluster show + ``` + + The output should be similar to the following: + + ```plaintext + Role | Name | Upstream | Connection String + ----------+----------|----------|---------------------------------------- + * master | HOSTNAME | | host=HOSTNAME user=gitlab_repmgr dbname=gitlab_repmgr + ``` + +1. Note down the hostname or IP address in the connection string: `host=HOSTNAME`. We will + refer to the hostname in the next section as `<primary_node_name>`. If the value + is not an IP address, it will need to be a resolvable name (via DNS or + `/etc/hosts`) + +SSH into the **secondary node**: + +1. Set up the repmgr standby: + + ```shell + gitlab-ctl repmgr standby setup <primary_node_name> + ``` + + Do note that this will remove the existing data on the node. The command + has a wait time. + + The output should be similar to the following: + + ```console + Doing this will delete the entire contents of /var/opt/gitlab/postgresql/data + If this is not what you want, hit Ctrl-C now to exit + To skip waiting, rerun with the -w option + Sleeping for 30 seconds + Stopping the database + Removing the data + Cloning the data + Starting the database + Registering the node with the cluster + ok: run: repmgrd: (pid 19068) 0s + ``` + +Before moving on, make sure the databases are configured correctly. Run the +following command on the **primary** node to verify that replication is working +properly and the secondary nodes appear in the cluster: + +```shell +gitlab-ctl repmgr cluster show +``` + +The output should be similar to the following: + +```plaintext +Role | Name | Upstream | Connection String +----------+---------|-----------|------------------------------------------------ +* master | MASTER | | host=<primary_node_name> user=gitlab_repmgr dbname=gitlab_repmgr + standby | STANDBY | MASTER | host=<secondary_node_name> user=gitlab_repmgr dbname=gitlab_repmgr + standby | STANDBY | MASTER | host=<secondary_node_name> user=gitlab_repmgr dbname=gitlab_repmgr +``` + +If the 'Role' column for any node says "FAILED", check the +[Troubleshooting section](troubleshooting.md) before proceeding. + +Also, check that the `repmgr-check-master` command works successfully on each node: + +```shell +su - gitlab-consul +gitlab-ctl repmgr-check-master || echo 'This node is a standby repmgr node' +``` + +This command relies on exit codes to tell Consul whether a particular node is a master +or secondary. The most important thing here is that this command does not produce errors. +If there are errors it's most likely due to incorrect `gitlab-consul` database user permissions. +Check the [Troubleshooting section](troubleshooting.md) before proceeding. + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure PgBouncer + +Now that the PostgreSQL servers are all set up, let's configure PgBouncer. +The following IPs will be used as an example: + +- `10.6.0.31`: PgBouncer 1 +- `10.6.0.32`: PgBouncer 2 +- `10.6.0.33`: PgBouncer 3 + +1. On each PgBouncer node, edit `/etc/gitlab/gitlab.rb`, and replace + `<consul_password_hash>` and `<pgbouncer_password_hash>` with the + password hashes you [set up previously](#postgresql-primary-node): + + ```ruby + # Disable all components except Pgbouncer and Consul agent + roles ['pgbouncer_role'] + + # Configure PgBouncer + pgbouncer['admin_users'] = %w(pgbouncer gitlab-consul) + + pgbouncer['users'] = { + 'gitlab-consul': { + password: '<consul_password_hash>' + }, + 'pgbouncer': { + password: '<pgbouncer_password_hash>' + } + } + + # Configure Consul agent + consul['watchers'] = %w(postgresql) + consul['enable'] = true + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13) + } + + # Enable service discovery for Prometheus + consul['monitoring_service_discovery'] = true + + # Set the network addresses that the exporters will listen on + node_exporter['listen_address'] = '0.0.0.0:9100' + ``` + +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace + the file of the same name on this server. If that file is not on this server, + add the file from your Consul server to this server. + +1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + + NOTE: **Note:** + If an error `execute[generate databases.ini]` occurs, this is due to an existing + [known issue](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/4713). + It will be resolved when you run a second `reconfigure` after the next step. + +1. Create a `.pgpass` file so Consul is able to + reload PgBouncer. Enter the PgBouncer password twice when asked: + + ```shell + gitlab-ctl write-pgpass --host 127.0.0.1 --database pgbouncer --user pgbouncer --hostuser gitlab-consul + ``` + +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) once again + to resolve any potential errors from the previous steps. +1. Ensure each node is talking to the current primary: + + ```shell + gitlab-ctl pgb-console # You will be prompted for PGBOUNCER_PASSWORD + ``` + +1. Once the console prompt is available, run the following queries: + + ```shell + show databases ; show clients ; + ``` + + The output should be similar to the following: + + ```plaintext + name | host | port | database | force_user | pool_size | reserve_pool | pool_mode | max_connections | current_connections + ---------------------+-------------+------+---------------------+------------+-----------+--------------+-----------+-----------------+--------------------- + gitlabhq_production | MASTER_HOST | 5432 | gitlabhq_production | | 20 | 0 | | 0 | 0 + pgbouncer | | 6432 | pgbouncer | pgbouncer | 2 | 0 | statement | 0 | 0 + (2 rows) + + type | user | database | state | addr | port | local_addr | local_port | connect_time | request_time | ptr | link | remote_pid | tls + ------+-----------+---------------------+---------+----------------+-------+------------+------------+---------------------+---------------------+-----------+------+------------+----- + C | pgbouncer | pgbouncer | active | 127.0.0.1 | 56846 | 127.0.0.1 | 6432 | 2017-08-21 18:09:59 | 2017-08-21 18:10:48 | 0x22b3880 | | 0 | + (2 rows) + ``` + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +### Configure the internal load balancer + +If you're running more than one PgBouncer node as recommended, then at this time you'll need to set +up a TCP internal load balancer to serve each correctly. + +The following IP will be used as an example: + +- `10.6.0.40`: Internal Load Balancer + +Here's how you could do it with [HAProxy](https://www.haproxy.org/): + +```plaintext +global + log /dev/log local0 + log localhost local1 notice + log stdout format raw local0 + +defaults + log global + default-server inter 10s fall 3 rise 2 + balance leastconn + +frontend internal-pgbouncer-tcp-in + bind *:6432 + mode tcp + option tcplog + + default_backend pgbouncer + +backend pgbouncer + mode tcp + option tcp-check + + server pgbouncer1 10.6.0.21:6432 check + server pgbouncer2 10.6.0.22:6432 check + server pgbouncer3 10.6.0.23:6432 check +``` + +Refer to your preferred Load Balancer's documentation for further guidance. + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure Redis + +Using [Redis](https://redis.io/) in scalable environment is possible using a **Primary** x **Replica** +topology with a [Redis Sentinel](https://redis.io/topics/sentinel) service to watch and automatically +start the failover procedure. + +Redis requires authentication if used with Sentinel. See +[Redis Security](https://redis.io/topics/security) documentation for more +information. We recommend using a combination of a Redis password and tight +firewall rules to secure your Redis service. +You are highly encouraged to read the [Redis Sentinel](https://redis.io/topics/sentinel) documentation +before configuring Redis with GitLab to fully understand the topology and +architecture. + +The requirements for a Redis setup are the following: + +1. All Redis nodes must be able to talk to each other and accept incoming + connections over Redis (`6379`) and Sentinel (`26379`) ports (unless you + change the default ones). +1. The server that hosts the GitLab application must be able to access the + Redis nodes. +1. Protect the nodes from access from external networks + ([Internet](https://gitlab.com/gitlab-org/gitlab-foss/uploads/c4cc8cd353604bd80315f9384035ff9e/The_Internet_IT_Crowd.png)), + using a firewall. + +In this section, you'll be guided through configuring two external Redis clusters +to be used with GitLab. The following IPs will be used as an example: + +- `10.6.0.51`: Redis - Cache Primary +- `10.6.0.52`: Redis - Cache Replica 1 +- `10.6.0.53`: Redis - Cache Replica 2 +- `10.6.0.71`: Sentinel - Cache 1 +- `10.6.0.72`: Sentinel - Cache 2 +- `10.6.0.73`: Sentinel - Cache 3 +- `10.6.0.61`: Redis - Queues Primary +- `10.6.0.62`: Redis - Queues Replica 1 +- `10.6.0.63`: Redis - Queues Replica 2 +- `10.6.0.81`: Sentinel - Queues 1 +- `10.6.0.82`: Sentinel - Queues 2 +- `10.6.0.83`: Sentinel - Queues 3 + +NOTE: **Providing your own Redis instance:** +Managed Redis from cloud providers such as AWS ElastiCache will work. If these +services support high availability, be sure it is **not** the Redis Cluster type. +Redis version 5.0 or higher is required, as this is what ships with +Omnibus GitLab packages starting with GitLab 13.0. Older Redis versions +do not support an optional count argument to SPOP which is now required for +[Merge Trains](../../ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md). +Note the Redis node's IP address or hostname, port, and password (if required). +These will be necessary when configuring the +[GitLab application servers](#configure-gitlab-rails) later. + +### Configure the Redis and Sentinel Cache cluster + +This is the section where we install and set up the new Redis Cache instances. + +NOTE: **Note:** +Redis nodes (both primary and replica) will need the same password defined in +`redis['password']`. At any time during a failover the Sentinels can +reconfigure a node and change its status from primary to replica and vice versa. + +#### Configure the primary Redis Cache node + +1. SSH into the **Primary** Redis server. +1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab + package you want using **steps 1 and 2** from the GitLab downloads page. + - Make sure you select the correct Omnibus package, with the same version + and type (Community, Enterprise editions) of your current install. + - Do not complete any other steps on the download page. + +1. Edit `/etc/gitlab/gitlab.rb` and add the contents: + + ```ruby + # Specify server role as 'redis_master_role' + roles ['redis_master_role'] + + # IP address pointing to a local IP that the other machines can reach to. + # You can also set bind to '0.0.0.0' which listen in all interfaces. + # If you really need to bind to an external accessible IP, make + # sure you add extra firewall rules to prevent unauthorized access. + redis['bind'] = '10.6.0.51' + + # Define a port so Redis can listen for TCP requests which will allow other + # machines to connect to it. + redis['port'] = 6379 + + # Set up password authentication for Redis (use the same password in all nodes). + redis['password'] = 'REDIS_PRIMARY_PASSWORD_OF_FIRST_CLUSTER' + + ## Enable service discovery for Prometheus + consul['enable'] = true + consul['monitoring_service_discovery'] = true + + ## The IPs of the Consul server nodes + ## You can also use FQDNs and intermix them with IPs + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), + } + + # Set the network addresses that the exporters will listen on + node_exporter['listen_address'] = '0.0.0.0:9100' + redis_exporter['listen_address'] = '0.0.0.0:9121' + + # Prevent database migrations from running on upgrade + gitlab_rails['auto_migrate'] = false + ``` + +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace + the file of the same name on this server. If that file is not on this server, + add the file from your Consul server to this server. + +1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + +NOTE: **Note:** +You can specify multiple roles like sentinel and Redis as: +`roles ['redis_sentinel_role', 'redis_master_role']`. +Read more about [roles](https://docs.gitlab.com/omnibus/roles/). + +#### Configure the replica Redis Cache nodes + +1. SSH into the **replica** Redis server. +1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab + package you want using **steps 1 and 2** from the GitLab downloads page. + - Make sure you select the correct Omnibus package, with the same version + and type (Community, Enterprise editions) of your current install. + - Do not complete any other steps on the download page. + +1. Edit `/etc/gitlab/gitlab.rb` and add the contents: + + ```ruby + # Specify server role as 'redis_replica_role' + roles ['redis_replica_role'] + + # IP address pointing to a local IP that the other machines can reach to. + # You can also set bind to '0.0.0.0' which listen in all interfaces. + # If you really need to bind to an external accessible IP, make + # sure you add extra firewall rules to prevent unauthorized access. + redis['bind'] = '10.6.0.52' + + # Define a port so Redis can listen for TCP requests which will allow other + # machines to connect to it. + redis['port'] = 6379 + + # The same password for Redis authentication you set up for the primary node. + redis['password'] = 'REDIS_PRIMARY_PASSWORD_OF_FIRST_CLUSTER' + + # The IP of the primary Redis node. + redis['master_ip'] = '10.6.0.51' + + # Port of primary Redis server, uncomment to change to non default. Defaults + # to `6379`. + #redis['master_port'] = 6379 + + ## Enable service discovery for Prometheus + consul['enable'] = true + consul['monitoring_service_discovery'] = true + + ## The IPs of the Consul server nodes + ## You can also use FQDNs and intermix them with IPs + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), + } + + # Set the network addresses that the exporters will listen on + node_exporter['listen_address'] = '0.0.0.0:9100' + redis_exporter['listen_address'] = '0.0.0.0:9121' + + # Prevent database migrations from running on upgrade + gitlab_rails['auto_migrate'] = false + ``` + +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace + the file of the same name on this server. If that file is not on this server, + add the file from your Consul server to this server. + +1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. +1. Go through the steps again for all the other replica nodes, and + make sure to set up the IPs correctly. + +NOTE: **Note:** +You can specify multiple roles like sentinel and Redis as: +`roles ['redis_sentinel_role', 'redis_master_role']`. +Read more about [roles](https://docs.gitlab.com/omnibus/roles/). + +These values don't have to be changed again in `/etc/gitlab/gitlab.rb` after +a failover, as the nodes will be managed by the [Sentinels](#configure-the-sentinel-cache-nodes), and even after a +`gitlab-ctl reconfigure`, they will get their configuration restored by +the same Sentinels. + +Advanced [configuration options](https://docs.gitlab.com/omnibus/settings/redis.html) +are supported and can be added if needed. + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +#### Configure the Sentinel Cache nodes + +NOTE: **Note:** +If you are using an external Redis Sentinel instance, be sure +to exclude the `requirepass` parameter from the Sentinel +configuration. This parameter will cause clients to report `NOAUTH +Authentication required.`. [Redis Sentinel 3.2.x does not support +password authentication](https://github.com/antirez/redis/issues/3279). + +Now that the Redis servers are all set up, let's configure the Sentinel +servers. The following IPs will be used as an example: + +- `10.6.0.71`: Sentinel - Cache 1 +- `10.6.0.72`: Sentinel - Cache 2 +- `10.6.0.73`: Sentinel - Cache 3 + +To configure the Sentinel Cache server: + +1. SSH into the server that will host Consul/Sentinel. +1. [Download/install](https://about.gitlab.com/install/) the + Omnibus GitLab Enterprise Edition package using **steps 1 and 2** from the + GitLab downloads page. + - Make sure you select the correct Omnibus package, with the same version + the GitLab application is running. + - Do not complete any other steps on the download page. + +1. Edit `/etc/gitlab/gitlab.rb` and add the contents: + + ```ruby + roles ['redis_sentinel_role'] + + ## Must be the same in every sentinel node + redis['master_name'] = 'gitlab-redis-cache' + + ## The same password for Redis authentication you set up for the primary node. + redis['master_password'] = 'REDIS_PRIMARY_PASSWORD_OF_FIRST_CLUSTER' + + ## The IP of the primary Redis node. + redis['master_ip'] = '10.6.0.51' + + ## Define a port so Redis can listen for TCP requests which will allow other + ## machines to connect to it. + redis['port'] = 6379 + + ## Port of primary Redis server, uncomment to change to non default. Defaults + ## to `6379`. + #redis['master_port'] = 6379 + + ## Configure Sentinel's IP + sentinel['bind'] = '10.6.0.71' + + ## Port that Sentinel listens on, uncomment to change to non default. Defaults + ## to `26379`. + #sentinel['port'] = 26379 + + ## Quorum must reflect the amount of voting sentinels it take to start a failover. + ## Value must NOT be greater then the amount of sentinels. + ## + ## The quorum can be used to tune Sentinel in two ways: + ## 1. If a the quorum is set to a value smaller than the majority of Sentinels + ## we deploy, we are basically making Sentinel more sensible to primary failures, + ## triggering a failover as soon as even just a minority of Sentinels is no longer + ## able to talk with the primary. + ## 1. If a quorum is set to a value greater than the majority of Sentinels, we are + ## making Sentinel able to failover only when there are a very large number (larger + ## than majority) of well connected Sentinels which agree about the primary being down.s + sentinel['quorum'] = 2 + + ## Consider unresponsive server down after x amount of ms. + #sentinel['down_after_milliseconds'] = 10000 + + ## Specifies the failover timeout in milliseconds. It is used in many ways: + ## + ## - The time needed to re-start a failover after a previous failover was + ## already tried against the same primary by a given Sentinel, is two + ## times the failover timeout. + ## + ## - The time needed for a replica replicating to a wrong primary according + ## to a Sentinel current configuration, to be forced to replicate + ## with the right primary, is exactly the failover timeout (counting since + ## the moment a Sentinel detected the misconfiguration). + ## + ## - The time needed to cancel a failover that is already in progress but + ## did not produced any configuration change (REPLICAOF NO ONE yet not + ## acknowledged by the promoted replica). + ## + ## - The maximum time a failover in progress waits for all the replica to be + ## reconfigured as replicas of the new primary. However even after this time + ## the replicas will be reconfigured by the Sentinels anyway, but not with + ## the exact parallel-syncs progression as specified. + #sentinel['failover_timeout'] = 60000 + + ## Enable service discovery for Prometheus + consul['enable'] = true + consul['monitoring_service_discovery'] = true + + ## The IPs of the Consul server nodes + ## You can also use FQDNs and intermix them with IPs + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), + } + + # Set the network addresses that the exporters will listen on + node_exporter['listen_address'] = '0.0.0.0:9100' + redis_exporter['listen_address'] = '0.0.0.0:9121' + + # Disable auto migrations + gitlab_rails['auto_migrate'] = false + ``` + +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace + the file of the same name on this server. If that file is not on this server, + add the file from your Consul server to this server. + +1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. +1. Go through the steps again for all the other Consul/Sentinel nodes, and + make sure you set up the correct IPs. + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +### Configure the Redis and Sentinel Queues cluster + +This is the section where we install and set up the new Redis Queues instances. + +NOTE: **Note:** +Redis nodes (both primary and replica) will need the same password defined in +`redis['password']`. At any time during a failover the Sentinels can +reconfigure a node and change its status from primary to replica and vice versa. + +#### Configure the primary Redis Queues node + +1. SSH into the **Primary** Redis server. +1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab + package you want using **steps 1 and 2** from the GitLab downloads page. + - Make sure you select the correct Omnibus package, with the same version + and type (Community, Enterprise editions) of your current install. + - Do not complete any other steps on the download page. + +1. Edit `/etc/gitlab/gitlab.rb` and add the contents: + + ```ruby + # Specify server role as 'redis_master_role' + roles ['redis_master_role'] + + # IP address pointing to a local IP that the other machines can reach to. + # You can also set bind to '0.0.0.0' which listen in all interfaces. + # If you really need to bind to an external accessible IP, make + # sure you add extra firewall rules to prevent unauthorized access. + redis['bind'] = '10.6.0.61' + + # Define a port so Redis can listen for TCP requests which will allow other + # machines to connect to it. + redis['port'] = 6379 + + # Set up password authentication for Redis (use the same password in all nodes). + redis['password'] = 'REDIS_PRIMARY_PASSWORD_OF_SECOND_CLUSTER' + + ## Enable service discovery for Prometheus + consul['enable'] = true + consul['monitoring_service_discovery'] = true + + ## The IPs of the Consul server nodes + ## You can also use FQDNs and intermix them with IPs + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), + } + + # Set the network addresses that the exporters will listen on + node_exporter['listen_address'] = '0.0.0.0:9100' + redis_exporter['listen_address'] = '0.0.0.0:9121' + ``` + +1. Only the primary GitLab application server should handle migrations. To + prevent database migrations from running on upgrade, add the following + configuration to your `/etc/gitlab/gitlab.rb` file: + + ```ruby + gitlab_rails['auto_migrate'] = false + ``` + +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace + the file of the same name on this server. If that file is not on this server, + add the file from your Consul server to this server. + +1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + +NOTE: **Note:** +You can specify multiple roles like sentinel and Redis as: +`roles ['redis_sentinel_role', 'redis_master_role']`. +Read more about [roles](https://docs.gitlab.com/omnibus/roles/). + +#### Configure the replica Redis Queues nodes + +1. SSH into the **replica** Redis Queue server. +1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab + package you want using **steps 1 and 2** from the GitLab downloads page. + - Make sure you select the correct Omnibus package, with the same version + and type (Community, Enterprise editions) of your current install. + - Do not complete any other steps on the download page. + +1. Edit `/etc/gitlab/gitlab.rb` and add the contents: + + ```ruby + # Specify server role as 'redis_replica_role' + roles ['redis_replica_role'] + + # IP address pointing to a local IP that the other machines can reach to. + # You can also set bind to '0.0.0.0' which listen in all interfaces. + # If you really need to bind to an external accessible IP, make + # sure you add extra firewall rules to prevent unauthorized access. + redis['bind'] = '10.6.0.62' + + # Define a port so Redis can listen for TCP requests which will allow other + # machines to connect to it. + redis['port'] = 6379 + + # The same password for Redis authentication you set up for the primary node. + redis['password'] = 'REDIS_PRIMARY_PASSWORD_OF_SECOND_CLUSTER' + + # The IP of the primary Redis node. + redis['master_ip'] = '10.6.0.61' + + # Port of primary Redis server, uncomment to change to non default. Defaults + # to `6379`. + #redis['master_port'] = 6379 + + ## Enable service discovery for Prometheus + consul['enable'] = true + consul['monitoring_service_discovery'] = true + + ## The IPs of the Consul server nodes + ## You can also use FQDNs and intermix them with IPs + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), + } + + # Set the network addresses that the exporters will listen on + node_exporter['listen_address'] = '0.0.0.0:9100' + redis_exporter['listen_address'] = '0.0.0.0:9121' + + # Disable auto migrations + gitlab_rails['auto_migrate'] = false + ``` + +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace + the file of the same name on this server. If that file is not on this server, + add the file from your Consul server to this server. + +1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. +1. Go through the steps again for all the other replica nodes, and + make sure to set up the IPs correctly. + +NOTE: **Note:** +You can specify multiple roles like sentinel and Redis as: +`roles ['redis_sentinel_role', 'redis_master_role']`. +Read more about [roles](https://docs.gitlab.com/omnibus/roles/). + +These values don't have to be changed again in `/etc/gitlab/gitlab.rb` after +a failover, as the nodes will be managed by the [Sentinels](#configure-the-sentinel-queues-nodes), and even after a +`gitlab-ctl reconfigure`, they will get their configuration restored by +the same Sentinels. + +Advanced [configuration options](https://docs.gitlab.com/omnibus/settings/redis.html) +are supported and can be added if needed. + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +#### Configure the Sentinel Queues nodes + +NOTE: **Note:** +If you are using an external Redis Sentinel instance, be sure +to exclude the `requirepass` parameter from the Sentinel +configuration. This parameter will cause clients to report `NOAUTH +Authentication required.`. [Redis Sentinel 3.2.x does not support +password authentication](https://github.com/antirez/redis/issues/3279). + +Now that the Redis servers are all set up, let's configure the Sentinel +servers. The following IPs will be used as an example: + +- `10.6.0.81`: Sentinel - Queues 1 +- `10.6.0.82`: Sentinel - Queues 2 +- `10.6.0.83`: Sentinel - Queues 3 + +To configure the Sentinel Queues server: + +1. SSH into the server that will host Sentinel. +1. [Download/install](https://about.gitlab.com/install/) the + Omnibus GitLab Enterprise Edition package using **steps 1 and 2** from the + GitLab downloads page. + - Make sure you select the correct Omnibus package, with the same version + the GitLab application is running. + - Do not complete any other steps on the download page. + +1. Edit `/etc/gitlab/gitlab.rb` and add the contents: + + ```ruby + roles ['redis_sentinel_role'] + + ## Must be the same in every sentinel node + redis['master_name'] = 'gitlab-redis-persistent' + + ## The same password for Redis authentication you set up for the primary node. + redis['master_password'] = 'REDIS_PRIMARY_PASSWORD_OF_SECOND_CLUSTER' + + ## The IP of the primary Redis node. + redis['master_ip'] = '10.6.0.61' + + ## Define a port so Redis can listen for TCP requests which will allow other + ## machines to connect to it. + redis['port'] = 6379 + + ## Port of primary Redis server, uncomment to change to non default. Defaults + ## to `6379`. + #redis['master_port'] = 6379 + + ## Configure Sentinel's IP + sentinel['bind'] = '10.6.0.81' + + ## Port that Sentinel listens on, uncomment to change to non default. Defaults + ## to `26379`. + #sentinel['port'] = 26379 + + ## Quorum must reflect the amount of voting sentinels it take to start a failover. + ## Value must NOT be greater then the amount of sentinels. + ## + ## The quorum can be used to tune Sentinel in two ways: + ## 1. If a the quorum is set to a value smaller than the majority of Sentinels + ## we deploy, we are basically making Sentinel more sensible to primary failures, + ## triggering a failover as soon as even just a minority of Sentinels is no longer + ## able to talk with the primary. + ## 1. If a quorum is set to a value greater than the majority of Sentinels, we are + ## making Sentinel able to failover only when there are a very large number (larger + ## than majority) of well connected Sentinels which agree about the primary being down.s + sentinel['quorum'] = 2 + + ## Consider unresponsive server down after x amount of ms. + #sentinel['down_after_milliseconds'] = 10000 + + ## Specifies the failover timeout in milliseconds. It is used in many ways: + ## + ## - The time needed to re-start a failover after a previous failover was + ## already tried against the same primary by a given Sentinel, is two + ## times the failover timeout. + ## + ## - The time needed for a replica replicating to a wrong primary according + ## to a Sentinel current configuration, to be forced to replicate + ## with the right primary, is exactly the failover timeout (counting since + ## the moment a Sentinel detected the misconfiguration). + ## + ## - The time needed to cancel a failover that is already in progress but + ## did not produced any configuration change (REPLICAOF NO ONE yet not + ## acknowledged by the promoted replica). + ## + ## - The maximum time a failover in progress waits for all the replica to be + ## reconfigured as replicas of the new primary. However even after this time + ## the replicas will be reconfigured by the Sentinels anyway, but not with + ## the exact parallel-syncs progression as specified. + #sentinel['failover_timeout'] = 60000 + + ## Enable service discovery for Prometheus + consul['enable'] = true + consul['monitoring_service_discovery'] = true + + ## The IPs of the Consul server nodes + ## You can also use FQDNs and intermix them with IPs + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), + } + + # Set the network addresses that the exporters will listen on + node_exporter['listen_address'] = '0.0.0.0:9100' + redis_exporter['listen_address'] = '0.0.0.0:9121' + + # Disable auto migrations + gitlab_rails['auto_migrate'] = false + ``` + +1. To prevent database migrations from running on upgrade, run: + + ```shell + sudo touch /etc/gitlab/skip-auto-reconfigure + ``` + + Only the primary GitLab application server should handle migrations. + +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace + the file of the same name on this server. If that file is not on this server, + add the file from your Consul server to this server. + +1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. +1. Go through the steps again for all the other Sentinel nodes, and + make sure you set up the correct IPs. + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure Gitaly + +Deploying Gitaly in its own server can benefit GitLab installations that are +larger than a single machine. + +The Gitaly node requirements are dependent on customer data, specifically the number of +projects and their repository sizes. Two nodes are recommended as an absolute minimum. +Each Gitaly node should store no more than 5TB of data and have the number of +[`gitaly-ruby` workers](../gitaly/index.md#gitaly-ruby) set to 20% of available CPUs. +Additional nodes should be considered in conjunction with a review of expected +data size and spread based on the recommendations above. + +It is also strongly recommended that all Gitaly nodes be set up with SSD disks with +a throughput of at least 8,000 IOPS for read operations and 2,000 IOPS for write, +as Gitaly has heavy I/O. These IOPS values are recommended only as a starter as with +time they may be adjusted higher or lower depending on the scale of your environment's workload. +If you're running the environment on a Cloud provider, you may need to refer to +their documentation on how to configure IOPS correctly. + +Some things to note: + +- The GitLab Rails application shards repositories into [repository storages](../repository_storage_paths.md). +- A Gitaly server can host one or more storages. +- A GitLab server can use one or more Gitaly servers. +- Gitaly addresses must be specified in such a way that they resolve + correctly for ALL Gitaly clients. +- Gitaly servers must not be exposed to the public internet, as Gitaly's network + traffic is unencrypted by default. The use of a firewall is highly recommended + to restrict access to the Gitaly server. Another option is to + [use TLS](#gitaly-tls-support). + +TIP: **Tip:** +For more information about Gitaly's history and network architecture see the +[standalone Gitaly documentation](../gitaly/index.md). + +Note: **Note:** +The token referred to throughout the Gitaly documentation is +just an arbitrary password selected by the administrator. It is unrelated to +tokens created for the GitLab API or other similar web API tokens. + +Below we describe how to configure two Gitaly servers, with IPs and +domain names: + +- `10.6.0.91`: Gitaly 1 (`gitaly1.internal`) +- `10.6.0.92`: Gitaly 2 (`gitaly2.internal`) + +The secret token is assumed to be `gitalysecret` and that +your GitLab installation has three repository storages: + +- `default` on Gitaly 1 +- `storage1` on Gitaly 1 +- `storage2` on Gitaly 2 + +On each node: + +1. [Download/Install](https://about.gitlab.com/install/) the Omnibus GitLab + package you want using **steps 1 and 2** from the GitLab downloads page but + **without** providing the `EXTERNAL_URL` value. +1. Edit `/etc/gitlab/gitlab.rb` to configure storage paths, enable + the network listener and configure the token: + + <!-- + updates to following example must also be made at + https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab + --> + + ```ruby + # /etc/gitlab/gitlab.rb + + # Gitaly and GitLab use two shared secrets for authentication, one to authenticate gRPC requests + # to Gitaly, and a second for authentication callbacks from GitLab-Shell to the GitLab internal API. + # The following two values must be the same as their respective values + # of the GitLab Rails application setup + gitaly['auth_token'] = 'gitalysecret' + gitlab_shell['secret_token'] = 'shellsecret' + + # Avoid running unnecessary services on the Gitaly server + postgresql['enable'] = false + redis['enable'] = false + nginx['enable'] = false + puma['enable'] = false + unicorn['enable'] = false + sidekiq['enable'] = false + gitlab_workhorse['enable'] = false + grafana['enable'] = false + + # If you run a seperate monitoring node you can disable these services + alertmanager['enable'] = false + prometheus['enable'] = false + + # Prevent database connections during 'gitlab-ctl reconfigure' + gitlab_rails['rake_cache_clear'] = false + gitlab_rails['auto_migrate'] = false + + # Configure the gitlab-shell API callback URL. Without this, `git push` will + # fail. This can be your 'front door' GitLab URL or an internal load + # balancer. + # Don't forget to copy `/etc/gitlab/gitlab-secrets.json` from web server to Gitaly server. + gitlab_rails['internal_api_url'] = 'https://gitlab.example.com' + + # Make Gitaly accept connections on all network interfaces. You must use + # firewalls to restrict access to this address/port. + # Comment out following line if you only want to support TLS connections + gitaly['listen_addr'] = "0.0.0.0:8075" + ``` + +1. Append the following to `/etc/gitlab/gitlab.rb` for each respective server: + 1. On `gitaly1.internal`: + + ```ruby + git_data_dirs({ + 'default' => { + 'path' => '/var/opt/gitlab/git-data' + }, + 'storage1' => { + 'path' => '/mnt/gitlab/git-data' + }, + }) + ``` + + 1. On `gitaly2.internal`: + + ```ruby + git_data_dirs({ + 'storage2' => { + 'path' => '/mnt/gitlab/git-data' + }, + }) + ``` + + <!-- + updates to following example must also be made at + https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab + --> + +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace + the file of the same name on this server. If that file is not on this server, + add the file from your Consul server to this server. + +1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). + +### Gitaly TLS support + +Gitaly supports TLS encryption. To be able to communicate +with a Gitaly instance that listens for secure connections you will need to use `tls://` URL +scheme in the `gitaly_address` of the corresponding storage entry in the GitLab configuration. + +You will need to bring your own certificates as this isn't provided automatically. +The certificate, or its certificate authority, must be installed on all Gitaly +nodes (including the Gitaly node using the certificate) and on all client nodes +that communicate with it following the procedure described in +[GitLab custom certificate configuration](https://docs.gitlab.com/omnibus/settings/ssl.html#install-custom-public-certificates). + +NOTE: **Note:** +The self-signed certificate must specify the address you use to access the +Gitaly server. If you are addressing the Gitaly server by a hostname, you can +either use the Common Name field for this, or add it as a Subject Alternative +Name. If you are addressing the Gitaly server by its IP address, you must add it +as a Subject Alternative Name to the certificate. +[gRPC does not support using an IP address as Common Name in a certificate](https://github.com/grpc/grpc/issues/2691). + +NOTE: **Note:** +It is possible to configure Gitaly servers with both an +unencrypted listening address `listen_addr` and an encrypted listening +address `tls_listen_addr` at the same time. This allows you to do a +gradual transition from unencrypted to encrypted traffic, if necessary. + +To configure Gitaly with TLS: + +1. Create the `/etc/gitlab/ssl` directory and copy your key and certificate there: + + ```shell + sudo mkdir -p /etc/gitlab/ssl + sudo chmod 755 /etc/gitlab/ssl + sudo cp key.pem cert.pem /etc/gitlab/ssl/ + sudo chmod 644 key.pem cert.pem + ``` + +1. Copy the cert to `/etc/gitlab/trusted-certs` so Gitaly will trust the cert when + calling into itself: + + ```shell + sudo cp /etc/gitlab/ssl/cert.pem /etc/gitlab/trusted-certs/ + ``` + +1. Edit `/etc/gitlab/gitlab.rb` and add: + + <!-- + updates to following example must also be made at + https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab + --> + + ```ruby + gitaly['tls_listen_addr'] = "0.0.0.0:9999" + gitaly['certificate_path'] = "/etc/gitlab/ssl/cert.pem" + gitaly['key_path'] = "/etc/gitlab/ssl/key.pem" + ``` + +1. Delete `gitaly['listen_addr']` to allow only encrypted connections. + +1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure Sidekiq + +Sidekiq requires connections to the Redis, PostgreSQL and Gitaly instances. +The following IPs will be used as an example: + +- `10.6.0.101`: Sidekiq 1 +- `10.6.0.102`: Sidekiq 2 +- `10.6.0.103`: Sidekiq 3 +- `10.6.0.104`: Sidekiq 4 + +To configure the Sidekiq nodes, on each one: + +1. SSH into the Sidekiq server. +1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab package +you want using steps 1 and 2 from the GitLab downloads page. +**Do not complete any other steps on the download page.** +1. Open `/etc/gitlab/gitlab.rb` with your editor: + + ```ruby + ######################################## + ##### Services Disabled ### + ######################################## + + nginx['enable'] = false + grafana['enable'] = false + prometheus['enable'] = false + alertmanager['enable'] = false + gitaly['enable'] = false + gitlab_workhorse['enable'] = false + nginx['enable'] = false + puma['enable'] = false + postgres_exporter['enable'] = false + postgresql['enable'] = false + redis['enable'] = false + redis_exporter['enable'] = false + gitlab_exporter['enable'] = false + + ######################################## + #### Redis ### + ######################################## + + ## Redis connection details + ## First cluster that will host the cache + gitlab_rails['redis_cache_instance'] = 'redis://:<REDIS_PRIMARY_PASSWORD_OF_FIRST_CLUSTER>@gitlab-redis-cache' + + gitlab_rails['redis_cache_sentinels'] = [ + {host: '10.6.0.71', port: 26379}, + {host: '10.6.0.72', port: 26379}, + {host: '10.6.0.73', port: 26379}, + ] + + ## Second cluster that will host the queues, shared state, and actioncable + gitlab_rails['redis_queues_instance'] = 'redis://:<REDIS_PRIMARY_PASSWORD_OF_SECOND_CLUSTER>@gitlab-redis-persistent' + gitlab_rails['redis_shared_state_instance'] = 'redis://:<REDIS_PRIMARY_PASSWORD_OF_SECOND_CLUSTER>@gitlab-redis-persistent' + gitlab_rails['redis_actioncable_instance'] = 'redis://:<REDIS_PRIMARY_PASSWORD_OF_SECOND_CLUSTER>@gitlab-redis-persistent' + + gitlab_rails['redis_queues_sentinels'] = [ + {host: '10.6.0.81', port: 26379}, + {host: '10.6.0.82', port: 26379}, + {host: '10.6.0.83', port: 26379}, + ] + gitlab_rails['redis_shared_state_sentinels'] = [ + {host: '10.6.0.81', port: 26379}, + {host: '10.6.0.82', port: 26379}, + {host: '10.6.0.83', port: 26379}, + ] + gitlab_rails['redis_actioncable_sentinels'] = [ + {host: '10.6.0.81', port: 26379}, + {host: '10.6.0.82', port: 26379}, + {host: '10.6.0.83', port: 26379}, + ] + + ####################################### + ### Gitaly ### + ####################################### + + git_data_dirs({ + 'default' => { 'gitaly_address' => 'tcp://gitaly1.internal:8075' }, + 'storage1' => { 'gitaly_address' => 'tcp://gitaly1.internal:8075' }, + 'storage2' => { 'gitaly_address' => 'tcp://gitaly2.internal:8075' }, + }) + gitlab_rails['gitaly_token'] = 'YOUR_TOKEN' + + ####################################### + ### Postgres ### + ####################################### + gitlab_rails['db_host'] = '10.6.0.20' # internal load balancer IP + gitlab_rails['db_port'] = 6432 + gitlab_rails['db_password'] = '<postgresql_user_password>' + gitlab_rails['db_adapter'] = 'postgresql' + gitlab_rails['db_encoding'] = 'unicode' + gitlab_rails['auto_migrate'] = false + + ####################################### + ### Sidekiq configuration ### + ####################################### + sidekiq['listen_address'] = "0.0.0.0" + sidekiq['cluster'] = true # no need to set this after GitLab 13.0 + + ####################################### + ### Monitoring configuration ### + ####################################### + consul['enable'] = true + consul['monitoring_service_discovery'] = true + + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13) + } + + # Set the network addresses that the exporters will listen on + node_exporter['listen_address'] = '0.0.0.0:9100' + + # Rails Status for prometheus + gitlab_rails['monitoring_whitelist'] = ['10.6.0.121/32', '127.0.0.0/8'] + ``` + +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace + the file of the same name on this server. If that file is not on this server, + add the file from your Consul server to this server. + +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + +TIP: **Tip:** +You can also run [multiple Sidekiq processes](../operations/extra_sidekiq_processes.md). + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure GitLab Rails + +NOTE: **Note:** +In our architectures we run each GitLab Rails node using the Puma webserver +and have its number of workers set to 90% of available CPUs along with four threads. For +nodes that are running Rails with other components the worker value should be reduced +accordingly where we've found 50% achieves a good balance but this is dependent +on workload. + +This section describes how to configure the GitLab application (Rails) component. + +The following IPs will be used as an example: + +- `10.6.0.111`: GitLab application 1 +- `10.6.0.112`: GitLab application 2 +- `10.6.0.113`: GitLab application 3 + +On each node perform the following: + +1. Download/install Omnibus GitLab using **steps 1 and 2** from + [GitLab downloads](https://about.gitlab.com/install/). Do not complete other + steps on the download page. + +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace + the file of the same name on this server. If that file is not on this server, + add the file from your Consul server to this server. + +1. Edit `/etc/gitlab/gitlab.rb` and use the following configuration. + To maintain uniformity of links across nodes, the `external_url` + on the application server should point to the external URL that users will use + to access GitLab. This would be the URL of the [external load balancer](#configure-the-external-load-balancer) + which will route traffic to the GitLab application server: + + ```ruby + external_url 'https://gitlab.example.com' + + # Gitaly and GitLab use two shared secrets for authentication, one to authenticate gRPC requests + # to Gitaly, and a second for authentication callbacks from GitLab-Shell to the GitLab internal API. + # The following two values must be the same as their respective values + # of the Gitaly setup + gitlab_rails['gitaly_token'] = 'gitalysecret' + gitlab_shell['secret_token'] = 'shellsecret' + + git_data_dirs({ + 'default' => { 'gitaly_address' => 'tcp://gitaly1.internal:8075' }, + 'storage1' => { 'gitaly_address' => 'tcp://gitaly1.internal:8075' }, + 'storage2' => { 'gitaly_address' => 'tcp://gitaly2.internal:8075' }, + }) + + ## Disable components that will not be on the GitLab application server + roles ['application_role'] + gitaly['enable'] = false + nginx['enable'] = true + + ## PostgreSQL connection details + # Disable PostgreSQL on the application node + postgresql['enable'] = false + gitlab_rails['db_host'] = '10.6.0.20' # internal load balancer IP + gitlab_rails['db_port'] = 6432 + gitlab_rails['db_password'] = '<postgresql_user_password>' + gitlab_rails['auto_migrate'] = false + + ## Redis connection details + ## First cluster that will host the cache + gitlab_rails['redis_cache_instance'] = 'redis://:<REDIS_PRIMARY_PASSWORD_OF_FIRST_CLUSTER>@gitlab-redis-cache' + + gitlab_rails['redis_cache_sentinels'] = [ + {host: '10.6.0.71', port: 26379}, + {host: '10.6.0.72', port: 26379}, + {host: '10.6.0.73', port: 26379}, + ] + + ## Second cluster that will host the queues, shared state, and actionable + gitlab_rails['redis_queues_instance'] = 'redis://:<REDIS_PRIMARY_PASSWORD_OF_SECOND_CLUSTER>@gitlab-redis-persistent' + gitlab_rails['redis_shared_state_instance'] = 'redis://:<REDIS_PRIMARY_PASSWORD_OF_SECOND_CLUSTER>@gitlab-redis-persistent' + gitlab_rails['redis_actioncable_instance'] = 'redis://:<REDIS_PRIMARY_PASSWORD_OF_SECOND_CLUSTER>@gitlab-redis-persistent' + + gitlab_rails['redis_queues_sentinels'] = [ + {host: '10.6.0.81', port: 26379}, + {host: '10.6.0.82', port: 26379}, + {host: '10.6.0.83', port: 26379}, + ] + gitlab_rails['redis_shared_state_sentinels'] = [ + {host: '10.6.0.81', port: 26379}, + {host: '10.6.0.82', port: 26379}, + {host: '10.6.0.83', port: 26379}, + ] + gitlab_rails['redis_actioncable_sentinels'] = [ + {host: '10.6.0.81', port: 26379}, + {host: '10.6.0.82', port: 26379}, + {host: '10.6.0.83', port: 26379}, + ] + + # Set the network addresses that the exporters used for monitoring will listen on + node_exporter['listen_address'] = '0.0.0.0:9100' + gitlab_workhorse['prometheus_listen_addr'] = '0.0.0.0:9229' + sidekiq['listen_address'] = "0.0.0.0" + puma['listen'] = '0.0.0.0' + + # Add the monitoring node's IP address to the monitoring whitelist and allow it to + # scrape the NGINX metrics + gitlab_rails['monitoring_whitelist'] = ['10.6.0.121/32', '127.0.0.0/8'] + nginx['status']['options']['allow'] = ['10.6.0.121/32', '127.0.0.0/8'] + ``` + +1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). +1. If you're using [Gitaly with TLS support](#gitaly-tls-support), make sure the + `git_data_dirs` entry is configured with `tls` instead of `tcp`: + + ```ruby + git_data_dirs({ + 'default' => { 'gitaly_address' => 'tls://gitaly1.internal:9999' }, + 'storage1' => { 'gitaly_address' => 'tls://gitaly1.internal:9999' }, + 'storage2' => { 'gitaly_address' => 'tls://gitaly2.internal:9999' }, + }) + ``` + + 1. Copy the cert into `/etc/gitlab/trusted-certs`: + + ```shell + sudo cp cert.pem /etc/gitlab/trusted-certs/ + ``` + +1. If you're [using NFS](#configure-nfs-optional): + 1. If necessary, install the NFS client utility packages using the following + commands: + + ```shell + # Ubuntu/Debian + apt-get install nfs-common + + # CentOS/Red Hat + yum install nfs-utils nfs-utils-lib + ``` + + 1. Specify the necessary NFS mounts in `/etc/fstab`. + The exact contents of `/etc/fstab` will depend on how you chose + to configure your NFS server. See the [NFS documentation](../high_availability/nfs.md) + for examples and the various options. + + 1. Create the shared directories. These may be different depending on your NFS + mount locations. + + ```shell + mkdir -p /var/opt/gitlab/.ssh /var/opt/gitlab/gitlab-rails/uploads /var/opt/gitlab/gitlab-rails/shared /var/opt/gitlab/gitlab-ci/builds /var/opt/gitlab/git-data + ``` + + 1. Edit `/etc/gitlab/gitlab.rb` and use the following configuration: + + ```ruby + ## Prevent GitLab from starting if NFS data mounts are not available + high_availability['mountpoint'] = '/var/opt/gitlab/git-data' + + ## Ensure UIDs and GIDs match between servers for permissions via NFS + user['uid'] = 9000 + user['gid'] = 9000 + web_server['uid'] = 9001 + web_server['gid'] = 9001 + registry['uid'] = 9002 + registry['gid'] = 9002 + ``` + +1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). +1. Confirm the node can connect to Gitaly: + + ```shell + sudo gitlab-rake gitlab:gitaly:check + ``` + + Then, tail the logs to see the requests: + + ```shell + sudo gitlab-ctl tail gitaly + ``` + +1. Optionally, from the Gitaly servers, confirm that Gitaly can perform callbacks to the internal API: + + ```shell + sudo /opt/gitlab/embedded/service/gitlab-shell/bin/check -config /opt/gitlab/embedded/service/gitlab-shell/config.yml + ``` + +NOTE: **Note:** +When you specify `https` in the `external_url`, as in the example +above, GitLab assumes you have SSL certificates in `/etc/gitlab/ssl/`. If +certificates are not present, NGINX will fail to start. See the +[NGINX documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https) +for more information. + +### GitLab Rails post-configuration + +Initialize the GitLab database, by running the following in one of the Rails nodes: + +```shell +sudo gitlab-rake gitlab:db:configure +``` + +NOTE: **Note:** +If you encounter a `rake aborted!` error stating that PgBouncer is failing to connect to +PostgreSQL it may be that your PgBouncer node's IP address is missing from +PostgreSQL's `trust_auth_cidr_addresses` in `gitlab.rb` on your database nodes. See +[PgBouncer error `ERROR: pgbouncer cannot connect to server`](troubleshooting.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server) +in the Troubleshooting section before proceeding. + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure Prometheus + +The Omnibus GitLab package can be used to configure a standalone Monitoring node +running [Prometheus](../monitoring/prometheus/index.md) and +[Grafana](../monitoring/performance/grafana_configuration.md). + +The following IP will be used as an example: + +- `10.6.0.121`: Prometheus + +To configure the Monitoring node: + +1. SSH into the Monitoring node. +1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab + package you want using **steps 1 and 2** from the GitLab downloads page. + Do not complete any other steps on the download page. + +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace + the file of the same name on this server. If that file is not on this server, + add the file from your Consul server to this server. + +1. Edit `/etc/gitlab/gitlab.rb` and add the contents: + + ```ruby + external_url 'http://gitlab.example.com' + + # Disable all other services + gitlab_rails['auto_migrate'] = false + alertmanager['enable'] = false + gitaly['enable'] = false + gitlab_exporter['enable'] = false + gitlab_workhorse['enable'] = false + nginx['enable'] = true + postgres_exporter['enable'] = false + postgresql['enable'] = false + redis['enable'] = false + redis_exporter['enable'] = false + sidekiq['enable'] = false + puma['enable'] = false + unicorn['enable'] = false + node_exporter['enable'] = false + gitlab_exporter['enable'] = false + + # Enable Prometheus + prometheus['enable'] = true + prometheus['listen_address'] = '0.0.0.0:9090' + prometheus['monitor_kubernetes'] = false + + # Enable Login form + grafana['disable_login_form'] = false + + # Enable Grafana + grafana['enable'] = true + grafana['admin_password'] = '<grafana_password>' + + # Enable service discovery for Prometheus + consul['enable'] = true + consul['monitoring_service_discovery'] = true + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13) + } + ``` + +1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). +1. In the GitLab UI, set `admin/application_settings/metrics_and_profiling` > Metrics - Grafana to `/-/grafana` to +`http[s]://<MONITOR NODE>/-/grafana` + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure the object storage + +GitLab supports using an object storage service for holding numerous types of data. +It's recommended over [NFS](#configure-nfs-optional) and in general it's better +in larger setups as object storage is typically much more performant, reliable, +and scalable. + +Object storage options that GitLab has tested, or is aware of customers using include: + +- SaaS/Cloud solutions such as [Amazon S3](https://aws.amazon.com/s3/), [Google cloud storage](https://cloud.google.com/storage). +- On-premises hardware and appliances from various storage vendors. +- MinIO. There is [a guide to deploying this](https://docs.gitlab.com/charts/advanced/external-object-storage/minio.html) within our Helm Chart documentation. + +For configuring GitLab to use Object Storage refer to the following guides +based on what features you intend to use: + +1. Configure [object storage for backups](../../raketasks/backup_restore.md#uploading-backups-to-a-remote-cloud-storage). +1. Configure [object storage for job artifacts](../job_artifacts.md#using-object-storage) + including [incremental logging](../job_logs.md#new-incremental-logging-architecture). +1. Configure [object storage for LFS objects](../lfs/index.md#storing-lfs-objects-in-remote-object-storage). +1. Configure [object storage for uploads](../uploads.md#using-object-storage-core-only). +1. Configure [object storage for merge request diffs](../merge_request_diffs.md#using-object-storage). +1. Configure [object storage for Container Registry](../packages/container_registry.md#use-object-storage) (optional feature). +1. Configure [object storage for Mattermost](https://docs.mattermost.com/administration/config-settings.html#file-storage) (optional feature). +1. Configure [object storage for packages](../packages/index.md#using-object-storage) (optional feature). **(PREMIUM ONLY)** +1. Configure [object storage for Dependency Proxy](../packages/dependency_proxy.md#using-object-storage) (optional feature). **(PREMIUM ONLY)** +1. Configure [object storage for Pseudonymizer](../pseudonymizer.md#configuration) (optional feature). **(ULTIMATE ONLY)** +1. Configure [object storage for autoscale Runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional - for improved performance). +1. Configure [object storage for Terraform state files](../terraform_state.md#using-object-storage-core-only). + +Using separate buckets for each data type is the recommended approach for GitLab. + +A limitation of our configuration is that each use of object storage is separately configured. +[We have an issue for improving this](https://gitlab.com/gitlab-org/gitlab/-/issues/23345) +and easily using one bucket with separate folders is one improvement that this might bring. + +There is at least one specific issue with using the same bucket: +when GitLab is deployed with the Helm chart restore from backup +[will not properly function](https://docs.gitlab.com/charts/advanced/external-object-storage/#lfs-artifacts-uploads-packages-external-diffs-pseudonymizer) +unless separate buckets are used. + +One risk of using a single bucket would be if your organization decided to +migrate GitLab to the Helm deployment in the future. GitLab would run, but the situation with +backups might not be realized until the organization had a critical requirement for the backups to +work. + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure NFS (optional) + +[Object storage](#configure-the-object-storage), along with [Gitaly](#configure-gitaly) +are recommended over NFS wherever possible for improved performance. If you intend +to use GitLab Pages, this currently [requires NFS](troubleshooting.md#gitlab-pages-requires-nfs). + +See how to [configure NFS](../high_availability/nfs.md). + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Troubleshooting + +See the [troubleshooting documentation](troubleshooting.md). + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> diff --git a/doc/administration/reference_architectures/5k_users.md b/doc/administration/reference_architectures/5k_users.md index 0b4114bca6e..14685ffa53d 100644 --- a/doc/administration/reference_architectures/5k_users.md +++ b/doc/administration/reference_architectures/5k_users.md @@ -1,49 +1,54 @@ --- reading_time: true +stage: Enablement +group: Distribution +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# Reference architecture: up to 5,000 users +# Reference architecture: up to 5,000 users **(PREMIUM ONLY)** -This page describes GitLab reference architecture for up to 5,000 users. -For a full list of reference architectures, see +This page describes GitLab reference architecture for up to 5,000 users. For a +full list of reference architectures, see [Available reference architectures](index.md#available-reference-architectures). NOTE: **Note:** -The 5,000-user reference architecture documented below is -designed to help your organization achieve a highly-available GitLab deployment. -If you do not have the expertise or need to maintain a highly-available -environment, you can have a simpler and less costly-to-operate environment by -following the [2,000-user reference architecture](2k_users.md). +This reference architecture is designed to help your organization achieve a +highly-available GitLab deployment. If you do not have the expertise or need to +maintain a highly-available environment, you can have a simpler and less +costly-to-operate environment by using the +[2,000-user reference architecture](2k_users.md). > - **Supported users (approximate):** 5,000 -> - **High Availability:** True -> - **Test RPS rates:** API: 100 RPS, Web: 10 RPS, Git: 10 RPS - -| Service | Nodes | Configuration | GCP | AWS | Azure | -|--------------------------------------------------------------|-------|---------------------------------|-----------------|-----------------------|----------------| -| External load balancing node | 1 | 2 vCPU, 1.8GB Memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| Redis | 3 | 2 vCPU, 7.5GB Memory | `n1-standard-2` | `m5.large` | `D2s v3` | -| Consul + Sentinel | 3 | 2 vCPU, 1.8GB Memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| PostgreSQL | 3 | 2 vCPU, 7.5GB Memory | `n1-standard-2` | `m5.large` | `D2s v3` | -| PgBouncer | 3 | 2 vCPU, 1.8GB Memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| Internal load balancing node | 1 | 2 vCPU, 1.8GB Memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| Gitaly | 2 minimum | 8 vCPU, 30GB Memory | `n1-standard-8` | `m5.2xlarge` | `D8s v3` | -| Sidekiq | 4 | 2 vCPU, 7.5GB Memory | `n1-standard-2` | `m5.large` | `D2s v3` | -| GitLab Rails | 3 | 16 vCPU, 14.4GB Memory | `n1-highcpu-16` | `c5.4xlarge` | `F16s v2` | -| Monitoring node | 1 | 2 vCPU, 1.8GB Memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| Object Storage | n/a | n/a | n/a | n/a | n/a | -| NFS Server (optional, not recommended) | 1 | 4 vCPU, 3.6GB Memory | `n1-highcpu-4` | `c5.xlarge` | `F4s v2` | - -The architectures were built and tested with the [Intel Xeon E5 v3 (Haswell)](https://cloud.google.com/compute/docs/cpu-platforms) -CPU platform on GCP. On different hardware you may find that adjustments, either lower -or higher, are required for your CPU or Node counts accordingly. For more information, a -[Sysbench](https://github.com/akopytov/sysbench) benchmark of the CPU can be found -[here](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Reference-Architectures/GCP-CPU-Benchmarks). - -For data objects such as LFS, Uploads, Artifacts, etc, an [object storage service](#configure-the-object-storage) -is recommended over NFS where possible, due to better performance and availability. -Since this doesn't require a node to be set up, it's marked as not applicable (n/a) -in the table above. +> - **High Availability:** Yes +> - **Test requests per second (RPS) rates:** API: 100 RPS, Web: 10 RPS, Git: 10 RPS + +| Service | Nodes | Configuration | GCP | AWS | Azure | +|--------------------------------------------|-------------|-------------------------|----------------|-------------|----------| +| External load balancing node | 1 | 2 vCPU, 1.8GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| Redis | 3 | 2 vCPU, 7.5GB memory | n1-standard-2 | m5.large | D2s v3 | +| Consul + Sentinel | 3 | 2 vCPU, 1.8GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| PostgreSQL | 3 | 2 vCPU, 7.5GB memory | n1-standard-2 | m5.large | D2s v3 | +| PgBouncer | 3 | 2 vCPU, 1.8GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| Internal load balancing node | 1 | 2 vCPU, 1.8GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| Gitaly | 2 (minimum) | 8 vCPU, 30GB memory | n1-standard-8 | m5.2xlarge | D8s v3 | +| Sidekiq | 4 | 2 vCPU, 7.5GB memory | n1-standard-2 | m5.large | D2s v3 | +| GitLab Rails | 3 | 16 vCPU, 14.4GB memory | n1-highcpu-16 | c5.4xlarge | F16s v2 | +| Monitoring node | 1 | 2 vCPU, 1.8GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| Object Storage | n/a | n/a | n/a | n/a | n/a | +| NFS Server (optional, not recommended) | 1 | 4 vCPU, 3.6GB memory | n1-highcpu-4 | c5.xlarge | F4s v2 | + +The Google Cloud Platform (GCP) architectures were built and tested using the +[Intel Xeon E5 v3 (Haswell)](https://cloud.google.com/compute/docs/cpu-platforms) +CPU platform. On different hardware you may find that adjustments, either lower +or higher, are required for your CPU or node counts. For more information, see +our [Sysbench](https://github.com/akopytov/sysbench)-based +[CPU benchmark](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Reference-Architectures/GCP-CPU-Benchmarks). + +For data objects (such as LFS, Uploads, or Artifacts), an +[object storage service](#configure-the-object-storage) is recommended instead +of NFS where possible, due to better performance and availability. Since this +doesn't require a node to be set up, *Object Storage* is noted as not +applicable (n/a) in the previous table. ## Setup components @@ -1439,7 +1444,7 @@ On each node perform the following: 1. Specify the necessary NFS mounts in `/etc/fstab`. The exact contents of `/etc/fstab` will depend on how you chose - to configure your NFS server. See the [NFS documentation](../high_availability/nfs.md) + to configure your NFS server. See the [NFS documentation](../nfs.md) for examples and the various options. 1. Create the shared directories. These may be different depending on your NFS @@ -1748,7 +1753,7 @@ work. are recommended over NFS wherever possible for improved performance. If you intend to use GitLab Pages, this currently [requires NFS](troubleshooting.md#gitlab-pages-requires-nfs). -See how to [configure NFS](../high_availability/nfs.md). +See how to [configure NFS](../nfs.md). <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> diff --git a/doc/administration/reference_architectures/img/reference-architectures.png b/doc/administration/reference_architectures/img/reference-architectures.png Binary files differindex e15609e78e1..0f8e663b57b 100644 --- a/doc/administration/reference_architectures/img/reference-architectures.png +++ b/doc/administration/reference_architectures/img/reference-architectures.png diff --git a/doc/administration/reference_architectures/index.md b/doc/administration/reference_architectures/index.md index 8fde71a66bf..4f7be2413dd 100644 --- a/doc/administration/reference_architectures/index.md +++ b/doc/administration/reference_architectures/index.md @@ -1,36 +1,45 @@ --- type: reference, concepts +stage: Enablement +group: Distribution +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# Reference architectures -<!-- TBD to be reviewed by Eric --> +# Reference architectures You can set up GitLab on a single server or scale it up to serve many users. -This page details the recommended Reference Architectures that were built and verified by GitLab's Quality and Support teams. +This page details the recommended Reference Architectures that were built and +verified by GitLab's Quality and Support teams. -Below is a chart representing each architecture tier and the number of users they can handle. As your number of users grow with time, it’s recommended that you scale GitLab accordingly. +Below is a chart representing each architecture tier and the number of users +they can handle. As your number of users grow with time, it’s recommended that +you scale GitLab accordingly.  <!-- Internal link: https://docs.google.com/spreadsheets/d/1obYP4fLKkVVDOljaI3-ozhmCiPtEeMblbBKkf2OADKs/edit#gid=1403207183 --> -Testing on these reference architectures were performed with [GitLab's Performance Tool](https://gitlab.com/gitlab-org/quality/performance) -at specific coded workloads, and the throughputs used for testing were calculated based on sample customer data. -After selecting the reference architecture that matches your scale, refer to -[Configure GitLab to Scale](#configure-gitlab-to-scale) to see the components -involved, and how to configure them. +Testing on these reference architectures were performed with +[GitLab's Performance Tool](https://gitlab.com/gitlab-org/quality/performance) +at specific coded workloads, and the throughputs used for testing were +calculated based on sample customer data. Select the +[reference architecture](#available-reference-architectures) that matches your scale. -Each endpoint type is tested with the following number of requests per second (RPS) per 1000 users: +Each endpoint type is tested with the following number of requests per second (RPS) +per 1,000 users: - API: 20 RPS - Web: 2 RPS - Git: 2 RPS -For GitLab instances with less than 2,000 users, it's recommended that you use the [default setup](#automated-backups-core-only) -by [installing GitLab](../../install/README.md) on a single machine to minimize maintenance and resource costs. +For GitLab instances with less than 2,000 users, it's recommended that you use +the [default setup](#automated-backups-core-only) by +[installing GitLab](../../install/README.md) on a single machine to minimize +maintenance and resource costs. -If your organization has more than 2,000 users, the recommendation is to scale GitLab's components to multiple -machine nodes. The machine nodes are grouped by component(s). The addition of these -nodes increases the performance and scalability of to your GitLab instance. +If your organization has more than 2,000 users, the recommendation is to scale +GitLab's components to multiple machine nodes. The machine nodes are grouped by +components. The addition of these nodes increases the performance and +scalability of to your GitLab instance. When scaling GitLab, there are several factors to consider: @@ -39,12 +48,13 @@ When scaling GitLab, there are several factors to consider: - The application nodes connects to a shared file server and PostgreSQL and Redis services on the backend. NOTE: **Note:** -Depending on your workflow, the following recommended -reference architectures may need to be adapted accordingly. Your workload -is influenced by factors including how active your users are, -how much automation you use, mirroring, and repository/change size. Additionally the -displayed memory values are provided by [GCP machine types](https://cloud.google.com/compute/docs/machine-types). -For different cloud vendors, attempt to select options that best match the provided architecture. +Depending on your workflow, the following recommended reference architectures +may need to be adapted accordingly. Your workload is influenced by factors +including how active your users are, how much automation you use, mirroring, +and repository/change size. Additionally the displayed memory values are +provided by [GCP machine types](https://cloud.google.com/compute/docs/machine-types). +For different cloud vendors, attempt to select options that best match the +provided architecture. ## Available reference architectures @@ -60,14 +70,14 @@ The following reference architectures are available: ## Availability Components -GitLab comes with the following components for your use, listed from -least to most complex: +GitLab comes with the following components for your use, listed from least to +most complex: -1. [Automated backups](#automated-backups-core-only) -1. [Traffic load balancer](#traffic-load-balancer-starter-only) -1. [Zero downtime updates](#zero-downtime-updates-starter-only) -1. [Automated database failover](#automated-database-failover-premium-only) -1. [Instance level replication with GitLab Geo](#instance-level-replication-with-gitlab-geo-premium-only) +- [Automated backups](#automated-backups-core-only) +- [Traffic load balancer](#traffic-load-balancer-starter-only) +- [Zero downtime updates](#zero-downtime-updates-starter-only) +- [Automated database failover](#automated-database-failover-premium-only) +- [Instance level replication with GitLab Geo](#instance-level-replication-with-gitlab-geo-premium-only) As you implement these components, begin with a single server and then do backups. Only after completing the first server should you proceed to the next. @@ -115,7 +125,8 @@ to the default installation: > - Supported tiers: [GitLab Starter, Premium, and Ultimate](https://about.gitlab.com/pricing/) GitLab supports [zero-downtime updates](https://docs.gitlab.com/omnibus/update/#zero-downtime-updates). -Although you can perform zero-downtime updates with a single GitLab node, the recommendation is to separate GitLab into several application nodes. +Single GitLab nodes can be updated with only a [few minutes of downtime](https://docs.gitlab.com/omnibus/update/README.html#single-node-deployment). +To avoid this, we recommend to separate GitLab into several application nodes. As long as at least one of each component is online and capable of handling the instance's usage load, your team's productivity will not be interrupted during the update. ### Automated database failover **(PREMIUM ONLY)** @@ -140,40 +151,7 @@ is recommended. instance to other geographical locations as a read-only fully operational instance that can also be promoted in case of disaster. -## Configure GitLab to scale - -NOTE: **Note:** -From GitLab 13.0, using NFS for Git repositories is deprecated. In GitLab 14.0, support for NFS for Git repositories is scheduled to be removed. Upgrade to [Gitaly Cluster](../gitaly/praefect.md) as soon as possible. - -The following components are the ones you need to configure in order to scale -GitLab. They are listed in the order you'll typically configure them if they are -required by your [reference architecture](#reference-architectures) of choice. - -Most of them are bundled in the GitLab deb/rpm package (called Omnibus GitLab), -but depending on your system architecture, you may require some components which are -not included in it. If required, those should be configured before -setting up components provided by GitLab. Advice on how to select the right -solution for your organization is provided in the configuration instructions -column. - -| Component | Description | Configuration instructions | Bundled with Omnibus GitLab | -|-----------|-------------|----------------------------| -| Load balancer(s) ([6](#footnotes)) | Handles load balancing, typically when you have multiple GitLab application services nodes | [Load balancer configuration](../high_availability/load_balancer.md) ([6](#footnotes)) | No | -| Object storage service ([4](#footnotes)) | Recommended store for shared data objects | [Object Storage configuration](../object_storage.md) | No | -| NFS ([5](#footnotes)) ([7](#footnotes)) | Shared disk storage service. Can be used as an alternative Object Storage. Required for GitLab Pages | [NFS configuration](../high_availability/nfs.md) | No | -| [Consul](../../development/architecture.md#consul) ([3](#footnotes)) | Service discovery and health checks/failover | [Consul configuration](../high_availability/consul.md) **(PREMIUM ONLY)** | Yes | -| [PostgreSQL](../../development/architecture.md#postgresql) | Database | [PostgreSQL configuration](https://docs.gitlab.com/omnibus/settings/database.html) | Yes | -| [PgBouncer](../../development/architecture.md#pgbouncer) | Database connection pooler | [PgBouncer configuration](../high_availability/pgbouncer.md#running-pgbouncer-as-part-of-a-non-ha-gitlab-installation) **(PREMIUM ONLY)** | Yes | -| Repmgr | PostgreSQL cluster management and failover | [PostgreSQL and Repmgr configuration](../postgresql/replication_and_failover.md) | Yes | -| Patroni | An alternative PostgreSQL cluster management and failover | [PostgreSQL and Patroni configuration](../postgresql/replication_and_failover.md#patroni) | Yes | -| [Redis](../../development/architecture.md#redis) ([3](#footnotes)) | Key/value store for fast data lookup and caching | [Redis configuration](../high_availability/redis.md) | Yes | -| Redis Sentinel | Redis | [Redis Sentinel configuration](../high_availability/redis.md) | Yes | -| [Gitaly](../../development/architecture.md#gitaly) ([2](#footnotes)) ([7](#footnotes)) | Provides access to Git repositories | [Gitaly configuration](../gitaly/index.md#run-gitaly-on-its-own-server) | Yes | -| [Sidekiq](../../development/architecture.md#sidekiq) | Asynchronous/background jobs | [Sidekiq configuration](../high_availability/sidekiq.md) | Yes | -| [GitLab application services](../../development/architecture.md#unicorn)([1](#footnotes)) | Puma/Unicorn, Workhorse, GitLab Shell - serves front-end requests (UI, API, Git over HTTP/SSH) | [GitLab app scaling configuration](../high_availability/gitlab.md) | Yes | -| [Prometheus](../../development/architecture.md#prometheus) and [Grafana](../../development/architecture.md#grafana) | GitLab environment monitoring | [Monitoring node for scaling](../high_availability/monitoring_node.md) | Yes | - -### Configuring select components with Cloud Native Helm +## Configuring select components with Cloud Native Helm We also provide [Helm charts](https://docs.gitlab.com/charts/) as a Cloud Native installation method for GitLab. For the reference architectures, select components can be set up in this @@ -191,50 +169,3 @@ specs, only translated into Kubernetes resources. For example, if you were to set up a 50k installation with the Rails nodes being run in Helm, then the same amount of resources as given for Omnibus should be given to the Kubernetes cluster with the Rails nodes broken down into a number of smaller Pods across that cluster. - -## Footnotes - -1. In our architectures we run each GitLab Rails node using the Puma webserver - and have its number of workers set to 90% of available CPUs along with four threads. For - nodes that are running Rails with other components the worker value should be reduced - accordingly where we've found 50% achieves a good balance but this is dependent - on workload. - -1. Gitaly node requirements are dependent on customer data, specifically the number of - projects and their sizes. We recommend that each Gitaly node should store no more than 5TB of data - and have the number of [`gitaly-ruby` workers](../gitaly/index.md#gitaly-ruby) - set to 20% of available CPUs. Additional nodes should be considered in conjunction - with a review of expected data size and spread based on the recommendations above. - -1. Recommended Redis setup differs depending on the size of the architecture. - For smaller architectures (less than 3,000 users) a single instance should suffice. - For medium sized installs (3,000 - 5,000) we suggest one Redis cluster for all - classes and that Redis Sentinel is hosted alongside Consul. - For larger architectures (10,000 users or more) we suggest running a separate - [Redis Cluster](../redis/replication_and_failover.md#running-multiple-redis-clusters) for the Cache class - and another for the Queues and Shared State classes respectively. We also recommend - that you run the Redis Sentinel clusters separately for each Redis Cluster. - -1. For data objects such as LFS, Uploads, Artifacts, etc. We recommend an [Object Storage service](../object_storage.md) - over NFS where possible, due to better performance. - -1. NFS can be used as an alternative for object storage but this isn't typically - recommended for performance reasons. Note however it is required for [GitLab - Pages](https://gitlab.com/gitlab-org/gitlab-pages/-/issues/196). - -1. Our architectures have been tested and validated with [HAProxy](https://www.haproxy.org/) - as the load balancer. Although other load balancers with similar feature sets - could also be used, those load balancers have not been validated. - -1. We strongly recommend that any Gitaly or NFS nodes be set up with SSD disks over - HDD with a throughput of at least 8,000 IOPS for read operations and 2,000 IOPS for write - as these components have heavy I/O. These IOPS values are recommended only as a starter - as with time they may be adjusted higher or lower depending on the scale of your - environment's workload. If you're running the environment on a Cloud provider - you may need to refer to their documentation on how configure IOPS correctly. - -1. The architectures were built and tested with the [Intel Xeon E5 v3 (Haswell)](https://cloud.google.com/compute/docs/cpu-platforms) - CPU platform on GCP. On different hardware you may find that adjustments, either lower - or higher, are required for your CPU or Node counts accordingly. For more information, a - [Sysbench](https://github.com/akopytov/sysbench) benchmark of the CPU can be found - [here](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Reference-Architectures/GCP-CPU-Benchmarks). diff --git a/doc/administration/reference_architectures/troubleshooting.md b/doc/administration/reference_architectures/troubleshooting.md index 35bdb65c810..db2e9b89ba2 100644 --- a/doc/administration/reference_architectures/troubleshooting.md +++ b/doc/administration/reference_architectures/troubleshooting.md @@ -1,3 +1,9 @@ +--- +stage: Enablement +group: Distribution +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Troubleshooting a reference architecture setup This page serves as the troubleshooting documentation if you followed one of @@ -17,7 +23,7 @@ with the Fog library that GitLab uses. Symptoms include: ### GitLab Pages requires NFS If you intend to use [GitLab Pages](../../user/project/pages/index.md), this currently requires -[NFS](../high_availability/nfs.md). There is [work in progress](https://gitlab.com/gitlab-org/gitlab-pages/-/issues/196) +[NFS](../nfs.md). There is [work in progress](https://gitlab.com/gitlab-org/gitlab-pages/-/issues/196) to remove this dependency. In the future, GitLab Pages may use [object storage](https://gitlab.com/gitlab-org/gitlab/-/issues/208135). @@ -524,7 +530,7 @@ To restart either service, run `gitlab-ctl restart SERVICE` For PostgreSQL, it is usually safe to restart the master node by default. Automatic failover defaults to a 1 minute timeout. Provided the database returns before then, nothing else needs to be done. To be safe, you can stop `repmgrd` on the standby nodes first with `gitlab-ctl stop repmgrd`, then start afterwards with `gitlab-ctl start repmgrd`. -On the Consul server nodes, it is important to restart the Consul service in a controlled fashion. Read our [Consul documentation](../high_availability/consul.md#restarting-the-server-cluster) for instructions on how to restart the service. +On the Consul server nodes, it is important to restart the Consul service in a controlled fashion. Read our [Consul documentation](../consul.md#restart-consul) for instructions on how to restart the service. ### `gitlab-ctl repmgr-check-master` command produces errors diff --git a/doc/administration/reply_by_email.md b/doc/administration/reply_by_email.md index 6d7069dd461..62645ad17a1 100644 --- a/doc/administration/reply_by_email.md +++ b/doc/administration/reply_by_email.md @@ -1,5 +1,13 @@ +--- +stage: Plan +group: Certify +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Reply by email +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/1173) in GitLab 8.0. + GitLab can be set up to allow users to comment on issues and merge requests by replying to notification emails. diff --git a/doc/administration/reply_by_email_postfix_setup.md b/doc/administration/reply_by_email_postfix_setup.md index 86854a2a7b6..f950134889d 100644 --- a/doc/administration/reply_by_email_postfix_setup.md +++ b/doc/administration/reply_by_email_postfix_setup.md @@ -306,7 +306,7 @@ Courier, which we will install later to add IMAP authentication, requires mailbo ```shell Trying 123.123.123.123... - Connected to mail.example.gitlab.com. + Connected to mail.gitlab.example.com. Escape character is '^]'. - OK [CAPABILITY IMAP4rev1 UIDPLUS CHILDREN NAMESPACE THREAD=ORDEREDSUBJECT THREAD=REFERENCES SORT QUOTA IDLE ACL ACL2=UNION] Courier-IMAP ready. Copyright 1998-2011 Double Precision, Inc. See COPYING for distribution information. ``` diff --git a/doc/administration/repository_checks.md b/doc/administration/repository_checks.md index 6d9ab723d2f..7d79840b56d 100644 --- a/doc/administration/repository_checks.md +++ b/doc/administration/repository_checks.md @@ -1,3 +1,10 @@ +--- +stage: Create +group: Editor +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference +--- + # Repository checks > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/3232) in GitLab 8.7. diff --git a/doc/administration/repository_storage_paths.md b/doc/administration/repository_storage_paths.md index 5520eeace0a..16e17458e10 100644 --- a/doc/administration/repository_storage_paths.md +++ b/doc/administration/repository_storage_paths.md @@ -68,7 +68,7 @@ the example below, we add two more mount points that are named `nfs_1` and `nfs_ respectively. NOTE: **Note:** -This example uses NFS. We do not recommend using EFS for storage as it may impact GitLab's performance. See the [relevant documentation](high_availability/nfs.md#avoid-using-awss-elastic-file-system-efs) for more details. +This example uses NFS. We do not recommend using EFS for storage as it may impact GitLab's performance. See the [relevant documentation](nfs.md#avoid-using-awss-elastic-file-system-efs) for more details. **For installations from source** @@ -89,7 +89,7 @@ This example uses NFS. We do not recommend using EFS for storage as it may impac 1. [Restart GitLab](restart_gitlab.md#installations-from-source) for the changes to take effect. ->**Note:** +NOTE: **Note:** The [`gitlab_shell: repos_path` entry](https://gitlab.com/gitlab-org/gitlab-foss/-/blob/8-9-stable/config/gitlab.yml.example#L457) in `gitlab.yml` will be deprecated and replaced by `repositories: storages` in the future, so if you are upgrading from a version prior to 8.10, make sure to add the configuration diff --git a/doc/administration/server_hooks.md b/doc/administration/server_hooks.md index 2fea000b799..ab808fc28d8 100644 --- a/doc/administration/server_hooks.md +++ b/doc/administration/server_hooks.md @@ -163,13 +163,13 @@ them as they can change. The following server hooks have been re-implemented in Go: - `pre-receive`, with the Go implementation used by default. To use the Ruby implementation instead, - [disable](../operations/feature_flags.md#enable-or-disable-feature-flag-strategies) the - `:gitaly_go_preceive_hook` feature flag. + [disable](feature_flags.md#enable-or-disable-the-feature) the `:gitaly_go_preceive_hook` feature + flag. - `update`, with the Go implementation used by default. To use the Ruby implementation instead, - [disable](../operations/feature_flags.md#enable-or-disable-feature-flag-strategies) the - `:gitaly_go_update_hook` feature flag. + [disable](feature_flags.md#enable-or-disable-the-feature) the `:gitaly_go_update_hook` feature + flag. - `post-receive`, however the Ruby implementation is used by default. To use the Go implementation - instead, [enable](../operations/feature_flags.md#enable-or-disable-feature-flag-strategies) the + instead, [enable](feature_flags.md#enable-or-disable-the-feature) the `:gitaly_go_postreceive_hook` feature flag. ## Custom error messages diff --git a/doc/administration/sidekiq.md b/doc/administration/sidekiq.md new file mode 100644 index 00000000000..23e870dbb82 --- /dev/null +++ b/doc/administration/sidekiq.md @@ -0,0 +1,188 @@ +--- +type: reference +--- + +# Configuring Sidekiq + +This section discusses how to configure an external Sidekiq instance using the +bundled Sidekiq in the GitLab package. + +Sidekiq requires connection to the Redis, PostgreSQL and Gitaly instance. +To configure the Sidekiq node: + +1. SSH into the Sidekiq server. +1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab package +you want using steps 1 and 2 from the GitLab downloads page. +**Do not complete any other steps on the download page.** +1. Open `/etc/gitlab/gitlab.rb` with your editor. +1. Generate the Sidekiq configuration: + + ```ruby + sidekiq['listen_address'] = "10.10.1.48" + + ## Optional: Enable extra Sidekiq processes + sidekiq_cluster['enable'] = true + sidekiq_cluster['enable'] = true + "elastic_indexer" + ] + ``` + +1. Setup Sidekiq's connection to Redis: + + ```ruby + ## Must be the same in every sentinel node + redis['master_name'] = 'gitlab-redis' + + ## The same password for Redis authentication you set up for the master node. + redis['master_password'] = 'YOUR_PASSOWORD' + + ## A list of sentinels with `host` and `port` + gitlab_rails['redis_sentinels'] = [ + {'host' => '10.10.1.34', 'port' => 26379}, + {'host' => '10.10.1.35', 'port' => 26379}, + {'host' => '10.10.1.36', 'port' => 26379}, + ] + ``` + +1. Set up Sidekiq's connection to Gitaly: + + ```ruby + git_data_dirs({ + 'default' => { 'gitaly_address' => 'tcp://gitaly:8075' }, + }) + gitlab_rails['gitaly_token'] = 'YOUR_TOKEN' + ``` + +1. Set up Sidekiq's connection to PostgreSQL: + + ```ruby + gitlab_rails['db_host'] = '10.10.1.30' + gitlab_rails['db_password'] = 'YOUR_PASSOWORD' + gitlab_rails['db_port'] = '5432' + gitlab_rails['db_adapter'] = 'postgresql' + gitlab_rails['db_encoding'] = 'unicode' + gitlab_rails['auto_migrate'] = false + ``` + + Remember to add the Sidekiq nodes to PostgreSQL's trusted addresses: + + ```ruby + postgresql['trust_auth_cidr_addresses'] = %w(127.0.0.1/32 10.10.1.30/32 10.10.1.31/32 10.10.1.32/32 10.10.1.33/32 10.10.1.38/32) + ``` + +1. Disable other services: + + ```ruby + nginx['enable'] = false + grafana['enable'] = false + prometheus['enable'] = false + gitlab_rails['auto_migrate'] = false + alertmanager['enable'] = false + gitaly['enable'] = false + gitlab_monitor['enable'] = false + gitlab_workhorse['enable'] = false + nginx['enable'] = false + postgres_exporter['enable'] = false + postgresql['enable'] = false + redis['enable'] = false + redis_exporter['enable'] = false + puma['enable'] = false + gitlab_exporter['enable'] = false + ``` + +1. Run `gitlab-ctl reconfigure`. + +NOTE: **Note:** +You will need to restart the Sidekiq nodes after an update has occurred and database +migrations performed. + +## Example configuration + +Here's what the ending `/etc/gitlab/gitlab.rb` would look like: + +```ruby +######################################## +##### Services Disabled ### +######################################## + +nginx['enable'] = false +grafana['enable'] = false +prometheus['enable'] = false +gitlab_rails['auto_migrate'] = false +alertmanager['enable'] = false +gitaly['enable'] = false +gitlab_monitor['enable'] = false +gitlab_workhorse['enable'] = false +nginx['enable'] = false +postgres_exporter['enable'] = false +postgresql['enable'] = false +redis['enable'] = false +redis_exporter['enable'] = false +puma['enable'] = false +gitlab_exporter['enable'] = false + +######################################## +#### Redis ### +######################################## + +## Must be the same in every sentinel node +redis['master_name'] = 'gitlab-redis' + +## The same password for Redis authentication you set up for the master node. +redis['master_password'] = 'YOUR_PASSOWORD' + +## A list of sentinels with `host` and `port` +gitlab_rails['redis_sentinels'] = [ + {'host' => '10.10.1.34', 'port' => 26379}, + {'host' => '10.10.1.35', 'port' => 26379}, + {'host' => '10.10.1.36', 'port' => 26379}, + ] + +####################################### +### Gitaly ### +####################################### + +git_data_dirs({ + 'default' => { 'gitaly_address' => 'tcp://gitaly:8075' }, +}) +gitlab_rails['gitaly_token'] = 'YOUR_TOKEN' + +####################################### +### Postgres ### +####################################### +gitlab_rails['db_host'] = '10.10.1.30' +gitlab_rails['db_password'] = 'YOUR_PASSOWORD' +gitlab_rails['db_port'] = '5432' +gitlab_rails['db_adapter'] = 'postgresql' +gitlab_rails['db_encoding'] = 'unicode' +gitlab_rails['auto_migrate'] = false + +####################################### +### Sidekiq configuration ### +####################################### +sidekiq['listen_address'] = "10.10.1.48" + +####################################### +### Monitoring configuration ### +####################################### +consul['enable'] = true +consul['monitoring_service_discovery'] = true + +consul['configuration'] = { + bind_addr: '10.10.1.48', + retry_join: %w(10.10.1.34 10.10.1.35 10.10.1.36) +} + +# Set the network addresses that the exporters will listen on +node_exporter['listen_address'] = '10.10.1.48:9100' + +# Rails Status for prometheus +gitlab_rails['monitoring_whitelist'] = ['10.10.1.42', '127.0.0.1'] +``` + +## Further reading + +Related Sidekiq configuration: + +1. [Extra Sidekiq processes](operations/extra_sidekiq_processes.md) +1. [Using the GitLab-Sidekiq chart](https://docs.gitlab.com/charts/charts/gitlab/sidekiq/) diff --git a/doc/administration/smime_signing_email.md b/doc/administration/smime_signing_email.md index 304b65917c1..b1e7e349978 100644 --- a/doc/administration/smime_signing_email.md +++ b/doc/administration/smime_signing_email.md @@ -3,7 +3,7 @@ Notification emails sent by GitLab can be signed with S/MIME for improved security. -> **Note:** +NOTE: **Note:** Please be aware that S/MIME certificates and TLS/SSL certificates are not the same and are used for different purposes: TLS creates a secure channel, whereas S/MIME signs and/or encrypts the message itself diff --git a/doc/administration/snippets/index.md b/doc/administration/snippets/index.md index cf3d8bec1a6..95de3b8c183 100644 --- a/doc/administration/snippets/index.md +++ b/doc/administration/snippets/index.md @@ -1,5 +1,8 @@ --- type: reference, howto +stage: Create +group: Editor +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" --- # Snippets settings **(CORE ONLY)** @@ -10,15 +13,15 @@ Adjust the snippets' settings of your GitLab instance. > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/31133) in GitLab 12.6. -You can set a content size max limit in snippets. This limit can prevent -abuses of the feature. The default content size limit is **52428800 Bytes** (50MB). +You can set a maximum content size limit for snippets. This limit can prevent +abuse of the feature. The default value is **52428800 Bytes** (50 MB). ### How does it work? -The content size limit will be applied when a snippet is created or -updated. Nevertheless, in order not to break any existing snippet, -the limit will only be enforced in stored snippets when the content -is updated. +The content size limit will be applied when a snippet is created or updated. + +In order not to break any existing snippets, the limit doesn't have any +effect on them until a snippet is edited again and the content changes. ### Snippets size limit configuration @@ -27,7 +30,7 @@ In order to configure this setting, use either the Rails console or the [Application settings API](../../api/settings.md). NOTE: **IMPORTANT:** -The value of the limit **must** be in Bytes. +The value of the limit **must** be in bytes. #### Through the Rails console diff --git a/doc/administration/static_objects_external_storage.md b/doc/administration/static_objects_external_storage.md index 973f4304115..34c7c8947fc 100644 --- a/doc/administration/static_objects_external_storage.md +++ b/doc/administration/static_objects_external_storage.md @@ -1,3 +1,10 @@ +--- +stage: Create +group: Editor +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference +--- + # Static objects external storage > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/31025) in GitLab 12.3. @@ -52,10 +59,10 @@ sequenceDiagram ## Set up external storage -While this procedure uses [CloudFlare Workers](https://workers.cloudflare.com) for external storage, +While this procedure uses [Cloudflare Workers](https://workers.cloudflare.com) for external storage, other CDNs or Function as a Service (FaaS) systems should work using the same principles. -1. Choose a CloudFlare Worker domain if you haven't done so already. +1. Choose a Cloudflare Worker domain if you haven't done so already. 1. In the following script, set the following values for the first two constants: - `ORIGIN_HOSTNAME`: the hostname of your GitLab installation. diff --git a/doc/administration/terraform_state.md b/doc/administration/terraform_state.md index 8d3ddc4e306..76e54acce16 100644 --- a/doc/administration/terraform_state.md +++ b/doc/administration/terraform_state.md @@ -26,7 +26,6 @@ below. `/etc/gitlab/gitlab.rb` and add the following line: ```ruby - gitlab_rails['terraform_state_enabled'] = true gitlab_rails['terraform_state_storage_path'] = "/mnt/storage/terraform_state" ``` @@ -76,7 +75,6 @@ See [the available connection settings for different providers](object_storage.m the values you want: ```ruby - gitlab_rails['terraform_state_enabled'] = true gitlab_rails['terraform_state_object_store_enabled'] = true gitlab_rails['terraform_state_object_store_remote_directory'] = "terraform" gitlab_rails['terraform_state_object_store_connection'] = { diff --git a/doc/administration/troubleshooting/debug.md b/doc/administration/troubleshooting/debug.md index 55fd6462bc3..5daf34c1011 100644 --- a/doc/administration/troubleshooting/debug.md +++ b/doc/administration/troubleshooting/debug.md @@ -7,6 +7,10 @@ in production. Troubleshooting and debugging your GitLab instance often requires a [Rails console](https://guides.rubyonrails.org/command_line.html#rails-console). +See also: + +- [GitLab Rails Console Cheat Sheet](gitlab_rails_cheat_sheet.md). +- [Navigating GitLab via Rails console](navigating_gitlab_via_rails_console.md). **For Omnibus installations** @@ -73,7 +77,7 @@ sudo gitlab-rails runner "RAILS_COMMAND" # Example with a two-line Ruby script sudo gitlab-rails runner "user = User.first; puts user.username" -# Example with a ruby script file +# Example with a ruby script file (make sure to use the full path) sudo gitlab-rails runner /path/to/script.rb ``` @@ -85,7 +89,7 @@ sudo -u git -H bundle exec rails runner -e production "RAILS_COMMAND" # Example with a two-line Ruby script sudo -u git -H bundle exec rails runner -e production "user = User.first; puts user.username" -# Example with a ruby script file +# Example with a ruby script file (make sure to use the full path) sudo -u git -H bundle exec rails runner -e production /path/to/script.rb ``` diff --git a/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md b/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md index 8e9749fb239..22d699b424b 100644 --- a/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md +++ b/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md @@ -167,7 +167,7 @@ user = User.find_by_username('root') # Find the project, update the xxx-changeme values from above project = Project.find_by_full_path('group-changeme/project-changeme') -# Delete the project +# Immediately delete the project ::Projects::DestroyService.new(project, user, {}).execute ``` @@ -248,7 +248,8 @@ end A Projects Wiki can be recreated by -**Note:** This is a destructive operation, the Wiki will be empty +NOTE: **Note:** +This is a destructive operation, the Wiki will be empty. ```ruby p = Project.find_by_full_path('<username-or-group>/<project-name>') ### enter your projects path @@ -689,10 +690,10 @@ end As a GitLab administrator, you may need to reduce disk space consumption. A common culprit is Docker Registry images that are no longer in use. To find the storage broken down by each project, run the following in the -GitLab Rails console: +[GitLab Rails console](../troubleshooting/navigating_gitlab_via_rails_console.md): ```ruby -projects_and_size = [] +projects_and_size = [["project_id", "creator_id", "registry_size_bytes", "project path"]] # You need to specify the projects that you want to look through. You can get these in any manner. projects = Project.last(100) @@ -707,17 +708,21 @@ projects.each do |p| end if project_total_size > 0 - projects_and_size << [p.full_path,project_total_size] + projects_and_size << [p.project_id, p.creator.id, project_total_size, p.full_path] end end # projects_and_size is filled out now # maybe print it as comma separated output? projects_and_size.each do |ps| - puts "%s,%s" % ps + puts "%s,%s,%s,%s" % ps end ``` +### Run the Cleanup policy now + +Find this content in the [Container Registry troubleshooting docs](../packages/container_registry.md#run-the-cleanup-policy-now). + ## Sidekiq This content has been moved to the [Troubleshooting Sidekiq docs](./sidekiq.md). @@ -831,19 +836,19 @@ Geo::JobArtifactRegistry.synced.missing_on_primary.pluck(:artifact_id) #### Get the number of verification failed repositories ```ruby -Geo::ProjectRegistryFinder.new.count_verification_failed_repositories +Geo::ProjectRegistry.verification_failed('repository').count ``` #### Find the verification failed repositories ```ruby -Geo::ProjectRegistry.verification_failed_repos +Geo::ProjectRegistry.verification_failed('repository') ``` ### Find repositories that failed to sync ```ruby -Geo::ProjectRegistryFinder.new.find_failed_project_registries('repository') +Geo::ProjectRegistry.sync_failed('repository') ``` ### Resync repositories diff --git a/doc/administration/troubleshooting/group_saml_scim.md b/doc/administration/troubleshooting/group_saml_scim.md index e2ce72d5a16..7492688ded0 100644 --- a/doc/administration/troubleshooting/group_saml_scim.md +++ b/doc/administration/troubleshooting/group_saml_scim.md @@ -2,7 +2,7 @@ type: reference --- -# Group SAML and SCIM troubleshooting **(SILVER ONLY)** +# Troubleshooting Group SAML and SCIM **(SILVER ONLY)** These are notes and screenshots regarding Group SAML and SCIM that the GitLab Support Team sometimes uses while troubleshooting, but which do not fit into the official documentation. GitLab is making this public, so that anyone can make use of the Support team’s collected knowledge. diff --git a/doc/administration/troubleshooting/img/network_monitor_xid.png b/doc/administration/troubleshooting/img/network_monitor_xid.png Binary files differnew file mode 100644 index 00000000000..5392f77327f --- /dev/null +++ b/doc/administration/troubleshooting/img/network_monitor_xid.png diff --git a/doc/administration/troubleshooting/postgresql.md b/doc/administration/troubleshooting/postgresql.md index 36c1f20818a..b7e33e4501d 100644 --- a/doc/administration/troubleshooting/postgresql.md +++ b/doc/administration/troubleshooting/postgresql.md @@ -57,7 +57,7 @@ This section is for links to information elsewhere in the GitLab documentation. - [GitLab database requirements](../../install/requirements.md#database) including - Support for MySQL was removed in GitLab 12.1; [migrate to PostgreSQL](../../update/mysql_to_postgresql.md) - required extension `pg_trgm` - - required extension `postgres_fdw` for Geo + - required extension `btree_gist` - Errors like this in the `production/sidekiq` log; see: [Set default_transaction_isolation into read committed](https://docs.gitlab.com/omnibus/settings/database.html#set-default_transaction_isolation-into-read-committed): @@ -84,7 +84,7 @@ This section is for links to information elsewhere in the GitLab documentation. PANIC: could not write to file ‘pg_xlog/xlogtemp.123’: No space left on device ``` -- [Checking Geo configuration](../geo/replication/troubleshooting.md#checking-configuration) including +- [Checking Geo configuration](../geo/replication/troubleshooting.md) including - reconfiguring hosts/ports - checking and fixing user/password mappings diff --git a/doc/administration/troubleshooting/sidekiq.md b/doc/administration/troubleshooting/sidekiq.md index 566e2686735..9125ddf545f 100644 --- a/doc/administration/troubleshooting/sidekiq.md +++ b/doc/administration/troubleshooting/sidekiq.md @@ -7,16 +7,18 @@ may be filling up. Users will notice when this happens because new branches may not show up and merge requests may not be updated. The following are some troubleshooting steps that will help you diagnose the bottleneck. -> **Note:** GitLab administrators/users should consider working through these -> debug steps with GitLab Support so the backtraces can be analyzed by our team. -> It may reveal a bug or necessary improvement in GitLab. -> -> **Note:** In any of the backtraces, be wary of suspecting cases where every -> thread appears to be waiting in the database, Redis, or waiting to acquire -> a mutex. This **may** mean there's contention in the database, for example, -> but look for one thread that is different than the rest. This other thread -> may be using all available CPU, or have a Ruby Global Interpreter Lock, -> preventing other threads from continuing. +NOTE **Note:** +GitLab administrators/users should consider working through these +debug steps with GitLab Support so the backtraces can be analyzed by our team. +It may reveal a bug or necessary improvement in GitLab. + +NOTE: **Note:** +In any of the backtraces, be wary of suspecting cases where every +thread appears to be waiting in the database, Redis, or waiting to acquire +a mutex. This **may** mean there's contention in the database, for example, +but look for one thread that is different than the rest. This other thread +may be using all available CPU, or have a Ruby Global Interpreter Lock, +preventing other threads from continuing. ## Log arguments to Sidekiq jobs diff --git a/doc/administration/troubleshooting/tracing_correlation_id.md b/doc/administration/troubleshooting/tracing_correlation_id.md new file mode 100644 index 00000000000..31f537beae5 --- /dev/null +++ b/doc/administration/troubleshooting/tracing_correlation_id.md @@ -0,0 +1,126 @@ +--- +type: reference +--- + +# Finding relevant log entries with a correlation ID + +Since GitLab 11.6, a unique request tracking ID, known as the "correlation ID" has been +logged by the GitLab instance for most requests. Each individual request to GitLab gets +its own correlation ID, which then gets logged in each GitLab component's logs for that +request. This makes it easier to trace behavior in a +distributed system. Without this ID it can be difficult or +impossible to match correlating log entries. + +## Identify the correlation ID for a request + +The correlation ID is logged in structured logs under the key `correlation_id` +and in all response headers GitLab sends under the header `x-request-id`. +You can find your correlation ID by searching in either place. + +### Getting the correlation ID in your browser + +You can use your browser's developer tools to monitor and inspect network +activity with the site that you're visiting. See the links below for network monitoring +documenation for some popular browsers. + +- [Network Monitor - Firefox Developer Tools](https://developer.mozilla.org/en-US/docs/Tools/Network_Monitor) +- [Inspect Network Activity In Chrome DevTools](https://developers.google.com/web/tools/chrome-devtools/network/) +- [Safari Web Development Tools](https://developer.apple.com/safari/tools/) +- [Microsoft Edge Network panel](https://docs.microsoft.com/en-us/microsoft-edge/devtools-guide/network#request-details) + +To locate a relevant request and view its correlation ID: + +1. Enable persistent logging in your network monitor. Some actions in GitLab will redirect you quickly after you submit a form, so this will help capture all relevant activity. +1. To help isolate the requests you are looking for, you can filter for `document` requests. +1. Click the request of interest to view further detail. +1. Go to the **Headers** section and look for **Response Headers**. There you should find an `x-request-id` header with a +value that was randomly generated by GitLab for the request. + +See the following example: + + + +### Getting the correlation ID from your logs + +Another approach to finding the correct correlation ID is to search or watch +your logs and find the `correlation_id` value for the log entry that you're +watching for. + +For example, let's say that you want learn what's happening or breaking when +you reproduce an action in GitLab. You could tail the GitLab logs, filtering +to requests by your user, and then watch the requests until you see what you're +interested in. + +### Getting the correlation ID from curl + +If you're using `curl` then you can use the verbose option to show request and response headers, as well as other debug info. + +```shell +➜ ~ curl --verbose https://gitlab.example.com/api/v4/projects +# look for a line that looks like this +< x-request-id: 4rAMkV3gof4 +``` + +#### Using jq + +This example uses [jq](https://stedolan.github.io/jq/) to filter results and +display values we most likely care about. + +```shell +sudo gitlab-ctl tail gitlab-rails/production_json.log | jq 'select(.username == "bob") | "User: \(.username), \(.method) \(.path), \(.controller)#\(.action), ID: \(.correlation_id)"' +``` + +```plaintext +"User: bob, GET /root/linux, ProjectsController#show, ID: U7k7fh6NpW3" +"User: bob, GET /root/linux/commits/master/signatures, Projects::CommitsController#signatures, ID: XPIHpctzEg1" +"User: bob, GET /root/linux/blob/master/README, Projects::BlobController#show, ID: LOt9hgi1TV4" +``` + +#### Using grep + +This example uses only `grep` and `tr`, which are more likely to be installed than `jq`. + +```shell +sudo gitlab-ctl tail gitlab-rails/production_json.log | grep '"username":"bob"' | tr ',' '\n' | egrep 'method|path|correlation_id' +``` + +```plaintext +{"method":"GET" +"path":"/root/linux" +"username":"bob" +"correlation_id":"U7k7fh6NpW3"} +{"method":"GET" +"path":"/root/linux/commits/master/signatures" +"username":"bob" +"correlation_id":"XPIHpctzEg1"} +{"method":"GET" +"path":"/root/linux/blob/master/README" +"username":"bob" +"correlation_id":"LOt9hgi1TV4"} +``` + +## Searching your logs for the correlation ID + +Once you have the correlation ID you can start searching for relevant log +entries. You can filter the lines by the correlation ID itself. +Combining a `find` and `grep` should be sufficient to find the entries you are looking for. + +```shell +# find <gitlab log directory> -type f -mtime -0 exec grep '<correlation ID>' '{}' '+' +find /var/log/gitlab -type f -mtime 0 -exec grep 'LOt9hgi1TV4' '{}' '+' +``` + +```plaintext +/var/log/gitlab/gitlab-workhorse/current:{"correlation_id":"LOt9hgi1TV4","duration_ms":2478,"host":"gitlab.domain.tld","level":"info","method":"GET","msg":"access","proto":"HTTP/1.1","referrer":"https://gitlab.domain.tld/root/linux","remote_addr":"68.0.116.160:0","remote_ip":"[filtered]","status":200,"system":"http","time":"2019-09-17T22:17:19Z","uri":"/root/linux/blob/master/README?format=json\u0026viewer=rich","user_agent":"Mozilla/5.0 (Mac) Gecko Firefox/69.0","written_bytes":1743} +/var/log/gitlab/gitaly/current:{"correlation_id":"LOt9hgi1TV4","grpc.code":"OK","grpc.meta.auth_version":"v2","grpc.meta.client_name":"gitlab-web","grpc.method":"FindCommits","grpc.request.deadline":"2019-09-17T22:17:47Z","grpc.request.fullMethod":"/gitaly.CommitService/FindCommits","grpc.request.glProjectPath":"root/linux","grpc.request.glRepository":"project-1","grpc.request.repoPath":"@hashed/6b/86/6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b.git","grpc.request.repoStorage":"default","grpc.request.topLevelGroup":"@hashed","grpc.service":"gitaly.CommitService","grpc.start_time":"2019-09-17T22:17:17Z","grpc.time_ms":2319.161,"level":"info","msg":"finished streaming call with code OK","peer.address":"@","span.kind":"server","system":"grpc","time":"2019-09-17T22:17:19Z"} +/var/log/gitlab/gitlab-rails/production_json.log:{"method":"GET","path":"/root/linux/blob/master/README","format":"json","controller":"Projects::BlobController","action":"show","status":200,"duration":2448.77,"view":0.49,"db":21.63,"time":"2019-09-17T22:17:19.800Z","params":[{"key":"viewer","value":"rich"},{"key":"namespace_id","value":"root"},{"key":"project_id","value":"linux"},{"key":"id","value":"master/README"}],"remote_ip":"[filtered]","user_id":2,"username":"bob","ua":"Mozilla/5.0 (Mac) Gecko Firefox/69.0","queue_duration":3.38,"gitaly_calls":1,"gitaly_duration":0.77,"rugged_calls":4,"rugged_duration_ms":28.74,"correlation_id":"LOt9hgi1TV4"} +``` + +### Searching in distributed architectures + +If you have done some horizontal scaling in your GitLab infrastructure, then +you will need to search across _all_ of your GitLab nodes. You can do this with +some sort of log aggregation software like Loki, ELK, Splunk, or others. + +You can use a tool like Ansible or PSSH (parellel SSH) that can execute identical commands across your servers in +parallel, or craft your own solution. diff --git a/doc/administration/wikis/index.md b/doc/administration/wikis/index.md new file mode 100644 index 00000000000..077b4f064dc --- /dev/null +++ b/doc/administration/wikis/index.md @@ -0,0 +1,75 @@ +--- +type: reference, howto +stage: Create +group: Knowledge +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Wiki settings **(CORE ONLY)** + +Adjust the wiki settings of your GitLab instance. + +## Wiki page content size limit + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/31176) in GitLab 13.2. + +You can set a maximum content size limit for wiki pages. This limit can prevent +abuse of the feature. The default value is **52428800 Bytes** (50 MB). + +### How does it work? + +The content size limit will be applied when a wiki page is created or updated +through the GitLab UI or API. Local changes pushed via Git will not be validated. + +In order not to break any existing wiki pages, the limit doesn't have any +effect on them until a wiki page is edited again and the content changes. + +### Wiki page content size limit configuration + +This setting is not available through the [Admin Area settings](../../user/admin_area/settings/index.md). +In order to configure this setting, use either the Rails console +or the [Application settings API](../../api/settings.md). + +NOTE: **Note:** +The value of the limit **must** be in bytes. The minimum value is 1024 bytes. + +#### Through the Rails console + +The steps to configure this setting through the Rails console are: + +1. Start the Rails console: + + ```shell + # For Omnibus installations + sudo gitlab-rails console + + # For installations from source + sudo -u git -H bundle exec rails console -e production + ``` + +1. Update the wiki page maximum content size: + + ```ruby + ApplicationSetting.first.update!(wiki_page_max_content_bytes: 50.megabytes) + ``` + +To retrieve the current value, start the Rails console and run: + + ```ruby + Gitlab::CurrentSettings.wiki_page_max_content_bytes + ``` + +#### Through the API + +The process to set the wiki page size limit through the Application Settings API is +exactly the same as you would do to [update any other setting](../../api/settings.md#change-application-settings). + +```shell +curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/application/settings?wiki_page_max_content_bytes=52428800 +``` + +You can also use the API to [retrieve the current value](../../api/settings.md#get-current-application-settings). + +```shell +curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/application/settings +``` diff --git a/doc/api/README.md b/doc/api/README.md index b07f14b5a7a..82cce57f47b 100644 --- a/doc/api/README.md +++ b/doc/api/README.md @@ -156,7 +156,7 @@ for example, without needing to explicitly pass an access token. With a few API endpoints you can use a [GitLab CI/CD job token](../user/project/new_ci_build_permissions_model.md#job-token) to authenticate with the API: -- [Get job artifacts](jobs.md#get-job-artifacts) +- [Get job artifacts](job_artifacts.md#get-job-artifacts) - [Pipeline triggers](pipeline_triggers.md) - [Release creation](releases/index.md#create-a-release) @@ -218,6 +218,7 @@ Only available to [administrators](../user/permissions.md). All API requests support performing an API call as if you were another user, provided you are authenticated as an administrator with an OAuth or Personal Access Token that has the `sudo` scope. +The API requests are executed with the permissions of the impersonated user. You need to pass the `sudo` parameter either via query string or a header with an ID/username of the user you want to perform the operation as. If passed as a header, the @@ -337,10 +338,10 @@ In the example below, we list 50 [namespaces](namespaces.md) per page. curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/namespaces?per_page=50" ``` -#### Pagination Link header +#### Pagination `Link` header -[Link headers](https://www.w3.org/wiki/LinkHeader) are sent back with each -response. They have `rel` set to prev/next/first/last and contain the relevant +[`Link` headers](https://www.w3.org/wiki/LinkHeader) are sent back with each +response. They have `rel` set to `prev`/`next`/`first`/`last` and contain the relevant URL. Please use these links instead of generating your own URLs. In the cURL example below, we limit the output to 3 items per page (`per_page=3`) @@ -423,12 +424,14 @@ Status: 200 OK ``` CAUTION: **Deprecation:** -The `Links` Header will be removed in GitLab 14.0 to be aligned with the [W3C `Link` specification](https://www.w3.org/wiki/LinkHeader) +The `Links` header will be removed in GitLab 14.0 to be aligned with the [W3C `Link` specification](https://www.w3.org/wiki/LinkHeader). +The `Link` header was [added in GitLab 13.1](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/33714) +and should be used instead. The link to the next page contains an additional filter `id_after=42` which excludes records we have retrieved already. Note the type of filter depends on the `order_by` option used and we may have more than one additional filter. -When the end of the collection has been reached and there are no additional records to retrieve, the `Links` header is absent and the resulting array is empty. +When the end of the collection has been reached and there are no additional records to retrieve, the `Link` header is absent and the resulting array is empty. We recommend using only the given link to retrieve the next page instead of building your own URL. Apart from the headers shown, we don't expose additional pagination headers. @@ -544,7 +547,7 @@ https://gitlab.example.com/api/v4/projects/import ### Array of hashes -`variables` is a parameter of type `array` containing hash key/value pairs `[{ 'key' => 'UPLOAD_TO_S3', 'value' => 'true' }]`: +`variables` is a parameter of type `array` containing hash key/value pairs `[{ 'key': 'UPLOAD_TO_S3', 'value': 'true' }]`: ```shell curl --globoff --request POST --header "PRIVATE-TOKEN: <your_access_token>" \ diff --git a/doc/api/access_requests.md b/doc/api/access_requests.md index 53198d05b46..c133a362788 100644 --- a/doc/api/access_requests.md +++ b/doc/api/access_requests.md @@ -1,6 +1,13 @@ +--- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference, api +--- + # Group and project access requests API - >**Note:** This feature was introduced in GitLab 8.11 +> Introduced in GitLab 8.11. ## Valid access levels diff --git a/doc/api/admin_sidekiq_queues.md b/doc/api/admin_sidekiq_queues.md index 32d336e79fe..5c841ae4076 100644 --- a/doc/api/admin_sidekiq_queues.md +++ b/doc/api/admin_sidekiq_queues.md @@ -1,6 +1,6 @@ # Admin Sidekiq queues API -> **Note:** This feature was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/25998) in GitLab 12.9 +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/25998) in GitLab 12.9 Delete jobs from a Sidekiq queue that match the given [metadata](../development/logging.md#logging-context-metadata-through-rails-or-grape-requests). diff --git a/doc/api/api_resources.md b/doc/api/api_resources.md index e93dfed3b1f..886f2e990f0 100644 --- a/doc/api/api_resources.md +++ b/doc/api/api_resources.md @@ -46,7 +46,7 @@ The following API resources are available in the project context: | [Merge requests](merge_requests.md) | `/projects/:id/merge_requests` (also available for groups and standalone) | | [Notes](notes.md) (comments) | `/projects/:id/issues/.../notes`, `/projects/:id/snippets/.../notes`, `/projects/:id/merge_requests/.../notes` (also available for groups) | | [Notification settings](notification_settings.md) | `/projects/:id/notification_settings` (also available for groups and standalone) | -| [Packages](packages.md) **(PREMIUM)** | `/projects/:id/packages` | +| [Packages](packages.md) | `/projects/:id/packages` | | [Pages domains](pages_domains.md) | `/projects/:id/pages` (also available standalone) | | [Pipelines](pipelines.md) | `/projects/:id/pipelines` | | [Pipeline schedules](pipeline_schedules.md) | `/projects/:id/pipeline_schedules` | @@ -140,6 +140,7 @@ The following API resources are available outside of project and group contexts | [Namespaces](namespaces.md) | `/namespaces` | | [Notification settings](notification_settings.md) | `/notification_settings` (also available for groups and projects) | | [Pages domains](pages_domains.md) | `/pages/domains` (also available for projects) | +| [Personal access tokens](personal_access_tokens.md) | `/personal_access_tokens` | | [Projects](projects.md) | `/users/:id/projects` (also available for projects) | | [Project repository storage moves](project_repository_storage_moves.md) **(CORE ONLY)** | `/project_repository_storage_moves` | | [Runners](runners.md) | `/runners` (also available for projects) | diff --git a/doc/api/branches.md b/doc/api/branches.md index 7a64f62189e..fbb5368cabc 100644 --- a/doc/api/branches.md +++ b/doc/api/branches.md @@ -1,3 +1,10 @@ +--- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference, api +--- + # Branches API This API operates on [repository branches](../user/project/repository/branches/index.md). diff --git a/doc/api/commits.md b/doc/api/commits.md index 9be4ce4fcdb..da95e9a943f 100644 --- a/doc/api/commits.md +++ b/doc/api/commits.md @@ -1,3 +1,10 @@ +--- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference, api +--- + # Commits API ## List repository commits @@ -292,9 +299,10 @@ Parameters: | Attribute | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | | `sha` | string | yes | The commit hash | | `branch` | string | yes | The name of the branch | +| `dry_run` | boolean | no | Does not commit any changes. Default is false. [Introduced in GitLab 13.3](https://gitlab.com/gitlab-org/gitlab/-/issues/231032) | ```shell curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --form "branch=master" "https://gitlab.example.com/api/v4/projects/5/repository/commits/master/cherry_pick" @@ -338,6 +346,19 @@ indicates that the commit already exists in the target branch. The other possible error code is `conflict`, which indicates that there was a merge conflict. +When `dry_run` is enabled, the server will attempt to apply the cherry-pick _but +not actually commit any resulting changes_. If the cherry-pick applies cleanly, +the API will respond with `200 OK`: + +```json +{ + "dry_run": "success" +} +``` + +In the event of a failure, you'll see an error identical to a failure without +dry run. + ## Revert a commit > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/22919) in GitLab 11.5. @@ -355,6 +376,7 @@ Parameters: | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) | | `sha` | string | yes | Commit SHA to revert | | `branch` | string | yes | Target branch name | +| `dry_run` | boolean | no | Does not commit any changes. Default is false. [Introduced in GitLab 13.3](https://gitlab.com/gitlab-org/gitlab/-/issues/231032) | ```shell curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --form "branch=master" "https://gitlab.example.com/api/v4/projects/5/repository/commits/a738f717824ff53aebad8b090c1b79a14f2bd9e8/revert" @@ -393,6 +415,19 @@ In this case, the revert failed because the attempted revert generated a merge conflict. The other possible error code is `empty`, which indicates that the changeset was empty, likely due to the change having already been reverted. +When `dry_run` is enabled, the server will attempt to apply the revert _but not +actually commit any resulting changes_. If the revert applies cleanly, the API +will respond with `200 OK`: + +```json +{ + "dry_run": "success" +} +``` + +In the event of a failure, you'll see an error identical to a failure without +dry run. + ## Get the diff of a commit Get the diff of a commit in a project. diff --git a/doc/api/discussions.md b/doc/api/discussions.md index aa1a691a8f8..b9feef843b1 100644 --- a/doc/api/discussions.md +++ b/doc/api/discussions.md @@ -1,7 +1,8 @@ --- -stage: Plan -group: Project Management +stage: Create +group: Source Code info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: reference, api --- # Discussions API diff --git a/doc/api/environments.md b/doc/api/environments.md index 2287ec9aad2..8b900ad2fd3 100644 --- a/doc/api/environments.md +++ b/doc/api/environments.md @@ -34,7 +34,7 @@ Example response: "id": 1, "name": "review/fix-foo", "slug": "review-fix-foo-dfjre3", - "external_url": "https://review-fix-foo-dfjre3.example.gitlab.com", + "external_url": "https://review-fix-foo-dfjre3.gitlab.example.com", "state": "available" } ] @@ -62,7 +62,7 @@ Example of response "id": 1, "name": "review/fix-foo", "slug": "review-fix-foo-dfjre3", - "external_url": "https://review-fix-foo-dfjre3.example.gitlab.com" + "external_url": "https://review-fix-foo-dfjre3.gitlab.example.com", "state": "available", "last_deployment": { "id": 100, @@ -78,7 +78,7 @@ Example of response "username": "root", "avatar_url": "http://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon", "web_url": "http://localhost:3000/root" - } + }, "deployable": { "id": 710, "status": "success", @@ -107,7 +107,7 @@ Example of response "twitter": "", "website_url": "", "organization": null - } + }, "commit": { "id": "416d8ea11849050d3d1f5104cf8cf51053e790ab", "short_id": "416d8ea1", @@ -164,7 +164,7 @@ POST /projects/:id/environments | `external_url` | string | no | Place to link to for this environment | ```shell -curl --data "name=deploy&external_url=https://deploy.example.gitlab.com" --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/environments" +curl --data "name=deploy&external_url=https://deploy.gitlab.example.com" --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/environments" ``` Example response: @@ -174,7 +174,7 @@ Example response: "id": 1, "name": "deploy", "slug": "deploy", - "external_url": "https://deploy.example.gitlab.com", + "external_url": "https://deploy.gitlab.example.com", "state": "available" } ``` @@ -197,7 +197,7 @@ PUT /projects/:id/environments/:environments_id | `external_url` | string | no | The new `external_url` | ```shell -curl --request PUT --data "name=staging&external_url=https://staging.example.gitlab.com" --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/environments/1" +curl --request PUT --data "name=staging&external_url=https://staging.gitlab.example.com" --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/environments/1" ``` Example response: @@ -207,7 +207,7 @@ Example response: "id": 1, "name": "staging", "slug": "staging", - "external_url": "https://staging.example.gitlab.com", + "external_url": "https://staging.gitlab.example.com", "state": "available" } ``` @@ -253,7 +253,7 @@ Example response: "id": 1, "name": "deploy", "slug": "deploy", - "external_url": "https://deploy.example.gitlab.com", + "external_url": "https://deploy.gitlab.example.com", "state": "stopped" } ``` diff --git a/doc/api/epic_links.md b/doc/api/epic_links.md index 1d54bfe01e3..a2477123ce4 100644 --- a/doc/api/epic_links.md +++ b/doc/api/epic_links.md @@ -1,7 +1,6 @@ # Epic Links API **(ULTIMATE)** ->**Note:** -> This endpoint was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/9188) in GitLab 11.8. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/9188) in GitLab 11.8. Manages parent-child [epic relationships](../user/group/epics/index.md#multi-level-child-epics-ultimate). diff --git a/doc/api/epics.md b/doc/api/epics.md index fcdbb8cea71..45bf406dec2 100644 --- a/doc/api/epics.md +++ b/doc/api/epics.md @@ -92,7 +92,7 @@ Example response: "description": "Molestias dolorem eos vitae expedita impedit necessitatibus quo voluptatum.", "state": "opened", "confidential": "false", - "web_url": "http://localhost:3001/groups/test/-/epics/4", + "web_url": "http://gitlab.example.com/groups/test/-/epics/4", "reference": "&4", "references": { "short": "&4", @@ -105,7 +105,7 @@ Example response: "username": "kam", "state": "active", "avatar_url": "http://www.gravatar.com/avatar/018729e129a6f31c80a6327a30196823?s=80&d=identicon", - "web_url": "http://localhost:3001/kam" + "web_url": "http://gitlab.example.com/kam" }, "start_date": null, "start_date_is_fixed": false, @@ -123,7 +123,12 @@ Example response: "closed_at": "2018-08-18T12:22:05.239Z", "labels": [], "upvotes": 4, - "downvotes": 0 + "downvotes": 0, + "_links":{ + "self": "http://gitlab.example.com/api/v4/groups/7/epics/4", + "epic_issues": "http://gitlab.example.com/api/v4/groups/7/epics/4/issues", + "group":"http://gitlab.example.com/api/v4/groups/7" + } }, { "id": 50, @@ -133,7 +138,7 @@ Example response: "title": "Accusamus iste et ullam ratione voluptatem omnis debitis dolor est.", "description": "Molestias dolorem eos vitae expedita impedit necessitatibus quo voluptatum.", "state": "opened", - "web_url": "http://localhost:3001/groups/test/sample/-/epics/4", + "web_url": "http://gitlab.example.com/groups/test/sample/-/epics/35", "reference": "&4", "references": { "short": "&4", @@ -146,7 +151,7 @@ Example response: "username": "kam", "state": "active", "avatar_url": "http://www.gravatar.com/avatar/018729e129a6f31c80a6327a30196823?s=80&d=identicon", - "web_url": "http://localhost:3001/kam" + "web_url": "http://gitlab.example.com/kam" }, "start_date": null, "start_date_is_fixed": false, @@ -164,7 +169,12 @@ Example response: "closed_at": "2018-08-18T12:22:05.239Z", "labels": [], "upvotes": 4, - "downvotes": 0 + "downvotes": 0, + "_links":{ + "self": "http://gitlab.example.com/api/v4/groups/17/epics/35", + "epic_issues": "http://gitlab.example.com/api/v4/groups/17/epics/35/issues", + "group":"http://gitlab.example.com/api/v4/groups/17" + } } ] ``` @@ -196,7 +206,7 @@ Example response: "title": "Ea cupiditate dolores ut vero consequatur quasi veniam voluptatem et non.", "description": "Molestias dolorem eos vitae expedita impedit necessitatibus quo voluptatum.", "state": "opened", - "web_url": "http://localhost:3001/groups/test/-/epics/5", + "web_url": "http://gitlab.example.com/groups/test/-/epics/5", "reference": "&5", "references": { "short": "&5", @@ -209,7 +219,7 @@ Example response: "username": "arnita", "state": "active", "avatar_url": "http://www.gravatar.com/avatar/a2f5c6fcef64c9c69cb8779cb292be1b?s=80&d=identicon", - "web_url": "http://localhost:3001/arnita" + "web_url": "http://gitlab.example.com/arnita" }, "start_date": null, "start_date_is_fixed": false, @@ -228,7 +238,12 @@ Example response: "labels": [], "upvotes": 4, "downvotes": 0, - "subscribed": true + "subscribed": true, + "_links":{ + "self": "http://gitlab.example.com/api/v4/groups/7/epics/5", + "epic_issues": "http://gitlab.example.com/api/v4/groups/7/epics/5/issues", + "group":"http://gitlab.example.com/api/v4/groups/7" + } } ``` @@ -273,7 +288,7 @@ Example response: "description": "Epic description", "state": "opened", "confidential": "false", - "web_url": "http://localhost:3001/groups/test/-/epics/6", + "web_url": "http://gitlab.example.com/groups/test/-/epics/6", "reference": "&6", "references": { "short": "&6", @@ -304,7 +319,12 @@ Example response: "closed_at": "2018-08-18T12:22:05.239Z", "labels": [], "upvotes": 4, - "downvotes": 0 + "downvotes": 0, + "_links":{ + "self": "http://gitlab.example.com/api/v4/groups/7/epics/6", + "epic_issues": "http://gitlab.example.com/api/v4/groups/7/epics/6/issues", + "group":"http://gitlab.example.com/api/v4/groups/7" + } } ``` @@ -350,7 +370,7 @@ Example response: "description": "Epic description", "state": "opened", "confidential": "false", - "web_url": "http://localhost:3001/groups/test/-/epics/6", + "web_url": "http://gitlab.example.com/groups/test/-/epics/6", "reference": "&6", "references": { "short": "&6", @@ -456,9 +476,9 @@ Example response: "username": "arnita", "state": "active", "avatar_url": "http://www.gravatar.com/avatar/a2f5c6fcef64c9c69cb8779cb292be1b?s=80&d=identicon", - "web_url": "http://localhost:3001/arnita" + "web_url": "http://gitlab.example.com/arnita" }, - "web_url": "http://localhost:3001/groups/test/-/epics/5", + "web_url": "http://gitlab.example.com/groups/test/-/epics/5", "reference": "&5", "references": { "short": "&5", diff --git a/doc/api/error_tracking.md b/doc/api/error_tracking.md index 658480ce6fa..5bb5016d0fd 100644 --- a/doc/api/error_tracking.md +++ b/doc/api/error_tracking.md @@ -10,7 +10,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w ## Error Tracking project settings -The project settings API allows you to retrieve the Error Tracking settings for a project. Only for project maintainers. +The project settings API allows you to retrieve the [Error Tracking](../operations/error_tracking.md) +settings for a project. Only for project maintainers. ### Get Error Tracking settings diff --git a/doc/api/events.md b/doc/api/events.md index 99bb6d5af2b..3f4f11b9786 100644 --- a/doc/api/events.md +++ b/doc/api/events.md @@ -80,6 +80,7 @@ Example response: ```json [ { + "id": 1, "title":null, "project_id":1, "action_name":"opened", @@ -99,6 +100,7 @@ Example response: "author_username":"user3" }, { + "id": 2, "title":null, "project_id":1, "action_name":"opened", @@ -152,6 +154,7 @@ Example response: ```json [ { + "id": 3, "title": null, "project_id": 15, "action_name": "closed", @@ -170,6 +173,7 @@ Example response: "author_username": "root" }, { + "id": 4, "title": null, "project_id": 15, "action_name": "pushed", @@ -197,6 +201,7 @@ Example response: "target_title": null }, { + "id": 5, "title": null, "project_id": 15, "action_name": "closed", @@ -215,6 +220,7 @@ Example response: "author_username": "root" }, { + "id": 7, "title": null, "project_id": 15, "action_name": "commented on", @@ -255,7 +261,8 @@ Example response: ## List a Project's visible events ->**Note:** This endpoint has been around longer than the others. Documentation was formerly located in the [Projects API pages](projects.md). +NOTE: **Note:** +This endpoint has been around longer than the others. Documentation was formerly located in the [Projects API pages](projects.md). Get a list of visible events for a particular project. @@ -285,6 +292,7 @@ Example response: ```json [ { + "id": 8 "title":null, "project_id":1, "action_name":"opened", @@ -305,6 +313,7 @@ Example response: "author_username":"user3" }, { + "id": 9, "title":null, "project_id":1, "action_name":"opened", @@ -325,6 +334,7 @@ Example response: "author_username":"ted" }, { + "id": 10, "title": null, "project_id": 1, "action_name": "commented on", diff --git a/doc/api/feature_flags.md b/doc/api/feature_flags.md index 99303e23c37..479f82914a9 100644 --- a/doc/api/feature_flags.md +++ b/doc/api/feature_flags.md @@ -45,6 +45,7 @@ Example response: { "name":"merge_train", "description":"This feature is about merge train", + "active": true, "version": "new_version_flag", "created_at":"2019-11-04T08:13:51.423Z", "updated_at":"2019-11-04T08:13:51.423Z", @@ -68,6 +69,7 @@ Example response: { "name":"new_live_trace", "description":"This is a new live trace feature", + "active": true, "version": "new_version_flag", "created_at":"2019-11-04T08:13:10.507Z", "updated_at":"2019-11-04T08:13:10.507Z", @@ -94,13 +96,13 @@ Example response: Gets a single feature flag. ```plaintext -GET /projects/:id/feature_flags/:name +GET /projects/:id/feature_flags/:feature_flag_name ``` | Attribute | Type | Required | Description | | ------------------- | ---------------- | ---------- | ---------------------------------------------------------------------------------------| | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | -| `name` | string | yes | The name of the feature flag. | +| `feature_flag_name` | string | yes | The name of the feature flag. | ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/1/feature_flags/awesome_feature @@ -112,6 +114,7 @@ Example response: { "name": "awesome_feature", "description": null, + "active": true, "version": "new_version_flag", "created_at": "2020-05-13T19:56:33.119Z", "updated_at": "2020-05-13T19:56:33.119Z", @@ -146,6 +149,7 @@ POST /projects/:id/feature_flags | `name` | string | yes | The name of the feature flag. | | `version` | string | yes | The version of the feature flag. Must be `new_version_flag`. Omit or set to `legacy_flag` to create a [Legacy Feature Flag](feature_flags_legacy.md). | | `description` | string | no | The description of the feature flag. | +| `active` | boolean | no | The active state of the flag. Defaults to true. [Supported](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/38350) in GitLab 13.3 and later. | | `strategies` | JSON | no | The feature flag [strategies](../operations/feature_flags.md#feature-flag-strategies). | | `strategies:name` | JSON | no | The strategy name. | | `strategies:parameters` | JSON | no | The strategy parameters. | @@ -171,6 +175,7 @@ Example response: { "name": "awesome_feature", "description": null, + "active": true, "version": "new_version_flag", "created_at": "2020-05-13T19:56:33.119Z", "updated_at": "2020-05-13T19:56:33.119Z", @@ -196,14 +201,16 @@ Example response: Updates a feature flag. ```plaintext -PUT /projects/:id/feature_flags/:name +PUT /projects/:id/feature_flags/:feature_flag_name ``` | Attribute | Type | Required | Description | | ------------------- | ---------------- | ---------- | ---------------------------------------------------------------------------------------| | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | -| `name` | string | yes | The name of the feature flag. | +| `feature_flag_name` | string | yes | The current name of the feature flag. | | `description` | string | no | The description of the feature flag. | +| `active` | boolean | no | The active state of the flag. [Supported](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/38350) in GitLab 13.3 and later. | +| `name` | string | no | The new name of the feature flag. [Supported](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/38350) in GitLab 13.3 and later. | | `strategies` | JSON | no | The feature flag [strategies](../operations/feature_flags.md#feature-flag-strategies). | | `strategies:id` | JSON | no | The feature flag strategy id. | | `strategies:name` | JSON | no | The strategy name. | @@ -229,6 +236,7 @@ Example response: { "name": "awesome_feature", "description": null, + "active": true, "version": "new_version_flag", "created_at": "2020-05-13T20:10:32.891Z", "updated_at": "2020-05-13T20:10:32.891Z", @@ -268,13 +276,13 @@ Example response: Deletes a feature flag. ```plaintext -DELETE /projects/:id/feature_flags/:name +DELETE /projects/:id/feature_flags/:feature_flag_name ``` | Attribute | Type | Required | Description | | ------------------- | ---------------- | ---------- | ---------------------------------------------------------------------------------------| | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | -| `name` | string | yes | The name of the feature flag. | +| `feature_flag_name` | string | yes | The name of the feature flag. | ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" --request DELETE "https://gitlab.example.com/api/v4/projects/1/feature_flags/awesome_feature" diff --git a/doc/api/feature_flags_legacy.md b/doc/api/feature_flags_legacy.md index 7e4fc47a1de..175261b3a7b 100644 --- a/doc/api/feature_flags_legacy.md +++ b/doc/api/feature_flags_legacy.md @@ -44,6 +44,7 @@ Example response: { "name":"merge_train", "description":"This feature is about merge train", + "active": true, "created_at":"2019-11-04T08:13:51.423Z", "updated_at":"2019-11-04T08:13:51.423Z", "scopes":[ @@ -97,6 +98,7 @@ Example response: { "name":"new_live_trace", "description":"This is a new live trace feature", + "active": true, "created_at":"2019-11-04T08:13:10.507Z", "updated_at":"2019-11-04T08:13:10.507Z", "scopes":[ @@ -163,6 +165,7 @@ POST /projects/:id/feature_flags | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | | `name` | string | yes | The name of the feature flag. | | `description` | string | no | The description of the feature flag. | +| `active` | boolean | no | The active state of the flag. Defaults to true. [Supported](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/38350) in GitLab 13.3 and later. | | `scopes` | JSON | no | The feature flag specs of the feature flag. | | `scopes:environment_scope` | string | no | The environment spec. | | `scopes:active` | boolean | no | Whether the spec is active. | @@ -187,6 +190,7 @@ Example response: { "name":"awesome_feature", "description":null, + "active": true, "created_at":"2019-11-04T08:32:27.288Z", "updated_at":"2019-11-04T08:32:27.288Z", "scopes":[ @@ -247,6 +251,7 @@ Example response: { "name":"new_live_trace", "description":"This is a new live trace feature", + "active": true, "created_at":"2019-11-04T08:13:10.507Z", "updated_at":"2019-11-04T08:13:10.507Z", "scopes":[ diff --git a/doc/api/geo_nodes.md b/doc/api/geo_nodes.md index 402170fba37..ba970d2cdb1 100644 --- a/doc/api/geo_nodes.md +++ b/doc/api/geo_nodes.md @@ -325,9 +325,11 @@ Example response: "design_repositories_failed_count": nil, "design_repositories_synced_in_percentage": "0.00%", "projects_count": 41, + "repositories_count": 41, "repositories_failed_count": nil, "repositories_synced_count": nil, "repositories_synced_in_percentage": "0.00%", + "wikis_count": 41, "wikis_failed_count": nil, "wikis_synced_count": nil, "wikis_synced_in_percentage": "0.00%", @@ -402,9 +404,11 @@ Example response: "design_repositories_failed_count": nil, "design_repositories_synced_in_percentage": "0.00%", "projects_count": 41, + "repositories_count": 41, "repositories_failed_count": 1, "repositories_synced_count": 40, "repositories_synced_in_percentage": "97.56%", + "wikis_count": 41, "wikis_failed_count": 0, "wikis_synced_count": 41, "wikis_synced_in_percentage": "100.00%", @@ -448,9 +452,6 @@ Example response: ] ``` -NOTE: **Note:** -In GitLab 12.0, deprecated fields `wikis_count` and `repositories_count` were removed. Use `projects_count` instead. - ## Retrieve status about a specific Geo node ```plaintext @@ -495,9 +496,11 @@ Example response: "design_repositories_failed_count": nil, "design_repositories_synced_in_percentage": "0.00%", "projects_count": 41, + "repositories_count": 41, "repositories_failed_count": 1, "repositories_synced_count": 40, "repositories_synced_in_percentage": "97.56%", + "wikis_count": 41, "wikis_failed_count": 0, "wikis_synced_count": 41, "wikis_synced_in_percentage": "100.00%", @@ -517,9 +520,6 @@ Example response: Note: The `health_status` parameter can only be in an "Healthy" or "Unhealthy" state, while the `health` parameter can be empty, "Healthy", or contain the actual error message. -NOTE: **Note:** -In GitLab 12.0, deprecated fields `wikis_count` and `repositories_count` were removed. Use `projects_count` instead. - ## Retrieve project sync or verification failures that occurred on the current node This only works on a secondary node. diff --git a/doc/api/graphql/audit_report.md b/doc/api/graphql/audit_report.md new file mode 100644 index 00000000000..36c3f44ff89 --- /dev/null +++ b/doc/api/graphql/audit_report.md @@ -0,0 +1,116 @@ +# Set up an Audit Report with GraphQL + +This page describes how you can use the GraphiQL explorer to set up an audit report +for a specific subset of users. + +You can run the same query directly via a HTTP endpoint, using `cURL`. For more information, see our +guidance on getting started from the [command line](getting_started.md#command-line). + +The [example users query](#set-up-the-graphiql-explorer) looks for a subset of users in +a GitLab instance either by username or +[Global ID](../../development/api_graphql_styleguide.md#global-ids). +The query includes: + +- [`pageInfo`](#pageinfo) +- [`nodes`](#nodes) + +## pageInfo + +This contains the data needed to implement pagination. GitLab uses cursor-based +[pagination](getting_started.md#pagination). For more information, see +[Pagination](https://graphql.org/learn/pagination/) in the GraphQL documentation. + +## nodes + +In a GraphQL query, `nodes` is used to represent a collection of [`nodes` on a graph](https://en.wikipedia.org/wiki/Vertex_(graph_theory)). +In this case, the collection of nodes is a collection of `User` objects. For each one, +we output: + +- Their user's `id`. +- The `membership` fragment, which represents a Project or Group membership belonging + to that user. Outputting a fragment is denoted with the `...memberships` notation. + +The GitLab GraphQL API is extensive and a large amount of data for a wide variety of entities can be output. +See the official [reference documentation](reference/index.md) for the most up-to-date information. + +## Set up the GraphiQL explorer + +This procedure presents a substantive example that you can copy and paste into GraphiQL +explorer. GraphiQL explorer is available for: + +- GitLab.com users at [https://gitlab.com/-/graphql-explorer](https://gitlab.com/-/graphql-explorer). +- Self-managed users at `https://gitlab.example.com/-/graphql-explorer`. + +1. Copy the following code excerpt: + + ```graphql + { + users(usernames: ["user1", "user2", "user3"]) { + pageInfo { + endCursor + startCursor + hasNextPage + } + nodes { + id + ...memberships + } + } + } + + fragment membership on MemberInterface { + createdAt + updatedAt + accessLevel { + integerValue + stringValue + } + createdBy { + id + } + } + + fragment memberships on User { + groupMemberships { + nodes { + ...membership + group { + id + name + } + } + } + + projectMemberships { + nodes { + ...membership + project { + id + name + } + } + } + } + ``` + +1. Open the [GraphiQL explorer tool](https://gitlab.com/-/graphql-explorer). +1. Paste the `query` listed above into the left window of your GraphiQL explorer tool. +1. Click Play to get the result shown here: + + + +NOTE: **Note:** +[The GraphQL API returns a GlobalID, rather than a standard ID.](getting_started.md#queries-and-mutations) It also expects a GlobalID as an input rather than +a single integer. + +This GraphQL query returns the groups and projects that the user has been *explicitly* made a member of. +Since the GraphiQL explorer uses the session token to authorize access to resources, +the output is limited to the projects and groups accessible to the currently signed-in user. + +If you've signed in as an instance administrator, you would have access to all records, regardless of ownership. + +For more information on: + +- GraphQL specific entities, such as Fragments and Interfaces, see the official + [GraphQL documentation](https://graphql.org/learn/). +- Individual attributes, see the [GraphQL API Resources](reference/index.md). diff --git a/doc/api/graphql/getting_started.md b/doc/api/graphql/getting_started.md index bf8a2120734..12665f68f25 100644 --- a/doc/api/graphql/getting_started.md +++ b/doc/api/graphql/getting_started.md @@ -59,8 +59,9 @@ The GitLab GraphQL API can be used to perform: - [Mutations](#mutations) for creating, updating, and deleting data. NOTE: **Note:** -In the GitLab GraphQL API, `id` generally refers to a global ID, -which is an object identifier in the format of `gid://gitlab/Issue/123`. +In the GitLab GraphQL API, `id` refers to a +[Global ID](https://graphql.org/learn/global-object-identification/), +which is an object identifier in the format of `"gid://gitlab/Issue/123"`. [GitLab's GraphQL Schema](reference/index.md) outlines which objects and fields are available for clients to query and their corresponding data types. diff --git a/doc/api/graphql/img/sample_issue_boards_v13_2.png b/doc/api/graphql/img/sample_issue_boards_v13_2.png Binary files differnew file mode 100644 index 00000000000..5afe5f5151a --- /dev/null +++ b/doc/api/graphql/img/sample_issue_boards_v13_2.png diff --git a/doc/api/graphql/img/user_query_example_v13_2.png b/doc/api/graphql/img/user_query_example_v13_2.png Binary files differnew file mode 100644 index 00000000000..270cf5da7fd --- /dev/null +++ b/doc/api/graphql/img/user_query_example_v13_2.png diff --git a/doc/api/graphql/index.md b/doc/api/graphql/index.md index d653c4e0f47..c513dea239a 100644 --- a/doc/api/graphql/index.md +++ b/doc/api/graphql/index.md @@ -14,7 +14,15 @@ For those new to the GitLab GraphQL API, see - Get an [introduction to GraphQL from graphql.org](https://graphql.org/). - GitLab supports a wide range of resources, listed in the [GraphQL API Reference](reference/index.md). -#### GraphiQL +### Examples + +To work with sample queries that pull data from public projects on GitLab.com, +see the menu options in the left-hand +documentation menu, under API > GraphQL at `https://docs.gitlab.com/ee/api/graphql/`. + +The [Getting started](getting_started.md) page includes different methods to customize GraphQL queries. + +### GraphiQL Explore the GraphQL API using the interactive [GraphiQL explorer](https://gitlab.com/-/graphql-explorer), or on your self-managed GitLab instance on diff --git a/doc/api/graphql/reference/gitlab_schema.graphql b/doc/api/graphql/reference/gitlab_schema.graphql index 2ed6bec104d..1d920894eec 100644 --- a/doc/api/graphql/reference/gitlab_schema.graphql +++ b/doc/api/graphql/reference/gitlab_schema.graphql @@ -210,6 +210,11 @@ type AlertManagementAlert implements Noteable { details: JSON """ + The URL of the alert detail page + """ + detailsUrl: String! + + """ All discussions on this noteable """ discussions( @@ -295,6 +300,16 @@ type AlertManagementAlert implements Noteable { ): NoteConnection! """ + The alert condition for Prometheus + """ + prometheusAlert: PrometheusAlert + + """ + Runbook for the alert as defined in alert details + """ + runbook: String + + """ Service the alert came from """ service: String @@ -320,6 +335,61 @@ type AlertManagementAlert implements Noteable { title: String """ + Todos of the current user for the alert + """ + todos( + """ + The action to be filtered + """ + action: [TodoActionEnum!] + + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + The ID of an author + """ + authorId: [ID!] + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + The ID of a group + """ + groupId: [ID!] + + """ + Returns the last _n_ elements from the list. + """ + last: Int + + """ + The ID of a project + """ + projectId: [ID!] + + """ + The state of the todo + """ + state: [TodoStateEnum!] + + """ + The type of the todo + """ + type: [TodoTargetEnum!] + ): TodoConnection + + """ Timestamp the alert was last updated """ updatedAt: Time @@ -874,6 +944,11 @@ type Blob implements Entry { type: EntryType! """ + Web path of the blob + """ + webPath: String + + """ Web URL of the blob """ webUrl: String @@ -928,6 +1003,51 @@ Represents a project or group board """ type Board { """ + The board assignee. + """ + assignee: User + + """ + Epics associated with board issues. + """ + epics( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Filters applied when selecting issues on the board + """ + issueFilters: BoardEpicIssueInput + + """ + Returns the last _n_ elements from the list. + """ + last: Int + ): EpicConnection + + """ + Whether or not backlog list is hidden. + """ + hideBacklogList: Boolean + + """ + Whether or not closed list is hidden. + """ + hideClosedList: Boolean + + """ ID (global ID) of the board """ id: ID! @@ -952,18 +1072,28 @@ type Board { first: Int """ + Find a list by its global ID + """ + id: ID + + """ Returns the last _n_ elements from the list. """ last: Int ): BoardListConnection """ + The board milestone. + """ + milestone: Milestone + + """ Name of the board """ name: String """ - Weight of the board + Weight of the board. """ weight: Int } @@ -1003,6 +1133,58 @@ type BoardEdge { node: Board } +input BoardEpicIssueInput { + """ + Filter by assignee username + """ + assigneeUsername: [String] + + """ + Filter by author username + """ + authorUsername: String + + """ + Filter by epic ID + """ + epicId: String + + """ + Filter by label name + """ + labelName: [String] + + """ + Filter by milestone title + """ + milestoneTitle: String + + """ + Filter by reaction emoji + """ + myReactionEmoji: String + + """ + List of negated params. Warning: this argument is experimental and a subject to change in future + """ + not: NegatedBoardEpicIssueInput + + """ + Filter by release tag + """ + releaseTag: String + + """ + Filter by weight + """ + weight: String +} + +""" +Identifier of Board +""" +scalar BoardID + """ Represents a list for an issue board """ @@ -1023,6 +1205,36 @@ type BoardList { id: ID! """ + Board issues + """ + issues( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + ): IssueConnection + + """ + Count of issues in the list + """ + issuesCount: Int + + """ Label of the list """ label: Label @@ -1061,6 +1273,11 @@ type BoardList { Title of the list """ title: String! + + """ + Total weight of all issues in the list + """ + totalWeight: Int } """ @@ -1084,6 +1301,51 @@ type BoardListConnection { } """ +Autogenerated input type of BoardListCreate +""" +input BoardListCreateInput { + """ + Create the backlog list + """ + backlog: Boolean + + """ + The Global ID of the issue board to mutate + """ + boardId: BoardID! + + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + ID of an existing label + """ + labelId: LabelID +} + +""" +Autogenerated return type of BoardListCreate +""" +type BoardListCreatePayload { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Errors encountered during execution of the mutation. + """ + errors: [String!]! + + """ + List of the issue board + """ + list: BoardList +} + +""" An edge in a connection. """ type BoardListEdge { @@ -1160,6 +1422,239 @@ type Branch { name: String! } +type CiGroup { + """ + Jobs in group + """ + jobs( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + ): CiJobConnection + + """ + Name of the job group + """ + name: String + + """ + Size of the group + """ + size: Int +} + +""" +The connection type for CiGroup. +""" +type CiGroupConnection { + """ + A list of edges. + """ + edges: [CiGroupEdge] + + """ + A list of nodes. + """ + nodes: [CiGroup] + + """ + Information to aid in pagination. + """ + pageInfo: PageInfo! +} + +""" +An edge in a connection. +""" +type CiGroupEdge { + """ + A cursor for use in pagination. + """ + cursor: String! + + """ + The item at the end of the edge. + """ + node: CiGroup +} + +type CiJob { + """ + Name of the job + """ + name: String + + """ + Builds that must complete before the jobs run + """ + needs( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + ): CiJobConnection +} + +""" +The connection type for CiJob. +""" +type CiJobConnection { + """ + A list of edges. + """ + edges: [CiJobEdge] + + """ + A list of nodes. + """ + nodes: [CiJob] + + """ + Information to aid in pagination. + """ + pageInfo: PageInfo! +} + +""" +An edge in a connection. +""" +type CiJobEdge { + """ + A cursor for use in pagination. + """ + cursor: String! + + """ + The item at the end of the edge. + """ + node: CiJob +} + +type CiStage { + """ + Group of jobs for the stage + """ + groups( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + ): CiGroupConnection + + """ + Name of the stage + """ + name: String +} + +""" +The connection type for CiStage. +""" +type CiStageConnection { + """ + A list of edges. + """ + edges: [CiStageEdge] + + """ + A list of nodes. + """ + nodes: [CiStage] + + """ + Information to aid in pagination. + """ + pageInfo: PageInfo! +} + +""" +An edge in a connection. +""" +type CiStageEdge { + """ + A cursor for use in pagination. + """ + cursor: String! + + """ + The item at the end of the edge. + """ + node: CiStage +} + +type ClusterAgent { + """ + Timestamp the cluster agent was created + """ + createdAt: Time + + """ + ID of the cluster agent + """ + id: ID! + + """ + Name of the cluster agent + """ + name: String + + """ + The project this cluster agent is associated with + """ + project: Project + + """ + Timestamp the cluster agent was updated + """ + updatedAt: Time +} + type Commit { """ Author of the commit @@ -1187,6 +1682,11 @@ type Commit { description: String """ + The GitLab Flavored Markdown rendering of `description` + """ + descriptionHtml: String + + """ ID (global ID) of the commit """ id: ID! @@ -1277,6 +1777,11 @@ type Commit { titleHtml: String """ + Web path of the commit + """ + webPath: String! + + """ Web URL of the commit """ webUrl: String! @@ -1457,6 +1962,46 @@ type ComplianceFrameworkEdge { } """ +Autogenerated input type of ConfigureSast +""" +input ConfigureSastInput { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Payload containing SAST variable values (https://docs.gitlab.com/ee/user/application_security/sast/#available-variables). + """ + configuration: JSON! + + """ + Full path of the project. + """ + projectPath: ID! +} + +""" +Autogenerated return type of ConfigureSast +""" +type ConfigureSastPayload { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Errors encountered during execution of the mutation. + """ + errors: [String!]! + + """ + JSON containing the status of MR creation. + """ + result: JSON +} + +""" A tag expiration policy designed to keep only the images that matter most """ type ContainerExpirationPolicy { @@ -1743,6 +2288,46 @@ type CreateBranchPayload { } """ +Autogenerated input type of CreateClusterAgent +""" +input CreateClusterAgentInput { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Name of the cluster agent + """ + name: String! + + """ + Full path of the associated project for this cluster agent + """ + projectPath: ID! +} + +""" +Autogenerated return type of CreateClusterAgent +""" +type CreateClusterAgentPayload { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Cluster agent created after mutation + """ + clusterAgent: ClusterAgent + + """ + Errors encountered during execution of the mutation. + """ + errors: [String!]! +} + +""" Autogenerated input type of CreateDiffNote """ input CreateDiffNoteInput { @@ -1944,7 +2529,12 @@ input CreateIterationInput { """ The target group for the iteration """ - groupPath: ID! + groupPath: ID + + """ + The target project for the iteration + """ + projectPath: ID """ The start date of the iteration @@ -2072,6 +2662,11 @@ Autogenerated input type of CreateSnippet """ input CreateSnippetInput { """ + Actions to perform over the snippet repository and blobs + """ + blobActions: [SnippetBlobActionInputType!] + + """ A unique identifier for the client performing the mutation. """ clientMutationId: String @@ -2092,11 +2687,6 @@ input CreateSnippetInput { fileName: String """ - The snippet files to create - """ - files: [SnippetFileInputType!] - - """ The project full path the snippet is associated with """ projectPath: ID @@ -2137,6 +2727,46 @@ type CreateSnippetPayload { snippet: Snippet } +""" +Autogenerated input type of DastOnDemandScanCreate +""" +input DastOnDemandScanCreateInput { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + ID of the site profile to be used for the scan. + """ + dastSiteProfileId: DastSiteProfileID! + + """ + The project the site profile belongs to. + """ + fullPath: ID! +} + +""" +Autogenerated return type of DastOnDemandScanCreate +""" +type DastOnDemandScanCreatePayload { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Errors encountered during execution of the mutation. + """ + errors: [String!]! + + """ + URL of the pipeline that was created. + """ + pipelineUrl: String +} + enum DastScanTypeEnum { """ Passive DAST scan. This scan will not make active attacks against the target site. @@ -2145,6 +2775,171 @@ enum DastScanTypeEnum { } """ +Represents a DAST scanner profile. +""" +type DastScannerProfile { + """ + ID of the DAST scanner profile + """ + id: ID! + + """ + Name of the DAST scanner profile + """ + profileName: String + + """ + The maximum number of seconds allowed for the spider to traverse the site + """ + spiderTimeout: Int + + """ + The maximum number of seconds allowed for the site under test to respond to a request + """ + targetTimeout: Int +} + +""" +The connection type for DastScannerProfile. +""" +type DastScannerProfileConnection { + """ + A list of edges. + """ + edges: [DastScannerProfileEdge] + + """ + A list of nodes. + """ + nodes: [DastScannerProfile] + + """ + Information to aid in pagination. + """ + pageInfo: PageInfo! +} + +""" +Autogenerated input type of DastScannerProfileCreate +""" +input DastScannerProfileCreateInput { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + The project the scanner profile belongs to. + """ + fullPath: ID! + + """ + The name of the scanner profile. + """ + profileName: String! + + """ + The maximum number of seconds allowed for the spider to traverse the site. + """ + spiderTimeout: Int + + """ + The maximum number of seconds allowed for the site under test to respond to a request. + """ + targetTimeout: Int +} + +""" +Autogenerated return type of DastScannerProfileCreate +""" +type DastScannerProfileCreatePayload { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Errors encountered during execution of the mutation. + """ + errors: [String!]! + + """ + ID of the scanner profile. + """ + id: ID +} + +""" +An edge in a connection. +""" +type DastScannerProfileEdge { + """ + A cursor for use in pagination. + """ + cursor: String! + + """ + The item at the end of the edge. + """ + node: DastScannerProfile +} + +""" +Represents a DAST Site Profile. +""" +type DastSiteProfile { + """ + Relative web path to the edit page of a site profile + """ + editPath: String + + """ + ID of the site profile + """ + id: DastSiteProfileID! + + """ + The name of the site profile + """ + profileName: String + + """ + The URL of the target to be scanned + """ + targetUrl: String + + """ + Permissions for the current user on the resource + """ + userPermissions: DastSiteProfilePermissions! + + """ + The current validation status of the site profile + """ + validationStatus: DastSiteProfileValidationStatusEnum +} + +""" +The connection type for DastSiteProfile. +""" +type DastSiteProfileConnection { + """ + A list of edges. + """ + edges: [DastSiteProfileEdge] + + """ + A list of nodes. + """ + nodes: [DastSiteProfile] + + """ + Information to aid in pagination. + """ + pageInfo: PageInfo! +} + +""" Autogenerated input type of DastSiteProfileCreate """ input DastSiteProfileCreateInput { @@ -2186,7 +2981,144 @@ type DastSiteProfileCreatePayload { """ ID of the site profile. """ - id: ID + id: DastSiteProfileID +} + +""" +Autogenerated input type of DastSiteProfileDelete +""" +input DastSiteProfileDeleteInput { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + The project the site profile belongs to. + """ + fullPath: ID! + + """ + ID of the site profile to be deleted. + """ + id: DastSiteProfileID! +} + +""" +Autogenerated return type of DastSiteProfileDelete +""" +type DastSiteProfileDeletePayload { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Errors encountered during execution of the mutation. + """ + errors: [String!]! +} + +""" +An edge in a connection. +""" +type DastSiteProfileEdge { + """ + A cursor for use in pagination. + """ + cursor: String! + + """ + The item at the end of the edge. + """ + node: DastSiteProfile +} + +""" +Identifier of DastSiteProfile +""" +scalar DastSiteProfileID + +""" +Check permissions for the current user on site profile +""" +type DastSiteProfilePermissions { + """ + Indicates the user can perform `create_on_demand_dast_scan` on this resource + """ + createOnDemandDastScan: Boolean! +} + +""" +Autogenerated input type of DastSiteProfileUpdate +""" +input DastSiteProfileUpdateInput { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + The project the site profile belongs to. + """ + fullPath: ID! + + """ + ID of the site profile to be updated. + """ + id: DastSiteProfileID! + + """ + The name of the site profile. + """ + profileName: String! + + """ + The URL of the target to be scanned. + """ + targetUrl: String +} + +""" +Autogenerated return type of DastSiteProfileUpdate +""" +type DastSiteProfileUpdatePayload { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Errors encountered during execution of the mutation. + """ + errors: [String!]! + + """ + ID of the site profile. + """ + id: DastSiteProfileID +} + +enum DastSiteProfileValidationStatusEnum { + """ + Site validation process finished but failed + """ + FAILED_VALIDATION + + """ + Site validation process is in progress + """ + INPROGRESS_VALIDATION + + """ + Site validation process finished successfully + """ + PASSED_VALIDATION + + """ + Site validation process has not started + """ + PENDING_VALIDATION } """ @@ -2765,6 +3697,56 @@ type DesignManagementDeletePayload { } """ +Identifier of DesignManagement::Design +""" +scalar DesignManagementDesignID + +""" +Autogenerated input type of DesignManagementMove +""" +input DesignManagementMoveInput { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + ID of the design to move + """ + id: DesignManagementDesignID! + + """ + ID of the immediately following design + """ + next: DesignManagementDesignID + + """ + ID of the immediately preceding design + """ + previous: DesignManagementDesignID +} + +""" +Autogenerated return type of DesignManagementMove +""" +type DesignManagementMovePayload { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + The current state of the collection + """ + designCollection: DesignCollection + + """ + Errors encountered during execution of the mutation. + """ + errors: [String!]! +} + +""" Autogenerated input type of DesignManagementUpload """ input DesignManagementUploadInput { @@ -3524,6 +4506,11 @@ type Environment { id: ID! """ + The most severe open alert for the environment. If multiple alerts have equal severity, the most recent is returned. + """ + latestOpenedMostSevereAlert: AlertManagementAlert + + """ Metrics dashboard schema for the environment """ metricsDashboard( @@ -3644,6 +4631,11 @@ type Epic implements Noteable { last: Int """ + Filter epics by milestone title, computed from epic's issues + """ + milestoneTitle: String + + """ Search query for epic title or description """ search: String @@ -3991,7 +4983,7 @@ input EpicAddIssueInput { issueIid: String! """ - The project the issue belongs to + The full path of the project the issue belongs to """ projectPath: ID! } @@ -4046,7 +5038,7 @@ Counts of descendent epics. """ type EpicDescendantCount { """ - Number of closed sub-epics + Number of closed child epics """ closedEpics: Int @@ -4056,7 +5048,7 @@ type EpicDescendantCount { closedIssues: Int """ - Number of opened sub-epics + Number of opened child epics """ openedEpics: Int @@ -4151,6 +5143,11 @@ type EpicIssue implements Noteable { author: User! """ + Indicates the issue is blocked + """ + blocked: Boolean! + + """ Timestamp of when the issue was closed """ closedAt: Time @@ -4361,6 +5358,11 @@ type EpicIssue implements Noteable { state: IssueState! """ + Indicates whether an issue is published to the status page + """ + statusPagePublishedIncident: Boolean + + """ Indicates the currently logged in user is subscribed to the issue """ subscribed: Boolean! @@ -4391,6 +5393,11 @@ type EpicIssue implements Noteable { totalTimeSpent: Int! """ + Type of the issue + """ + type: IssueType + + """ Timestamp of when the issue was last updated """ updatedAt: Time! @@ -4926,6 +5933,11 @@ type Group { labelName: [String!] """ + Filter epics by milestone title, computed from epic's issues + """ + milestoneTitle: String + + """ Search query for epic title or description """ search: String @@ -5003,6 +6015,11 @@ type Group { last: Int """ + Filter epics by milestone title, computed from epic's issues + """ + milestoneTitle: String + + """ Search query for epic title or description """ search: String @@ -5050,6 +6067,11 @@ type Group { id: ID! """ + Status of the temporary storage increase + """ + isTemporaryStorageIncreaseEnabled: Boolean! + + """ Issues of the group """ issues( @@ -5064,7 +6086,7 @@ type Group { assigneeId: String """ - Username of a user assigned to the issues + Username of a user assigned to the issue """ assigneeUsername: String @@ -5109,6 +6131,11 @@ type Group { iids: [String!] """ + Include issues belonging to subgroups. + """ + includeSubgroups: Boolean = false + + """ Iterations applied to the issue """ iterationId: [ID] @@ -5124,7 +6151,7 @@ type Group { last: Int """ - Milestones applied to this issue + Milestone applied to this issue """ milestoneTitle: [String] @@ -5144,6 +6171,11 @@ type Group { state: IssuableState """ + Filter issues by the given issue types + """ + types: [IssueType!] + + """ Issues updated after this date """ updatedAfter: Time @@ -5267,7 +6299,7 @@ type Group { mentionsDisabled: Boolean """ - Find milestones + Milestones of the group """ milestones( """ @@ -5292,7 +6324,12 @@ type Group { first: Int """ - Return also milestones in all subgroups and subprojects + Array of global milestone IDs, e.g., "gid://gitlab/Milestone/1" + """ + ids: [ID!] + + """ + Also return milestones in all subgroups and subprojects """ includeDescendants: Boolean @@ -5514,7 +6551,43 @@ type Group { ): VulnerabilityConnection """ - Number of vulnerabilities per severity level, per day, for the projects in the group and its subgroups + Number of vulnerabilities per day for the projects in the group and its subgroups + """ + vulnerabilitiesCountByDay( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Last day for which to fetch vulnerability history + """ + endDate: ISO8601Date! + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + + """ + First day for which to fetch vulnerability history + """ + startDate: ISO8601Date! + ): VulnerabilitiesCountByDayConnection + + """ + Number of vulnerabilities per severity level, per day, for the projects in the + group and its subgroups. Deprecated in 13.3: Use `vulnerabilitiesCountByDay` """ vulnerabilitiesCountByDayAndSeverity( """ @@ -5546,7 +6619,12 @@ type Group { First day for which to fetch vulnerability history """ startDate: ISO8601Date! - ): VulnerabilitiesCountByDayAndSeverityConnection + ): VulnerabilitiesCountByDayAndSeverityConnection @deprecated(reason: "Use `vulnerabilitiesCountByDay`. Deprecated in 13.3") + + """ + Represents vulnerable project counts for each grade + """ + vulnerabilityGrades: [VulnerableProjectsByGrade!]! """ Vulnerability scanners reported on the project vulnerabilties of the group and its subgroups @@ -5702,6 +6780,11 @@ type InstanceSecurityDashboard { ): ProjectConnection! """ + Represents vulnerable project counts for each grade + """ + vulnerabilityGrades: [VulnerableProjectsByGrade!]! + + """ Vulnerability scanners reported on the vulnerabilties from projects selected in Instance Security Dashboard """ vulnerabilityScanners( @@ -5731,6 +6814,7 @@ type InstanceSecurityDashboard { State of a GitLab issue or merge request """ enum IssuableState { + all closed locked opened @@ -5768,6 +6852,11 @@ type Issue implements Noteable { author: User! """ + Indicates the issue is blocked + """ + blocked: Boolean! + + """ Timestamp of when the issue was closed """ closedAt: Time @@ -5968,6 +7057,11 @@ type Issue implements Noteable { state: IssueState! """ + Indicates whether an issue is published to the status page + """ + statusPagePublishedIncident: Boolean + + """ Indicates the currently logged in user is subscribed to the issue """ subscribed: Boolean! @@ -5998,6 +7092,11 @@ type Issue implements Noteable { totalTimeSpent: Int! """ + Type of the issue + """ + type: IssueType + + """ Timestamp of when the issue was last updated """ updatedAt: Time! @@ -6074,6 +7173,71 @@ type IssueEdge { } """ +Autogenerated input type of IssueMoveList +""" +input IssueMoveListInput { + """ + Global ID of the board that the issue is in + """ + boardId: ID! + + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + ID of the board list that the issue will be moved from + """ + fromListId: ID + + """ + IID of the issue to mutate + """ + iid: String! + + """ + ID of issue after which the current issue will be positioned at + """ + moveAfterId: ID + + """ + ID of issue before which the current issue will be positioned at + """ + moveBeforeId: ID + + """ + Project the issue to mutate is in + """ + projectPath: ID! + + """ + ID of the board list that the issue will be moved to + """ + toListId: ID +} + +""" +Autogenerated return type of IssueMoveList +""" +type IssueMoveListPayload { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Errors encountered during execution of the mutation. + """ + errors: [String!]! + + """ + The issue after mutation + """ + issue: Issue +} + +""" Check permissions for the current user on a issue """ type IssuePermissions { @@ -6119,6 +7283,56 @@ type IssuePermissions { } """ +Autogenerated input type of IssueSetAssignees +""" +input IssueSetAssigneesInput { + """ + The usernames to assign to the resource. Replaces existing assignees by default. + """ + assigneeUsernames: [String!]! + + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + The IID of the issue to mutate + """ + iid: String! + + """ + The operation to perform. Defaults to REPLACE. + """ + operationMode: MutationOperationMode + + """ + The project the issue to mutate is in + """ + projectPath: ID! +} + +""" +Autogenerated return type of IssueSetAssignees +""" +type IssueSetAssigneesPayload { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Errors encountered during execution of the mutation. + """ + errors: [String!]! + + """ + The issue after mutation + """ + issue: Issue +} + +""" Autogenerated input type of IssueSetConfidential """ input IssueSetConfidentialInput { @@ -6133,7 +7347,7 @@ input IssueSetConfidentialInput { confidential: Boolean! """ - The iid of the issue to mutate + The IID of the issue to mutate """ iid: String! @@ -6178,7 +7392,7 @@ input IssueSetDueDateInput { dueDate: Time! """ - The iid of the issue to mutate + The IID of the issue to mutate """ iid: String! @@ -6209,6 +7423,51 @@ type IssueSetDueDatePayload { } """ +Autogenerated input type of IssueSetEpic +""" +input IssueSetEpicInput { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Global ID of the epic to be assigned to the issue, epic will be removed if absent or set to null + """ + epicId: ID + + """ + The IID of the issue to mutate + """ + iid: String! + + """ + The project the issue to mutate is in + """ + projectPath: ID! +} + +""" +Autogenerated return type of IssueSetEpic +""" +type IssueSetEpicPayload { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Errors encountered during execution of the mutation. + """ + errors: [String!]! + + """ + The issue after mutation + """ + issue: Issue +} + +""" Autogenerated input type of IssueSetIteration """ input IssueSetIterationInput { @@ -6218,7 +7477,7 @@ input IssueSetIterationInput { clientMutationId: String """ - The iid of the issue to mutate + The IID of the issue to mutate """ iid: String! @@ -6263,7 +7522,7 @@ input IssueSetLockedInput { clientMutationId: String """ - The iid of the issue to mutate + The IID of the issue to mutate """ iid: String! @@ -6299,6 +7558,51 @@ type IssueSetLockedPayload { } """ +Autogenerated input type of IssueSetSubscription +""" +input IssueSetSubscriptionInput { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + The IID of the issue to mutate + """ + iid: String! + + """ + The project the issue to mutate is in + """ + projectPath: ID! + + """ + The desired state of the subscription + """ + subscribedState: Boolean! +} + +""" +Autogenerated return type of IssueSetSubscription +""" +type IssueSetSubscriptionPayload { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Errors encountered during execution of the mutation. + """ + errors: [String!]! + + """ + The issue after mutation + """ + issue: Issue +} + +""" Autogenerated input type of IssueSetWeight """ input IssueSetWeightInput { @@ -6308,7 +7612,7 @@ input IssueSetWeightInput { clientMutationId: String """ - The iid of the issue to mutate + The IID of the issue to mutate """ iid: String! @@ -6427,12 +7731,48 @@ enum IssueSort { State of a GitLab issue """ enum IssueState { + all closed locked opened } """ +Represents total number of issues for the represented statuses. +""" +type IssueStatusCountsType { + """ + Number of issues with status ALL for the project + """ + all: Int + + """ + Number of issues with status CLOSED for the project + """ + closed: Int + + """ + Number of issues with status OPENED for the project + """ + opened: Int +} + +""" +Issue type +""" +enum IssueType { + """ + Incident issue type + """ + INCIDENT + + """ + Issue issue type + """ + ISSUE +} + +""" Represents an iteration object. """ type Iteration { @@ -6447,6 +7787,11 @@ type Iteration { description: String """ + The GitLab Flavored Markdown rendering of `description` + """ + descriptionHtml: String + + """ Timestamp of the iteration due date """ dueDate: Time @@ -6462,6 +7807,16 @@ type Iteration { iid: ID! """ + Web path of the iteration, scoped to the query parent. Only valid for Project parents. Returns null in other contexts + """ + scopedPath: String + + """ + Web URL of the iteration, scoped to the query parent. Only valid for Project parents. Returns null in other contexts + """ + scopedUrl: String + + """ Timestamp of the iteration start date """ startDate: Time @@ -6528,6 +7883,11 @@ type IterationEdge { } """ +Identifier of Iteration +""" +scalar IterationID + +""" State of a GitLab iteration """ enum IterationState { @@ -6911,6 +8271,11 @@ type LabelEdge { } """ +Identifier of Label +""" +scalar LabelID + +""" List limit metric setting """ enum ListLimitMetric { @@ -6988,6 +8353,31 @@ type MergeRequest implements Noteable { allowCollaboration: Boolean """ + Users who approved the merge request + """ + approvedBy( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + ): UserConnection + + """ Assignees of the merge request """ assignees( @@ -7018,6 +8408,11 @@ type MergeRequest implements Noteable { author: User """ + Number of commits in the merge request + """ + commitCount: Int + + """ Timestamp of when the merge request was created """ createdAt: Time! @@ -7433,6 +8828,11 @@ The connection type for MergeRequest. """ type MergeRequestConnection { """ + Total count of collection + """ + count: Int! + + """ A list of edges. """ edges: [MergeRequestEdge] @@ -7463,6 +8863,11 @@ input MergeRequestCreateInput { description: String """ + Labels of the merge request + """ + labels: [String!] + + """ Project full path the merge request is associated with """ projectPath: ID! @@ -7568,7 +8973,7 @@ Autogenerated input type of MergeRequestSetAssignees """ input MergeRequestSetAssigneesInput { """ - The usernames to assign to the merge request. Replaces existing assignees by default. + The usernames to assign to the resource. Replaces existing assignees by default. """ assigneeUsernames: [String!]! @@ -7847,6 +9252,7 @@ type MergeRequestSetWipPayload { State of a GitLab merge request """ enum MergeRequestState { + all closed locked merged @@ -8134,6 +9540,11 @@ type MilestoneEdge { node: Milestone } +""" +Identifier of Milestone +""" +scalar MilestoneID + enum MilestoneStateEnum { active closed @@ -8178,11 +9589,14 @@ type Mutation { awardEmojiAdd(input: AwardEmojiAddInput!): AwardEmojiAddPayload awardEmojiRemove(input: AwardEmojiRemoveInput!): AwardEmojiRemovePayload awardEmojiToggle(input: AwardEmojiToggleInput!): AwardEmojiTogglePayload + boardListCreate(input: BoardListCreateInput!): BoardListCreatePayload boardListUpdateLimitMetrics(input: BoardListUpdateLimitMetricsInput!): BoardListUpdateLimitMetricsPayload commitCreate(input: CommitCreateInput!): CommitCreatePayload + configureSast(input: ConfigureSastInput!): ConfigureSastPayload createAlertIssue(input: CreateAlertIssueInput!): CreateAlertIssuePayload createAnnotation(input: CreateAnnotationInput!): CreateAnnotationPayload createBranch(input: CreateBranchInput!): CreateBranchPayload + createClusterAgent(input: CreateClusterAgentInput!): CreateClusterAgentPayload createDiffNote(input: CreateDiffNoteInput!): CreateDiffNotePayload createEpic(input: CreateEpicInput!): CreateEpicPayload createImageDiffNote(input: CreateImageDiffNoteInput!): CreateImageDiffNotePayload @@ -8190,9 +9604,14 @@ type Mutation { createNote(input: CreateNoteInput!): CreateNotePayload createRequirement(input: CreateRequirementInput!): CreateRequirementPayload createSnippet(input: CreateSnippetInput!): CreateSnippetPayload + dastOnDemandScanCreate(input: DastOnDemandScanCreateInput!): DastOnDemandScanCreatePayload + dastScannerProfileCreate(input: DastScannerProfileCreateInput!): DastScannerProfileCreatePayload dastSiteProfileCreate(input: DastSiteProfileCreateInput!): DastSiteProfileCreatePayload + dastSiteProfileDelete(input: DastSiteProfileDeleteInput!): DastSiteProfileDeletePayload + dastSiteProfileUpdate(input: DastSiteProfileUpdateInput!): DastSiteProfileUpdatePayload deleteAnnotation(input: DeleteAnnotationInput!): DeleteAnnotationPayload designManagementDelete(input: DesignManagementDeleteInput!): DesignManagementDeletePayload + designManagementMove(input: DesignManagementMoveInput!): DesignManagementMovePayload designManagementUpload(input: DesignManagementUploadInput!): DesignManagementUploadPayload destroyNote(input: DestroyNoteInput!): DestroyNotePayload destroySnippet(input: DestroySnippetInput!): DestroySnippetPayload @@ -8205,10 +9624,14 @@ type Mutation { epicAddIssue(input: EpicAddIssueInput!): EpicAddIssuePayload epicSetSubscription(input: EpicSetSubscriptionInput!): EpicSetSubscriptionPayload epicTreeReorder(input: EpicTreeReorderInput!): EpicTreeReorderPayload + issueMoveList(input: IssueMoveListInput!): IssueMoveListPayload + issueSetAssignees(input: IssueSetAssigneesInput!): IssueSetAssigneesPayload issueSetConfidential(input: IssueSetConfidentialInput!): IssueSetConfidentialPayload issueSetDueDate(input: IssueSetDueDateInput!): IssueSetDueDatePayload + issueSetEpic(input: IssueSetEpicInput!): IssueSetEpicPayload issueSetIteration(input: IssueSetIterationInput!): IssueSetIterationPayload issueSetLocked(input: IssueSetLockedInput!): IssueSetLockedPayload + issueSetSubscription(input: IssueSetSubscriptionInput!): IssueSetSubscriptionPayload issueSetWeight(input: IssueSetWeightInput!): IssueSetWeightPayload jiraImportStart(input: JiraImportStartInput!): JiraImportStartPayload jiraImportUsers(input: JiraImportUsersInput!): JiraImportUsersPayload @@ -8225,6 +9648,7 @@ type Mutation { Update attributes of a merge request """ mergeRequestUpdate(input: MergeRequestUpdateInput!): MergeRequestUpdatePayload + namespaceIncreaseStorageTemporarily(input: NamespaceIncreaseStorageTemporarilyInput!): NamespaceIncreaseStorageTemporarilyPayload removeAwardEmoji(input: RemoveAwardEmojiInput!): RemoveAwardEmojiPayload @deprecated(reason: "Use awardEmojiRemove. Deprecated in 13.2") removeProjectFromSecurityDashboard(input: RemoveProjectFromSecurityDashboardInput!): RemoveProjectFromSecurityDashboardPayload runDastScan(input: RunDASTScanInput!): RunDASTScanPayload @@ -8234,6 +9658,8 @@ type Mutation { todosMarkAllDone(input: TodosMarkAllDoneInput!): TodosMarkAllDonePayload toggleAwardEmoji(input: ToggleAwardEmojiInput!): ToggleAwardEmojiPayload @deprecated(reason: "Use awardEmojiToggle. Deprecated in 13.2") updateAlertStatus(input: UpdateAlertStatusInput!): UpdateAlertStatusPayload + updateBoard(input: UpdateBoardInput!): UpdateBoardPayload + updateBoardList(input: UpdateBoardListInput!): UpdateBoardListPayload updateContainerExpirationPolicy(input: UpdateContainerExpirationPolicyInput!): UpdateContainerExpirationPolicyPayload updateEpic(input: UpdateEpicInput!): UpdateEpicPayload @@ -8302,6 +9728,11 @@ type Namespace { id: ID! """ + Status of the temporary storage increase + """ + isTemporaryStorageIncreaseEnabled: Boolean! + + """ Indicates if Large File Storage (LFS) is enabled for namespace """ lfsEnabled: Boolean @@ -8412,6 +9843,83 @@ type NamespaceEdge { node: Namespace } +""" +Autogenerated input type of NamespaceIncreaseStorageTemporarily +""" +input NamespaceIncreaseStorageTemporarilyInput { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + The global id of the namespace to mutate + """ + id: ID! +} + +""" +Autogenerated return type of NamespaceIncreaseStorageTemporarily +""" +type NamespaceIncreaseStorageTemporarilyPayload { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Errors encountered during execution of the mutation. + """ + errors: [String!]! + + """ + The namespace after mutation + """ + namespace: Namespace +} + +input NegatedBoardEpicIssueInput { + """ + Filter by assignee username + """ + assigneeUsername: [String] + + """ + Filter by author username + """ + authorUsername: String + + """ + Filter by epic ID + """ + epicId: String + + """ + Filter by label name + """ + labelName: [String] + + """ + Filter by milestone title + """ + milestoneTitle: String + + """ + Filter by reaction emoji + """ + myReactionEmoji: String + + """ + Filter by release tag + """ + releaseTag: String + + """ + Filter by weight + """ + weight: String +} + type Note implements ResolvableInterface { """ User who wrote this note @@ -8832,6 +10340,13 @@ type Pipeline { committedAt: Time """ + Config source of the pipeline (UNKNOWN_SOURCE, REPOSITORY_SOURCE, + AUTO_DEVOPS_SOURCE, WEBIDE_SOURCE, REMOTE_SOURCE, EXTERNAL_PROJECT_SOURCE, + BRIDGE_SOURCE, PARAMETER_SOURCE) + """ + configSource: PipelineConfigSourceEnum + + """ Coverage percentage """ coverage: Float @@ -8877,6 +10392,31 @@ type Pipeline { sha: String! """ + Stages of the pipeline + """ + stages( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + ): CiStageConnection + + """ Timestamp when the pipeline was started """ startedAt: Time @@ -8893,16 +10433,37 @@ type Pipeline { updatedAt: Time! """ + Pipeline user + """ + user: User + + """ Permissions for the current user on the resource """ userPermissions: PipelinePermissions! } +enum PipelineConfigSourceEnum { + AUTO_DEVOPS_SOURCE + BRIDGE_SOURCE + EXTERNAL_PROJECT_SOURCE + PARAMETER_SOURCE + REMOTE_SOURCE + REPOSITORY_SOURCE + UNKNOWN_SOURCE + WEBIDE_SOURCE +} + """ The connection type for Pipeline. """ type PipelineConnection { """ + Total count of collection + """ + count: Int! + + """ A list of edges. """ edges: [PipelineEdge] @@ -9147,6 +10708,56 @@ type Project { createdAt: Time """ + The DAST scanner profiles associated with the project + """ + dastScannerProfiles( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + ): DastScannerProfileConnection + + """ + DAST Site Profiles associated with the project + """ + dastSiteProfiles( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + ): DastSiteProfileConnection + + """ Short description of the project """ description: String @@ -9157,6 +10768,26 @@ type Project { descriptionHtml: String """ + A single environment of the project + """ + environment( + """ + Name of the environment + """ + name: String + + """ + Search query for environment name + """ + search: String + + """ + States of environments that should be included in result + """ + states: [String!] + ): Environment + + """ Environments of the project """ environments( @@ -9241,7 +10872,7 @@ type Project { assigneeId: String """ - Username of a user assigned to the issues + Username of a user assigned to the issue """ assigneeUsername: String @@ -9286,7 +10917,7 @@ type Project { labelName: [String] """ - Milestones applied to this issue + Milestone applied to this issue """ milestoneTitle: [String] @@ -9306,6 +10937,11 @@ type Project { state: IssuableState """ + Filter issues by the given issue types + """ + types: [IssueType!] + + """ Issues updated after this date """ updatedAfter: Time @@ -9317,6 +10953,81 @@ type Project { ): Issue """ + Counts of issues by status for the project + """ + issueStatusCounts( + """ + ID of a user assigned to the issues, "none" and "any" values supported + """ + assigneeId: String + + """ + Username of a user assigned to the issue + """ + assigneeUsername: String + + """ + Issues closed after this date + """ + closedAfter: Time + + """ + Issues closed before this date + """ + closedBefore: Time + + """ + Issues created after this date + """ + createdAfter: Time + + """ + Issues created before this date + """ + createdBefore: Time + + """ + IID of the issue. For example, "1" + """ + iid: String + + """ + List of IIDs of issues. For example, [1, 2] + """ + iids: [String!] + + """ + Labels applied to this issue + """ + labelName: [String] + + """ + Milestone applied to this issue + """ + milestoneTitle: [String] + + """ + Search query for issue title or description + """ + search: String + + """ + Filter issues by the given issue types + """ + types: [IssueType!] + + """ + Issues updated after this date + """ + updatedAfter: Time + + """ + Issues updated before this date + """ + updatedBefore: Time + ): IssueStatusCountsType + + """ Issues of the project """ issues( @@ -9331,7 +11042,7 @@ type Project { assigneeId: String """ - Username of a user assigned to the issues + Username of a user assigned to the issue """ assigneeUsername: String @@ -9391,7 +11102,7 @@ type Project { last: Int """ - Milestones applied to this issue + Milestone applied to this issue """ milestoneTitle: [String] @@ -9411,6 +11122,11 @@ type Project { state: IssuableState """ + Filter issues by the given issue types + """ + types: [IssueType!] + + """ Issues updated after this date """ updatedAfter: Time @@ -9427,6 +11143,68 @@ type Project { issuesEnabled: Boolean """ + Find iterations + """ + iterations( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + List items within a time frame where items.end_date is between startDate and + endDate parameters (startDate parameter must be present) + """ + endDate: Time + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + The ID of the Iteration to look up + """ + id: ID + + """ + The internal ID of the Iteration to look up + """ + iid: ID + + """ + Whether to include ancestor iterations. Defaults to true + """ + includeAncestors: Boolean + + """ + Returns the last _n_ elements from the list. + """ + last: Int + + """ + List items within a time frame where items.start_date is between startDate + and endDate parameters (endDate parameter must be present) + """ + startDate: Time + + """ + Filter iterations by state + """ + state: IterationState + + """ + Fuzzy search by title + """ + title: String + ): IterationConnection + + """ Status of Jira import background job of the project """ jiraImportStatus: String @@ -9556,6 +11334,16 @@ type Project { last: Int """ + Merge requests merged after this date + """ + mergedAfter: Time + + """ + Merge requests merged before this date + """ + mergedBefore: Time + + """ Array of source branch names. All resolved merge requests will have one of these branches as their source. """ sourceBranches: [String!] @@ -9584,6 +11372,58 @@ type Project { mergeRequestsFfOnlyEnabled: Boolean """ + Milestones of the project + """ + milestones( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + List items within a time frame where items.end_date is between startDate and + endDate parameters (startDate parameter must be present) + """ + endDate: Time + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Array of global milestone IDs, e.g., "gid://gitlab/Milestone/1" + """ + ids: [ID!] + + """ + Also return milestones in the project's parent group and its ancestors + """ + includeAncestors: Boolean + + """ + Returns the last _n_ elements from the list. + """ + last: Int + + """ + List items within a time frame where items.start_date is between startDate + and endDate parameters (endDate parameter must be present) + """ + startDate: Time + + """ + Filter milestones by state + """ + state: MilestoneStateEnum + ): MilestoneConnection + + """ Name of the project (without namespace) """ name: String! @@ -9885,6 +11725,11 @@ type Project { sastCiConfiguration: SastCiConfiguration """ + Path to project's security dashboard + """ + securityDashboardPath: String + + """ Information about security analyzers used in the project """ securityScanners: SecurityScanners @@ -10505,6 +12350,21 @@ type ProjectStatistics { wikiSize: Float } +""" +The alert condition for Prometheus +""" +type PrometheusAlert { + """ + The human-readable text of the alert condition + """ + humanizedText: String! + + """ + ID of the alert condition + """ + id: ID! +} + type Query { """ Get information about current user @@ -10552,11 +12412,31 @@ type Query { instanceSecurityDashboard: InstanceSecurityDashboard """ + Find an iteration + """ + iteration( + """ + Find an iteration by its ID + """ + id: IterationID! + ): Iteration + + """ Metadata about GitLab """ metadata: Metadata """ + Find a milestone + """ + milestone( + """ + Find a milestone by its ID + """ + id: MilestoneID! + ): Milestone + + """ Find a namespace """ namespace( @@ -10772,7 +12652,44 @@ type Query { ): VulnerabilityConnection """ - Number of vulnerabilities per severity level, per day, for the projects on the current user's instance security dashboard + Number of vulnerabilities per day for the projects on the current user's instance security dashboard + """ + vulnerabilitiesCountByDay( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Last day for which to fetch vulnerability history + """ + endDate: ISO8601Date! + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + + """ + First day for which to fetch vulnerability history + """ + startDate: ISO8601Date! + ): VulnerabilitiesCountByDayConnection + + """ + Number of vulnerabilities per severity level, per day, for the projects on the + current user's instance security dashboard. Deprecated in 13.3: Use + `vulnerabilitiesCountByDay` """ vulnerabilitiesCountByDayAndSeverity( """ @@ -10804,7 +12721,7 @@ type Query { First day for which to fetch vulnerability history """ startDate: ISO8601Date! - ): VulnerabilitiesCountByDayAndSeverityConnection + ): VulnerabilitiesCountByDayAndSeverityConnection @deprecated(reason: "Use `vulnerabilitiesCountByDay`. Deprecated in 13.3") } """ @@ -11833,6 +13750,11 @@ type SastCiConfigurationEntity { ): SastCiConfigurationOptionsEntityConnection """ + Size of the UI component. + """ + size: SastUiComponentSize + + """ Type of the field value. """ type: String @@ -11929,6 +13851,15 @@ type SastCiConfigurationOptionsEntityEdge { } """ +Size of UI component in SAST configuration page +""" +enum SastUiComponentSize { + LARGE + MEDIUM + SMALL +} + +""" Represents a resource scanned by a security scan """ type ScannedResource { @@ -12063,6 +13994,7 @@ The type of the security scanner. """ enum SecurityScannerType { CONTAINER_SCANNING + COVERAGE_FUZZING DAST DEPENDENCY_SCANNING SAST @@ -12636,9 +14568,9 @@ type Snippet implements Noteable { author: User """ - Snippet blob + Snippet blob. Deprecated in 13.3: Use `blobs` """ - blob: SnippetBlob! + blob: SnippetBlob! @deprecated(reason: "Use `blobs`. Deprecated in 13.3") """ Snippet blobs @@ -12832,6 +14764,41 @@ type SnippetBlob { } """ +Type of a snippet blob input action +""" +enum SnippetBlobActionEnum { + create + delete + move + update +} + +""" +Represents an action to perform over a snippet file +""" +input SnippetBlobActionInputType { + """ + Type of input action + """ + action: SnippetBlobActionEnum! + + """ + Snippet file content + """ + content: String + + """ + Path of the snippet file + """ + filePath: String! + + """ + Previous path of the snippet file + """ + previousPath: String +} + +""" Represents how the blob content should be displayed """ type SnippetBlobViewer { @@ -12906,41 +14873,6 @@ type SnippetEdge { node: Snippet } -""" -Type of a snippet file input action -""" -enum SnippetFileInputActionEnum { - create - delete - move - update -} - -""" -Represents an action to perform over a snippet file -""" -input SnippetFileInputType { - """ - Type of input action - """ - action: SnippetFileInputActionEnum! - - """ - Snippet file content - """ - content: String - - """ - Path of the snippet file - """ - filePath: String! - - """ - Previous path of the snippet file - """ - previousPath: String -} - type SnippetPermissions { """ Indicates the user can perform `admin_snippet` on this resource @@ -13664,6 +15596,11 @@ type TreeEntry implements Entry { type: EntryType! """ + Web path for the tree entry (directory) + """ + webPath: String + + """ Web URL for the tree entry (directory) """ webUrl: String @@ -13770,6 +15707,116 @@ type UpdateAlertStatusPayload { } """ +Autogenerated input type of UpdateBoard +""" +input UpdateBoardInput { + """ + The id of user to be assigned to the board. + """ + assigneeId: ID + + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Whether or not backlog list is hidden. + """ + hideBacklogList: Boolean + + """ + Whether or not closed list is hidden. + """ + hideClosedList: Boolean + + """ + The board global id. + """ + id: ID! + + """ + The id of milestone to be assigned to the board. + """ + milestoneId: ID + + """ + Name of the board + """ + name: String + + """ + The weight value to be assigned to the board. + """ + weight: Int +} + +""" +Autogenerated input type of UpdateBoardList +""" +input UpdateBoardListInput { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Indicates if list is collapsed for this user + """ + collapsed: Boolean + + """ + Global ID of the list. + """ + listId: ID! + + """ + Position of list within the board + """ + position: Int +} + +""" +Autogenerated return type of UpdateBoardList +""" +type UpdateBoardListPayload { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Errors encountered during execution of the mutation. + """ + errors: [String!]! + + """ + Mutated list + """ + list: BoardList +} + +""" +Autogenerated return type of UpdateBoard +""" +type UpdateBoardPayload { + """ + The board after mutation. + """ + board: Board + + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Errors encountered during execution of the mutation. + """ + errors: [String!]! +} + +""" Autogenerated input type of UpdateContainerExpirationPolicy """ input UpdateContainerExpirationPolicyInput { @@ -13996,6 +16043,11 @@ Autogenerated input type of UpdateIssue """ input UpdateIssueInput { """ + The IDs of labels to be added to the issue. + """ + addLabelIds: [ID!] + + """ A unique identifier for the client performing the mutation. """ clientMutationId: String @@ -14021,16 +16073,31 @@ input UpdateIssueInput { healthStatus: HealthStatus """ - The iid of the issue to mutate + The IID of the issue to mutate """ iid: String! """ + Indicates discussion is locked on the issue + """ + locked: Boolean + + """ + The ID of the milestone to be assigned, milestone will be removed if set to null. + """ + milestoneId: ID + + """ The project the issue to mutate is in """ projectPath: ID! """ + The IDs of labels to be removed from the issue. + """ + removeLabelIds: [ID!] + + """ Title of the issue """ title: String @@ -14123,7 +16190,7 @@ input UpdateNoteInput { """ Content of the note """ - body: String! + body: String """ A unique identifier for the client performing the mutation. @@ -14131,6 +16198,11 @@ input UpdateNoteInput { clientMutationId: String """ + The confidentiality flag of a note. Default is false. + """ + confidential: Boolean + + """ The global id of the note to update """ id: ID! @@ -14211,6 +16283,11 @@ Autogenerated input type of UpdateSnippet """ input UpdateSnippetInput { """ + Actions to perform over the snippet repository and blobs + """ + blobActions: [SnippetBlobActionInputType!] + + """ A unique identifier for the client performing the mutation. """ clientMutationId: String @@ -14231,11 +16308,6 @@ input UpdateSnippetInput { fileName: String """ - The snippet files to update - """ - files: [SnippetFileInputType!] - - """ The global id of the snippet to update """ id: ID! @@ -14309,6 +16381,16 @@ type User { last: Int """ + Merge requests merged after this date + """ + mergedAfter: Time + + """ + Merge requests merged before this date + """ + mergedBefore: Time + + """ The global ID of the project the authored merge requests should be in. Incompatible with projectPath. """ projectId: ID @@ -14369,6 +16451,16 @@ type User { last: Int """ + Merge requests merged after this date + """ + mergedAfter: Time + + """ + Merge requests merged before this date + """ + mergedBefore: Time + + """ The global ID of the project the authored merge requests should be in. Incompatible with projectPath. """ projectId: ID @@ -14400,6 +16492,11 @@ type User { avatarUrl: String """ + User email + """ + email: String + + """ Group memberships of the user """ groupMemberships( @@ -14505,6 +16602,11 @@ type User { state: UserState! """ + User status + """ + status: UserStatus + + """ Todos of the user """ todos( @@ -14570,6 +16672,11 @@ type User { username: String! """ + Web path of the user + """ + webPath: String! + + """ Web URL of the user """ webUrl: String! @@ -14637,6 +16744,23 @@ enum UserState { deactivated } +type UserStatus { + """ + String representation of emoji + """ + emoji: String + + """ + User status message + """ + message: String + + """ + HTML of the user status message + """ + messageHtml: String +} + enum VisibilityLevelsEnum { internal private @@ -14650,6 +16774,51 @@ enum VisibilityScopesEnum { } """ +Represents the count of vulnerabilities by severity on a particular day +""" +type VulnerabilitiesCountByDay { + """ + Total number of vulnerabilities on a particular day with critical severity + """ + critical: Int! + + """ + Date for the count + """ + date: ISO8601Date! + + """ + Total number of vulnerabilities on a particular day with high severity + """ + high: Int! + + """ + Total number of vulnerabilities on a particular day with info severity + """ + info: Int! + + """ + Total number of vulnerabilities on a particular day with low severity + """ + low: Int! + + """ + Total number of vulnerabilities on a particular day with medium severity + """ + medium: Int! + + """ + Total number of vulnerabilities on a particular day + """ + total: Int! + + """ + Total number of vulnerabilities on a particular day with unknown severity + """ + unknown: Int! +} + +""" Represents the number of vulnerabilities for a particular severity on a particular day """ type VulnerabilitiesCountByDayAndSeverity { @@ -14705,6 +16874,41 @@ type VulnerabilitiesCountByDayAndSeverityEdge { } """ +The connection type for VulnerabilitiesCountByDay. +""" +type VulnerabilitiesCountByDayConnection { + """ + A list of edges. + """ + edges: [VulnerabilitiesCountByDayEdge] + + """ + A list of nodes. + """ + nodes: [VulnerabilitiesCountByDay] + + """ + Information to aid in pagination. + """ + pageInfo: PageInfo! +} + +""" +An edge in a connection. +""" +type VulnerabilitiesCountByDayEdge { + """ + A cursor for use in pagination. + """ + cursor: String! + + """ + The item at the end of the edge. + """ + node: VulnerabilitiesCountByDay +} + +""" Represents a vulnerability. """ type Vulnerability { @@ -14776,6 +16980,11 @@ type Vulnerability { reportType: VulnerabilityReportType """ + Indicates whether the vulnerability is fixed on the default branch or not + """ + resolvedOnDefaultBranch: Boolean! + + """ Scanner metadata for the vulnerability. """ scanner: VulnerabilityScanner @@ -14847,6 +17056,17 @@ type VulnerabilityEdge { } """ +The grade of the vulnerable project +""" +enum VulnerabilityGrade { + A + B + C + D + F +} + +""" Represents a vulnerability identifier. """ type VulnerabilityIdentifier { @@ -14937,7 +17157,7 @@ enum VulnerabilityIssueLinkType { """ Represents a vulnerability location. The fields with data will depend on the vulnerability report type """ -union VulnerabilityLocation = VulnerabilityLocationContainerScanning | VulnerabilityLocationDast | VulnerabilityLocationDependencyScanning | VulnerabilityLocationSast | VulnerabilityLocationSecretDetection +union VulnerabilityLocation = VulnerabilityLocationContainerScanning | VulnerabilityLocationCoverageFuzzing | VulnerabilityLocationDast | VulnerabilityLocationDependencyScanning | VulnerabilityLocationSast | VulnerabilityLocationSecretDetection """ Represents the location of a vulnerability found by a container security scan @@ -14960,6 +17180,36 @@ type VulnerabilityLocationContainerScanning { } """ +Represents the location of a vulnerability found by a Coverage Fuzzing scan +""" +type VulnerabilityLocationCoverageFuzzing { + """ + Number of the last relevant line in the vulnerable file + """ + endLine: String + + """ + Path to the vulnerable file + """ + file: String + + """ + Number of the first relevant line in the vulnerable file + """ + startLine: String + + """ + Class containing the vulnerability + """ + vulnerableClass: String + + """ + Method containing the vulnerability + """ + vulnerableMethod: String +} + +""" Represents the location of a vulnerability found by a DAST scan """ type VulnerabilityLocationDast { @@ -15256,4 +17506,44 @@ type VulnerablePackage { The name of the vulnerable package """ name: String +} + +""" +Represents vulnerability letter grades with associated projects +""" +type VulnerableProjectsByGrade { + """ + Number of projects within this grade + """ + count: Int! + + """ + Grade based on the highest severity vulnerability present + """ + grade: VulnerabilityGrade! + + """ + Projects within this grade + """ + projects( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + ): ProjectConnection! }
\ No newline at end of file diff --git a/doc/api/graphql/reference/gitlab_schema.json b/doc/api/graphql/reference/gitlab_schema.json index 80aaa4aa949..7ee37fb4d43 100644 --- a/doc/api/graphql/reference/gitlab_schema.json +++ b/doc/api/graphql/reference/gitlab_schema.json @@ -578,6 +578,24 @@ "deprecationReason": null }, { + "name": "detailsUrl", + "description": "The URL of the alert detail page", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "discussions", "description": "All discussions on this noteable", "args": [ @@ -802,6 +820,34 @@ "deprecationReason": null }, { + "name": "prometheusAlert", + "description": "The alert condition for Prometheus", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "PrometheusAlert", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "runbook", + "description": "Runbook for the alert as defined in alert details", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "service", "description": "Service the alert came from", "args": [ @@ -872,6 +918,167 @@ "deprecationReason": null }, { + "name": "todos", + "description": "Todos of the current user for the alert", + "args": [ + { + "name": "action", + "description": "The action to be filtered", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "ENUM", + "name": "TodoActionEnum", + "ofType": null + } + } + }, + "defaultValue": null + }, + { + "name": "authorId", + "description": "The ID of an author", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + } + }, + "defaultValue": null + }, + { + "name": "projectId", + "description": "The ID of a project", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + } + }, + "defaultValue": null + }, + { + "name": "groupId", + "description": "The ID of a group", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + } + }, + "defaultValue": null + }, + { + "name": "state", + "description": "The state of the todo", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "ENUM", + "name": "TodoStateEnum", + "ofType": null + } + } + }, + "defaultValue": null + }, + { + "name": "type", + "description": "The type of the todo", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "ENUM", + "name": "TodoTargetEnum", + "ofType": null + } + } + }, + "defaultValue": null + }, + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "TodoConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "updatedAt", "description": "Timestamp the alert was last updated", "args": [ @@ -2301,6 +2508,20 @@ "deprecationReason": null }, { + "name": "webPath", + "description": "Web path of the blob", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "webUrl", "description": "Web URL of the blob", "args": [ @@ -2473,6 +2694,111 @@ "description": "Represents a project or group board", "fields": [ { + "name": "assignee", + "description": "The board assignee.", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "User", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "epics", + "description": "Epics associated with board issues.", + "args": [ + { + "name": "issueFilters", + "description": "Filters applied when selecting issues on the board", + "type": { + "kind": "INPUT_OBJECT", + "name": "BoardEpicIssueInput", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "EpicConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "hideBacklogList", + "description": "Whether or not backlog list is hidden.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "hideClosedList", + "description": "Whether or not closed list is hidden.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "id", "description": "ID (global ID) of the board", "args": [ @@ -2495,6 +2821,16 @@ "description": "Lists of the board", "args": [ { + "name": "id", + "description": "Find a list by its global ID", + "type": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + }, + "defaultValue": null + }, + { "name": "after", "description": "Returns the elements in the list that come after the specified cursor.", "type": { @@ -2544,6 +2880,20 @@ "deprecationReason": null }, { + "name": "milestone", + "description": "The board milestone.", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "Milestone", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "name", "description": "Name of the board", "args": [ @@ -2559,7 +2909,7 @@ }, { "name": "weight", - "description": "Weight of the board", + "description": "Weight of the board.", "args": [ ], @@ -2692,6 +3042,125 @@ "possibleTypes": null }, { + "kind": "INPUT_OBJECT", + "name": "BoardEpicIssueInput", + "description": null, + "fields": null, + "inputFields": [ + { + "name": "labelName", + "description": "Filter by label name", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "milestoneTitle", + "description": "Filter by milestone title", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "assigneeUsername", + "description": "Filter by assignee username", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "authorUsername", + "description": "Filter by author username", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "releaseTag", + "description": "Filter by release tag", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "epicId", + "description": "Filter by epic ID", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "myReactionEmoji", + "description": "Filter by reaction emoji", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "weight", + "description": "Filter by weight", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "not", + "description": "List of negated params. Warning: this argument is experimental and a subject to change in future", + "type": { + "kind": "INPUT_OBJECT", + "name": "NegatedBoardEpicIssueInput", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "SCALAR", + "name": "BoardID", + "description": "Identifier of Board", + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { "kind": "OBJECT", "name": "BoardList", "description": "Represents a list for an issue board", @@ -2743,6 +3212,73 @@ "deprecationReason": null }, { + "name": "issues", + "description": "Board issues", + "args": [ + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "IssueConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "issuesCount", + "description": "Count of issues in the list", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "label", "description": "Label of the list", "args": [ @@ -2861,6 +3397,20 @@ }, "isDeprecated": false, "deprecationReason": null + }, + { + "name": "totalWeight", + "description": "Total weight of all issues in the list", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null } ], "inputFields": null, @@ -2938,6 +3488,128 @@ "possibleTypes": null }, { + "kind": "INPUT_OBJECT", + "name": "BoardListCreateInput", + "description": "Autogenerated input type of BoardListCreate", + "fields": null, + "inputFields": [ + { + "name": "boardId", + "description": "The Global ID of the issue board to mutate", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "BoardID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "backlog", + "description": "Create the backlog list", + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "labelId", + "description": "ID of an existing label", + "type": { + "kind": "SCALAR", + "name": "LabelID", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "BoardListCreatePayload", + "description": "Autogenerated return type of BoardListCreate", + "fields": [ + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "errors", + "description": "Errors encountered during execution of the mutation.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "list", + "description": "List of the issue board", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "BoardList", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { "kind": "OBJECT", "name": "BoardListEdge", "description": "An edge in a connection.", @@ -3171,6 +3843,683 @@ }, { "kind": "OBJECT", + "name": "CiGroup", + "description": null, + "fields": [ + { + "name": "jobs", + "description": "Jobs in group", + "args": [ + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "CiJobConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "name", + "description": "Name of the job group", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "size", + "description": "Size of the group", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "CiGroupConnection", + "description": "The connection type for CiGroup.", + "fields": [ + { + "name": "edges", + "description": "A list of edges.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "CiGroupEdge", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "nodes", + "description": "A list of nodes.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "CiGroup", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "pageInfo", + "description": "Information to aid in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "PageInfo", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "CiGroupEdge", + "description": "An edge in a connection.", + "fields": [ + { + "name": "cursor", + "description": "A cursor for use in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "node", + "description": "The item at the end of the edge.", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "CiGroup", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "CiJob", + "description": null, + "fields": [ + { + "name": "name", + "description": "Name of the job", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "needs", + "description": "Builds that must complete before the jobs run", + "args": [ + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "CiJobConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "CiJobConnection", + "description": "The connection type for CiJob.", + "fields": [ + { + "name": "edges", + "description": "A list of edges.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "CiJobEdge", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "nodes", + "description": "A list of nodes.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "CiJob", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "pageInfo", + "description": "Information to aid in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "PageInfo", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "CiJobEdge", + "description": "An edge in a connection.", + "fields": [ + { + "name": "cursor", + "description": "A cursor for use in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "node", + "description": "The item at the end of the edge.", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "CiJob", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "CiStage", + "description": null, + "fields": [ + { + "name": "groups", + "description": "Group of jobs for the stage", + "args": [ + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "CiGroupConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "name", + "description": "Name of the stage", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "CiStageConnection", + "description": "The connection type for CiStage.", + "fields": [ + { + "name": "edges", + "description": "A list of edges.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "CiStageEdge", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "nodes", + "description": "A list of nodes.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "CiStage", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "pageInfo", + "description": "Information to aid in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "PageInfo", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "CiStageEdge", + "description": "An edge in a connection.", + "fields": [ + { + "name": "cursor", + "description": "A cursor for use in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "node", + "description": "The item at the end of the edge.", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "CiStage", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "ClusterAgent", + "description": null, + "fields": [ + { + "name": "createdAt", + "description": "Timestamp the cluster agent was created", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "id", + "description": "ID of the cluster agent", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "name", + "description": "Name of the cluster agent", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "project", + "description": "The project this cluster agent is associated with", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "Project", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "updatedAt", + "description": "Timestamp the cluster agent was updated", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", "name": "Commit", "description": null, "fields": [ @@ -3245,6 +4594,20 @@ "deprecationReason": null }, { + "name": "descriptionHtml", + "description": "The GitLab Flavored Markdown rendering of `description`", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "id", "description": "ID (global ID) of the commit", "args": [ @@ -3463,6 +4826,24 @@ "deprecationReason": null }, { + "name": "webPath", + "description": "Web path of the commit", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "webUrl", "description": "Web URL of the commit", "args": [ @@ -3937,6 +5318,122 @@ "possibleTypes": null }, { + "kind": "INPUT_OBJECT", + "name": "ConfigureSastInput", + "description": "Autogenerated input type of ConfigureSast", + "fields": null, + "inputFields": [ + { + "name": "projectPath", + "description": "Full path of the project.", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "configuration", + "description": "Payload containing SAST variable values (https://docs.gitlab.com/ee/user/application_security/sast/#available-variables).", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "JSON", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "ConfigureSastPayload", + "description": "Autogenerated return type of ConfigureSast", + "fields": [ + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "errors", + "description": "Errors encountered during execution of the mutation.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "result", + "description": "JSON containing the status of MR creation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "JSON", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { "kind": "OBJECT", "name": "ContainerExpirationPolicy", "description": "A tag expiration policy designed to keep only the images that matter most", @@ -4650,6 +6147,122 @@ }, { "kind": "INPUT_OBJECT", + "name": "CreateClusterAgentInput", + "description": "Autogenerated input type of CreateClusterAgent", + "fields": null, + "inputFields": [ + { + "name": "projectPath", + "description": "Full path of the associated project for this cluster agent", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "name", + "description": "Name of the cluster agent", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "CreateClusterAgentPayload", + "description": "Autogenerated return type of CreateClusterAgent", + "fields": [ + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "clusterAgent", + "description": "Cluster agent created after mutation", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "ClusterAgent", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "errors", + "description": "Errors encountered during execution of the mutation.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "INPUT_OBJECT", "name": "CreateDiffNoteInput", "description": "Autogenerated input type of CreateDiffNote", "fields": null, @@ -5146,13 +6759,19 @@ "name": "groupPath", "description": "The target group for the iteration", "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "ID", - "ofType": null - } + "kind": "SCALAR", + "name": "ID", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "projectPath", + "description": "The target project for the iteration", + "type": { + "kind": "SCALAR", + "name": "ID", + "ofType": null }, "defaultValue": null }, @@ -5623,8 +7242,8 @@ "defaultValue": null }, { - "name": "files", - "description": "The snippet files to create", + "name": "blobActions", + "description": "Actions to perform over the snippet repository and blobs", "type": { "kind": "LIST", "name": null, @@ -5633,7 +7252,7 @@ "name": null, "ofType": { "kind": "INPUT_OBJECT", - "name": "SnippetFileInputType", + "name": "SnippetBlobActionInputType", "ofType": null } } @@ -5723,6 +7342,122 @@ "possibleTypes": null }, { + "kind": "INPUT_OBJECT", + "name": "DastOnDemandScanCreateInput", + "description": "Autogenerated input type of DastOnDemandScanCreate", + "fields": null, + "inputFields": [ + { + "name": "fullPath", + "description": "The project the site profile belongs to.", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "dastSiteProfileId", + "description": "ID of the site profile to be used for the scan.", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "DastSiteProfileID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "DastOnDemandScanCreatePayload", + "description": "Autogenerated return type of DastOnDemandScanCreate", + "fields": [ + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "errors", + "description": "Errors encountered during execution of the mutation.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "pipelineUrl", + "description": "URL of the pipeline that was created.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { "kind": "ENUM", "name": "DastScanTypeEnum", "description": null, @@ -5740,6 +7475,499 @@ "possibleTypes": null }, { + "kind": "OBJECT", + "name": "DastScannerProfile", + "description": "Represents a DAST scanner profile.", + "fields": [ + { + "name": "id", + "description": "ID of the DAST scanner profile", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "profileName", + "description": "Name of the DAST scanner profile", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "spiderTimeout", + "description": "The maximum number of seconds allowed for the spider to traverse the site", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "targetTimeout", + "description": "The maximum number of seconds allowed for the site under test to respond to a request", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "DastScannerProfileConnection", + "description": "The connection type for DastScannerProfile.", + "fields": [ + { + "name": "edges", + "description": "A list of edges.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "DastScannerProfileEdge", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "nodes", + "description": "A list of nodes.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "DastScannerProfile", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "pageInfo", + "description": "Information to aid in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "PageInfo", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "INPUT_OBJECT", + "name": "DastScannerProfileCreateInput", + "description": "Autogenerated input type of DastScannerProfileCreate", + "fields": null, + "inputFields": [ + { + "name": "fullPath", + "description": "The project the scanner profile belongs to.", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "profileName", + "description": "The name of the scanner profile.", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "spiderTimeout", + "description": "The maximum number of seconds allowed for the spider to traverse the site.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "targetTimeout", + "description": "The maximum number of seconds allowed for the site under test to respond to a request.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "DastScannerProfileCreatePayload", + "description": "Autogenerated return type of DastScannerProfileCreate", + "fields": [ + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "errors", + "description": "Errors encountered during execution of the mutation.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "id", + "description": "ID of the scanner profile.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "DastScannerProfileEdge", + "description": "An edge in a connection.", + "fields": [ + { + "name": "cursor", + "description": "A cursor for use in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "node", + "description": "The item at the end of the edge.", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "DastScannerProfile", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "DastSiteProfile", + "description": "Represents a DAST Site Profile.", + "fields": [ + { + "name": "editPath", + "description": "Relative web path to the edit page of a site profile", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "id", + "description": "ID of the site profile", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "DastSiteProfileID", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "profileName", + "description": "The name of the site profile", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "targetUrl", + "description": "The URL of the target to be scanned", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "userPermissions", + "description": "Permissions for the current user on the resource", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "DastSiteProfilePermissions", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "validationStatus", + "description": "The current validation status of the site profile", + "args": [ + + ], + "type": { + "kind": "ENUM", + "name": "DastSiteProfileValidationStatusEnum", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "DastSiteProfileConnection", + "description": "The connection type for DastSiteProfile.", + "fields": [ + { + "name": "edges", + "description": "A list of edges.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "DastSiteProfileEdge", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "nodes", + "description": "A list of nodes.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "DastSiteProfile", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "pageInfo", + "description": "Information to aid in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "PageInfo", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { "kind": "INPUT_OBJECT", "name": "DastSiteProfileCreateInput", "description": "Autogenerated input type of DastSiteProfileCreate", @@ -5851,7 +8079,154 @@ ], "type": { "kind": "SCALAR", - "name": "ID", + "name": "DastSiteProfileID", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "INPUT_OBJECT", + "name": "DastSiteProfileDeleteInput", + "description": "Autogenerated input type of DastSiteProfileDelete", + "fields": null, + "inputFields": [ + { + "name": "fullPath", + "description": "The project the site profile belongs to.", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "id", + "description": "ID of the site profile to be deleted.", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "DastSiteProfileID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "DastSiteProfileDeletePayload", + "description": "Autogenerated return type of DastSiteProfileDelete", + "fields": [ + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "errors", + "description": "Errors encountered during execution of the mutation.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "DastSiteProfileEdge", + "description": "An edge in a connection.", + "fields": [ + { + "name": "cursor", + "description": "A cursor for use in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "node", + "description": "The item at the end of the edge.", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "DastSiteProfile", "ofType": null }, "isDeprecated": false, @@ -5866,6 +8241,222 @@ "possibleTypes": null }, { + "kind": "SCALAR", + "name": "DastSiteProfileID", + "description": "Identifier of DastSiteProfile", + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "DastSiteProfilePermissions", + "description": "Check permissions for the current user on site profile", + "fields": [ + { + "name": "createOnDemandDastScan", + "description": "Indicates the user can perform `create_on_demand_dast_scan` on this resource", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "INPUT_OBJECT", + "name": "DastSiteProfileUpdateInput", + "description": "Autogenerated input type of DastSiteProfileUpdate", + "fields": null, + "inputFields": [ + { + "name": "fullPath", + "description": "The project the site profile belongs to.", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "id", + "description": "ID of the site profile to be updated.", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "DastSiteProfileID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "profileName", + "description": "The name of the site profile.", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "targetUrl", + "description": "The URL of the target to be scanned.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "DastSiteProfileUpdatePayload", + "description": "Autogenerated return type of DastSiteProfileUpdate", + "fields": [ + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "errors", + "description": "Errors encountered during execution of the mutation.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "id", + "description": "ID of the site profile.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "DastSiteProfileID", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "ENUM", + "name": "DastSiteProfileValidationStatusEnum", + "description": null, + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": [ + { + "name": "PENDING_VALIDATION", + "description": "Site validation process has not started", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "INPROGRESS_VALIDATION", + "description": "Site validation process is in progress", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "PASSED_VALIDATION", + "description": "Site validation process finished successfully", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "FAILED_VALIDATION", + "description": "Site validation process finished but failed", + "isDeprecated": false, + "deprecationReason": null + } + ], + "possibleTypes": null + }, + { "kind": "INPUT_OBJECT", "name": "DeleteAnnotationInput", "description": "Autogenerated input type of DeleteAnnotation", @@ -7576,6 +10167,138 @@ "possibleTypes": null }, { + "kind": "SCALAR", + "name": "DesignManagementDesignID", + "description": "Identifier of DesignManagement::Design", + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "INPUT_OBJECT", + "name": "DesignManagementMoveInput", + "description": "Autogenerated input type of DesignManagementMove", + "fields": null, + "inputFields": [ + { + "name": "id", + "description": "ID of the design to move", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "DesignManagementDesignID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "previous", + "description": "ID of the immediately preceding design", + "type": { + "kind": "SCALAR", + "name": "DesignManagementDesignID", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "next", + "description": "ID of the immediately following design", + "type": { + "kind": "SCALAR", + "name": "DesignManagementDesignID", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "DesignManagementMovePayload", + "description": "Autogenerated return type of DesignManagementMove", + "fields": [ + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "designCollection", + "description": "The current state of the collection", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "DesignCollection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "errors", + "description": "Errors encountered during execution of the mutation.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { "kind": "INPUT_OBJECT", "name": "DesignManagementUploadInput", "description": "Autogenerated input type of DesignManagementUpload", @@ -9879,6 +12602,20 @@ "deprecationReason": null }, { + "name": "latestOpenedMostSevereAlert", + "description": "The most severe open alert for the environment. If multiple alerts have equal severity, the most recent is returned.", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "AlertManagementAlert", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "metricsDashboard", "description": "Metrics dashboard schema for the environment", "args": [ @@ -10195,6 +12932,16 @@ "defaultValue": null }, { + "name": "milestoneTitle", + "description": "Filter epics by milestone title, computed from epic's issues", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { "name": "iidStartsWith", "description": "Filter epics by iid for autocomplete", "type": { @@ -11115,7 +13862,7 @@ }, { "name": "projectPath", - "description": "The project the issue belongs to", + "description": "The full path of the project the issue belongs to", "type": { "kind": "NON_NULL", "name": null, @@ -11311,7 +14058,7 @@ "fields": [ { "name": "closedEpics", - "description": "Number of closed sub-epics", + "description": "Number of closed child epics", "args": [ ], @@ -11339,7 +14086,7 @@ }, { "name": "openedEpics", - "description": "Number of opened sub-epics", + "description": "Number of opened child epics", "args": [ ], @@ -11591,6 +14338,24 @@ "deprecationReason": null }, { + "name": "blocked", + "description": "Indicates the issue is blocked", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "closedAt", "description": "Timestamp of when the issue was closed", "args": [ @@ -12146,6 +14911,20 @@ "deprecationReason": null }, { + "name": "statusPagePublishedIncident", + "description": "Indicates whether an issue is published to the status page", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "subscribed", "description": "Indicates the currently logged in user is subscribed to the issue", "args": [ @@ -12250,6 +15029,20 @@ "deprecationReason": null }, { + "name": "type", + "description": "Type of the issue", + "args": [ + + ], + "type": { + "kind": "ENUM", + "name": "IssueType", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "updatedAt", "description": "Timestamp of when the issue was last updated", "args": [ @@ -13792,6 +16585,16 @@ "defaultValue": null }, { + "name": "milestoneTitle", + "description": "Filter epics by milestone title, computed from epic's issues", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { "name": "iidStartsWith", "description": "Filter epics by iid for autocomplete", "type": { @@ -13921,6 +16724,16 @@ "defaultValue": null }, { + "name": "milestoneTitle", + "description": "Filter epics by milestone title, computed from epic's issues", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { "name": "iidStartsWith", "description": "Filter epics by iid for autocomplete", "type": { @@ -14062,6 +16875,24 @@ "deprecationReason": null }, { + "name": "isTemporaryStorageIncreaseEnabled", + "description": "Status of the temporary storage increase", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "issues", "description": "Issues of the group", "args": [ @@ -14094,16 +16925,6 @@ "defaultValue": null }, { - "name": "state", - "description": "Current state of this issue", - "type": { - "kind": "ENUM", - "name": "IssuableState", - "ofType": null - }, - "defaultValue": null - }, - { "name": "labelName", "description": "Labels applied to this issue", "type": { @@ -14119,7 +16940,7 @@ }, { "name": "milestoneTitle", - "description": "Milestones applied to this issue", + "description": "Milestone applied to this issue", "type": { "kind": "LIST", "name": null, @@ -14133,7 +16954,7 @@ }, { "name": "assigneeUsername", - "description": "Username of a user assigned to the issues", + "description": "Username of a user assigned to the issue", "type": { "kind": "SCALAR", "name": "String", @@ -14222,6 +17043,34 @@ "defaultValue": null }, { + "name": "types", + "description": "Filter issues by the given issue types", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "ENUM", + "name": "IssueType", + "ofType": null + } + } + }, + "defaultValue": null + }, + { + "name": "state", + "description": "Current state of this issue", + "type": { + "kind": "ENUM", + "name": "IssuableState", + "ofType": null + }, + "defaultValue": null + }, + { "name": "sort", "description": "Sort issues by this criteria", "type": { @@ -14246,6 +17095,16 @@ "defaultValue": null }, { + "name": "includeSubgroups", + "description": "Include issues belonging to subgroups.", + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "defaultValue": "false" + }, + { "name": "after", "description": "Returns the elements in the list that come after the specified cursor.", "type": { @@ -14537,7 +17396,7 @@ }, { "name": "milestones", - "description": "Find milestones", + "description": "Milestones of the group", "args": [ { "name": "startDate", @@ -14560,6 +17419,24 @@ "defaultValue": null }, { + "name": "ids", + "description": "Array of global milestone IDs, e.g., \"gid://gitlab/Milestone/1\"", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + } + }, + "defaultValue": null + }, + { "name": "state", "description": "Filter milestones by state", "type": { @@ -14571,7 +17448,7 @@ }, { "name": "includeDescendants", - "description": "Return also milestones in all subgroups and subprojects", + "description": "Also return milestones in all subgroups and subprojects", "type": { "kind": "SCALAR", "name": "Boolean", @@ -15154,8 +18031,89 @@ "deprecationReason": null }, { + "name": "vulnerabilitiesCountByDay", + "description": "Number of vulnerabilities per day for the projects in the group and its subgroups", + "args": [ + { + "name": "startDate", + "description": "First day for which to fetch vulnerability history", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ISO8601Date", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "endDate", + "description": "Last day for which to fetch vulnerability history", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ISO8601Date", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "VulnerabilitiesCountByDayConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "vulnerabilitiesCountByDayAndSeverity", - "description": "Number of vulnerabilities per severity level, per day, for the projects in the group and its subgroups", + "description": "Number of vulnerabilities per severity level, per day, for the projects in the group and its subgroups. Deprecated in 13.3: Use `vulnerabilitiesCountByDay`", "args": [ { "name": "startDate", @@ -15231,6 +18189,32 @@ "name": "VulnerabilitiesCountByDayAndSeverityConnection", "ofType": null }, + "isDeprecated": true, + "deprecationReason": "Use `vulnerabilitiesCountByDay`. Deprecated in 13.3" + }, + { + "name": "vulnerabilityGrades", + "description": "Represents vulnerable project counts for each grade", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "VulnerableProjectsByGrade", + "ofType": null + } + } + } + }, "isDeprecated": false, "deprecationReason": null }, @@ -15687,6 +18671,32 @@ "deprecationReason": null }, { + "name": "vulnerabilityGrades", + "description": "Represents vulnerable project counts for each grade", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "VulnerableProjectsByGrade", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "vulnerabilityScanners", "description": "Vulnerability scanners reported on the vulnerabilties from projects selected in Instance Security Dashboard", "args": [ @@ -15782,6 +18792,12 @@ "description": null, "isDeprecated": false, "deprecationReason": null + }, + { + "name": "all", + "description": null, + "isDeprecated": false, + "deprecationReason": null } ], "possibleTypes": null @@ -15863,6 +18879,24 @@ "deprecationReason": null }, { + "name": "blocked", + "description": "Indicates the issue is blocked", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "closedAt", "description": "Timestamp of when the issue was closed", "args": [ @@ -16390,6 +19424,20 @@ "deprecationReason": null }, { + "name": "statusPagePublishedIncident", + "description": "Indicates whether an issue is published to the status page", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "subscribed", "description": "Indicates the currently logged in user is subscribed to the issue", "args": [ @@ -16494,6 +19542,20 @@ "deprecationReason": null }, { + "name": "type", + "description": "Type of the issue", + "args": [ + + ], + "type": { + "kind": "ENUM", + "name": "IssueType", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "updatedAt", "description": "Timestamp of when the issue was last updated", "args": [ @@ -16758,6 +19820,176 @@ "possibleTypes": null }, { + "kind": "INPUT_OBJECT", + "name": "IssueMoveListInput", + "description": "Autogenerated input type of IssueMoveList", + "fields": null, + "inputFields": [ + { + "name": "projectPath", + "description": "Project the issue to mutate is in", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "iid", + "description": "IID of the issue to mutate", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "boardId", + "description": "Global ID of the board that the issue is in", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "fromListId", + "description": "ID of the board list that the issue will be moved from", + "type": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "toListId", + "description": "ID of the board list that the issue will be moved to", + "type": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "moveBeforeId", + "description": "ID of issue before which the current issue will be positioned at", + "type": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "moveAfterId", + "description": "ID of issue after which the current issue will be positioned at", + "type": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "IssueMoveListPayload", + "description": "Autogenerated return type of IssueMoveList", + "fields": [ + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "errors", + "description": "Errors encountered during execution of the mutation.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "issue", + "description": "The issue after mutation", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "Issue", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { "kind": "OBJECT", "name": "IssuePermissions", "description": "Check permissions for the current user on a issue", @@ -16916,6 +20148,154 @@ }, { "kind": "INPUT_OBJECT", + "name": "IssueSetAssigneesInput", + "description": "Autogenerated input type of IssueSetAssignees", + "fields": null, + "inputFields": [ + { + "name": "projectPath", + "description": "The project the issue to mutate is in", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "iid", + "description": "The IID of the issue to mutate", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "assigneeUsernames", + "description": "The usernames to assign to the resource. Replaces existing assignees by default.", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + } + }, + "defaultValue": null + }, + { + "name": "operationMode", + "description": "The operation to perform. Defaults to REPLACE.", + "type": { + "kind": "ENUM", + "name": "MutationOperationMode", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "IssueSetAssigneesPayload", + "description": "Autogenerated return type of IssueSetAssignees", + "fields": [ + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "errors", + "description": "Errors encountered during execution of the mutation.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "issue", + "description": "The issue after mutation", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "Issue", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "INPUT_OBJECT", "name": "IssueSetConfidentialInput", "description": "Autogenerated input type of IssueSetConfidential", "fields": null, @@ -16936,7 +20316,7 @@ }, { "name": "iid", - "description": "The iid of the issue to mutate", + "description": "The IID of the issue to mutate", "type": { "kind": "NON_NULL", "name": null, @@ -17066,7 +20446,7 @@ }, { "name": "iid", - "description": "The iid of the issue to mutate", + "description": "The IID of the issue to mutate", "type": { "kind": "NON_NULL", "name": null, @@ -17176,6 +20556,132 @@ }, { "kind": "INPUT_OBJECT", + "name": "IssueSetEpicInput", + "description": "Autogenerated input type of IssueSetEpic", + "fields": null, + "inputFields": [ + { + "name": "projectPath", + "description": "The project the issue to mutate is in", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "iid", + "description": "The IID of the issue to mutate", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "epicId", + "description": "Global ID of the epic to be assigned to the issue, epic will be removed if absent or set to null", + "type": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "IssueSetEpicPayload", + "description": "Autogenerated return type of IssueSetEpic", + "fields": [ + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "errors", + "description": "Errors encountered during execution of the mutation.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "issue", + "description": "The issue after mutation", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "Issue", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "INPUT_OBJECT", "name": "IssueSetIterationInput", "description": "Autogenerated input type of IssueSetIteration", "fields": null, @@ -17196,7 +20702,7 @@ }, { "name": "iid", - "description": "The iid of the issue to mutate", + "description": "The IID of the issue to mutate", "type": { "kind": "NON_NULL", "name": null, @@ -17322,7 +20828,7 @@ }, { "name": "iid", - "description": "The iid of the issue to mutate", + "description": "The IID of the issue to mutate", "type": { "kind": "NON_NULL", "name": null, @@ -17432,6 +20938,136 @@ }, { "kind": "INPUT_OBJECT", + "name": "IssueSetSubscriptionInput", + "description": "Autogenerated input type of IssueSetSubscription", + "fields": null, + "inputFields": [ + { + "name": "projectPath", + "description": "The project the issue to mutate is in", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "iid", + "description": "The IID of the issue to mutate", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "subscribedState", + "description": "The desired state of the subscription", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "IssueSetSubscriptionPayload", + "description": "Autogenerated return type of IssueSetSubscription", + "fields": [ + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "errors", + "description": "Errors encountered during execution of the mutation.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "issue", + "description": "The issue after mutation", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "Issue", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "INPUT_OBJECT", "name": "IssueSetWeightInput", "description": "Autogenerated input type of IssueSetWeight", "fields": null, @@ -17452,7 +21088,7 @@ }, { "name": "iid", - "description": "The iid of the issue to mutate", + "description": "The IID of the issue to mutate", "type": { "kind": "NON_NULL", "name": null, @@ -17686,6 +21322,90 @@ "description": null, "isDeprecated": false, "deprecationReason": null + }, + { + "name": "all", + "description": null, + "isDeprecated": false, + "deprecationReason": null + } + ], + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "IssueStatusCountsType", + "description": "Represents total number of issues for the represented statuses.", + "fields": [ + { + "name": "all", + "description": "Number of issues with status ALL for the project", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "closed", + "description": "Number of issues with status CLOSED for the project", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "opened", + "description": "Number of issues with status OPENED for the project", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "ENUM", + "name": "IssueType", + "description": "Issue type", + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": [ + { + "name": "ISSUE", + "description": "Issue issue type", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "INCIDENT", + "description": "Incident issue type", + "isDeprecated": false, + "deprecationReason": null } ], "possibleTypes": null @@ -17728,6 +21448,20 @@ "deprecationReason": null }, { + "name": "descriptionHtml", + "description": "The GitLab Flavored Markdown rendering of `description`", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "dueDate", "description": "Timestamp of the iteration due date", "args": [ @@ -17778,6 +21512,34 @@ "deprecationReason": null }, { + "name": "scopedPath", + "description": "Web path of the iteration, scoped to the query parent. Only valid for Project parents. Returns null in other contexts", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "scopedUrl", + "description": "Web URL of the iteration, scoped to the query parent. Only valid for Project parents. Returns null in other contexts", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "startDate", "description": "Timestamp of the iteration start date", "args": [ @@ -18002,6 +21764,16 @@ "possibleTypes": null }, { + "kind": "SCALAR", + "name": "IterationID", + "description": "Identifier of Iteration", + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { "kind": "ENUM", "name": "IterationState", "description": "State of a GitLab iteration", @@ -19204,6 +22976,16 @@ "possibleTypes": null }, { + "kind": "SCALAR", + "name": "LabelID", + "description": "Identifier of Label", + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { "kind": "ENUM", "name": "ListLimitMetric", "description": "List limit metric setting", @@ -19446,6 +23228,59 @@ "deprecationReason": null }, { + "name": "approvedBy", + "description": "Users who approved the merge request", + "args": [ + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "UserConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "assignees", "description": "Assignees of the merge request", "args": [ @@ -19513,6 +23348,20 @@ "deprecationReason": null }, { + "name": "commitCount", + "description": "Number of commits in the merge request", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "createdAt", "description": "Timestamp of when the merge request was created", "args": [ @@ -20699,6 +24548,24 @@ "description": "The connection type for MergeRequest.", "fields": [ { + "name": "count", + "description": "Total count of collection", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "edges", "description": "A list of edges.", "args": [ @@ -20833,6 +24700,24 @@ "defaultValue": null }, { + "name": "labels", + "description": "Labels of the merge request", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + }, + "defaultValue": null + }, + { "name": "clientMutationId", "description": "A unique identifier for the client performing the mutation.", "type": { @@ -21152,7 +25037,7 @@ }, { "name": "assigneeUsernames", - "description": "The usernames to assign to the merge request. Replaces existing assignees by default.\n", + "description": "The usernames to assign to the resource. Replaces existing assignees by default.", "type": { "kind": "NON_NULL", "name": null, @@ -21174,7 +25059,7 @@ }, { "name": "operationMode", - "description": "The operation to perform. Defaults to REPLACE.\n", + "description": "The operation to perform. Defaults to REPLACE.", "type": { "kind": "ENUM", "name": "MutationOperationMode", @@ -21955,6 +25840,12 @@ "deprecationReason": null }, { + "name": "all", + "description": null, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "merged", "description": null, "isDeprecated": false, @@ -22827,6 +26718,16 @@ "possibleTypes": null }, { + "kind": "SCALAR", + "name": "MilestoneID", + "description": "Identifier of Milestone", + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { "kind": "ENUM", "name": "MilestoneStateEnum", "description": null, @@ -23135,6 +27036,33 @@ "deprecationReason": null }, { + "name": "boardListCreate", + "description": null, + "args": [ + { + "name": "input", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "BoardListCreateInput", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "BoardListCreatePayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "boardListUpdateLimitMetrics", "description": null, "args": [ @@ -23189,6 +27117,33 @@ "deprecationReason": null }, { + "name": "configureSast", + "description": null, + "args": [ + { + "name": "input", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "ConfigureSastInput", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "ConfigureSastPayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "createAlertIssue", "description": null, "args": [ @@ -23270,6 +27225,33 @@ "deprecationReason": null }, { + "name": "createClusterAgent", + "description": null, + "args": [ + { + "name": "input", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "CreateClusterAgentInput", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "CreateClusterAgentPayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "createDiffNote", "description": null, "args": [ @@ -23459,6 +27441,60 @@ "deprecationReason": null }, { + "name": "dastOnDemandScanCreate", + "description": null, + "args": [ + { + "name": "input", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "DastOnDemandScanCreateInput", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "DastOnDemandScanCreatePayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "dastScannerProfileCreate", + "description": null, + "args": [ + { + "name": "input", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "DastScannerProfileCreateInput", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "DastScannerProfileCreatePayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "dastSiteProfileCreate", "description": null, "args": [ @@ -23486,6 +27522,60 @@ "deprecationReason": null }, { + "name": "dastSiteProfileDelete", + "description": null, + "args": [ + { + "name": "input", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "DastSiteProfileDeleteInput", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "DastSiteProfileDeletePayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "dastSiteProfileUpdate", + "description": null, + "args": [ + { + "name": "input", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "DastSiteProfileUpdateInput", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "DastSiteProfileUpdatePayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "deleteAnnotation", "description": null, "args": [ @@ -23540,6 +27630,33 @@ "deprecationReason": null }, { + "name": "designManagementMove", + "description": null, + "args": [ + { + "name": "input", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "DesignManagementMoveInput", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "DesignManagementMovePayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "designManagementUpload", "description": null, "args": [ @@ -23756,6 +27873,60 @@ "deprecationReason": null }, { + "name": "issueMoveList", + "description": null, + "args": [ + { + "name": "input", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "IssueMoveListInput", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "IssueMoveListPayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "issueSetAssignees", + "description": null, + "args": [ + { + "name": "input", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "IssueSetAssigneesInput", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "IssueSetAssigneesPayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "issueSetConfidential", "description": null, "args": [ @@ -23810,6 +27981,33 @@ "deprecationReason": null }, { + "name": "issueSetEpic", + "description": null, + "args": [ + { + "name": "input", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "IssueSetEpicInput", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "IssueSetEpicPayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "issueSetIteration", "description": null, "args": [ @@ -23864,6 +28062,33 @@ "deprecationReason": null }, { + "name": "issueSetSubscription", + "description": null, + "args": [ + { + "name": "input", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "IssueSetSubscriptionInput", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "IssueSetSubscriptionPayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "issueSetWeight", "description": null, "args": [ @@ -24188,6 +28413,33 @@ "deprecationReason": null }, { + "name": "namespaceIncreaseStorageTemporarily", + "description": null, + "args": [ + { + "name": "input", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "NamespaceIncreaseStorageTemporarilyInput", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "NamespaceIncreaseStorageTemporarilyPayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "removeAwardEmoji", "description": null, "args": [ @@ -24431,6 +28683,60 @@ "deprecationReason": null }, { + "name": "updateBoard", + "description": null, + "args": [ + { + "name": "input", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "UpdateBoardInput", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "UpdateBoardPayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "updateBoardList", + "description": null, + "args": [ + { + "name": "input", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "UpdateBoardListInput", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "UpdateBoardListPayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "updateContainerExpirationPolicy", "description": null, "args": [ @@ -24771,6 +29077,24 @@ "deprecationReason": null }, { + "name": "isTemporaryStorageIncreaseEnabled", + "description": "Status of the temporary storage increase", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "lfsEnabled", "description": "Indicates if Large File Storage (LFS) is enabled for namespace", "args": [ @@ -25088,6 +29412,207 @@ "possibleTypes": null }, { + "kind": "INPUT_OBJECT", + "name": "NamespaceIncreaseStorageTemporarilyInput", + "description": "Autogenerated input type of NamespaceIncreaseStorageTemporarily", + "fields": null, + "inputFields": [ + { + "name": "id", + "description": "The global id of the namespace to mutate", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "NamespaceIncreaseStorageTemporarilyPayload", + "description": "Autogenerated return type of NamespaceIncreaseStorageTemporarily", + "fields": [ + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "errors", + "description": "Errors encountered during execution of the mutation.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "namespace", + "description": "The namespace after mutation", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "Namespace", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "INPUT_OBJECT", + "name": "NegatedBoardEpicIssueInput", + "description": null, + "fields": null, + "inputFields": [ + { + "name": "labelName", + "description": "Filter by label name", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "milestoneTitle", + "description": "Filter by milestone title", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "assigneeUsername", + "description": "Filter by assignee username", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "authorUsername", + "description": "Filter by author username", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "releaseTag", + "description": "Filter by release tag", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "epicId", + "description": "Filter by epic ID", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "myReactionEmoji", + "description": "Filter by reaction emoji", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "weight", + "description": "Filter by weight", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { "kind": "OBJECT", "name": "Note", "description": null, @@ -26386,6 +30911,20 @@ "deprecationReason": null }, { + "name": "configSource", + "description": "Config source of the pipeline (UNKNOWN_SOURCE, REPOSITORY_SOURCE, AUTO_DEVOPS_SOURCE, WEBIDE_SOURCE, REMOTE_SOURCE, EXTERNAL_PROJECT_SOURCE, BRIDGE_SOURCE, PARAMETER_SOURCE)", + "args": [ + + ], + "type": { + "kind": "ENUM", + "name": "PipelineConfigSourceEnum", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "coverage", "description": "Coverage percentage", "args": [ @@ -26532,6 +31071,59 @@ "deprecationReason": null }, { + "name": "stages", + "description": "Stages of the pipeline", + "args": [ + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "CiStageConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "startedAt", "description": "Timestamp when the pipeline was started", "args": [ @@ -26582,6 +31174,20 @@ "deprecationReason": null }, { + "name": "user", + "description": "Pipeline user", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "User", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "userPermissions", "description": "Permissions for the current user on the resource", "args": [ @@ -26608,11 +31214,88 @@ "possibleTypes": null }, { + "kind": "ENUM", + "name": "PipelineConfigSourceEnum", + "description": null, + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": [ + { + "name": "UNKNOWN_SOURCE", + "description": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "REPOSITORY_SOURCE", + "description": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "AUTO_DEVOPS_SOURCE", + "description": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "WEBIDE_SOURCE", + "description": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "REMOTE_SOURCE", + "description": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "EXTERNAL_PROJECT_SOURCE", + "description": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "BRIDGE_SOURCE", + "description": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "PARAMETER_SOURCE", + "description": null, + "isDeprecated": false, + "deprecationReason": null + } + ], + "possibleTypes": null + }, + { "kind": "OBJECT", "name": "PipelineConnection", "description": "The connection type for Pipeline.", "fields": [ { + "name": "count", + "description": "Total count of collection", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "edges", "description": "A list of edges.", "args": [ @@ -27291,6 +31974,112 @@ "deprecationReason": null }, { + "name": "dastScannerProfiles", + "description": "The DAST scanner profiles associated with the project", + "args": [ + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "DastScannerProfileConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "dastSiteProfiles", + "description": "DAST Site Profiles associated with the project", + "args": [ + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "DastSiteProfileConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "description", "description": "Short description of the project", "args": [ @@ -27319,6 +32108,57 @@ "deprecationReason": null }, { + "name": "environment", + "description": "A single environment of the project", + "args": [ + { + "name": "name", + "description": "Name of the environment", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "search", + "description": "Search query for environment name", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "states", + "description": "States of environments that should be included in result", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "Environment", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "environments", "description": "Environments of the project", "args": [ @@ -27552,16 +32392,6 @@ "defaultValue": null }, { - "name": "state", - "description": "Current state of this issue", - "type": { - "kind": "ENUM", - "name": "IssuableState", - "ofType": null - }, - "defaultValue": null - }, - { "name": "labelName", "description": "Labels applied to this issue", "type": { @@ -27577,7 +32407,7 @@ }, { "name": "milestoneTitle", - "description": "Milestones applied to this issue", + "description": "Milestone applied to this issue", "type": { "kind": "LIST", "name": null, @@ -27591,7 +32421,7 @@ }, { "name": "assigneeUsername", - "description": "Username of a user assigned to the issues", + "description": "Username of a user assigned to the issue", "type": { "kind": "SCALAR", "name": "String", @@ -27680,6 +32510,34 @@ "defaultValue": null }, { + "name": "types", + "description": "Filter issues by the given issue types", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "ENUM", + "name": "IssueType", + "ofType": null + } + } + }, + "defaultValue": null + }, + { + "name": "state", + "description": "Current state of this issue", + "type": { + "kind": "ENUM", + "name": "IssuableState", + "ofType": null + }, + "defaultValue": null + }, + { "name": "sort", "description": "Sort issues by this criteria", "type": { @@ -27713,8 +32571,8 @@ "deprecationReason": null }, { - "name": "issues", - "description": "Issues of the project", + "name": "issueStatusCounts", + "description": "Counts of issues by status for the project", "args": [ { "name": "iid", @@ -27745,16 +32603,183 @@ "defaultValue": null }, { - "name": "state", - "description": "Current state of this issue", + "name": "labelName", + "description": "Labels applied to this issue", "type": { - "kind": "ENUM", - "name": "IssuableState", + "kind": "LIST", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "milestoneTitle", + "description": "Milestone applied to this issue", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "assigneeUsername", + "description": "Username of a user assigned to the issue", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "assigneeId", + "description": "ID of a user assigned to the issues, \"none\" and \"any\" values supported", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "createdBefore", + "description": "Issues created before this date", + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "createdAfter", + "description": "Issues created after this date", + "type": { + "kind": "SCALAR", + "name": "Time", "ofType": null }, "defaultValue": null }, { + "name": "updatedBefore", + "description": "Issues updated before this date", + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "updatedAfter", + "description": "Issues updated after this date", + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "closedBefore", + "description": "Issues closed before this date", + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "closedAfter", + "description": "Issues closed after this date", + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "search", + "description": "Search query for issue title or description", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "types", + "description": "Filter issues by the given issue types", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "ENUM", + "name": "IssueType", + "ofType": null + } + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "IssueStatusCountsType", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "issues", + "description": "Issues of the project", + "args": [ + { + "name": "iid", + "description": "IID of the issue. For example, \"1\"", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "iids", + "description": "List of IIDs of issues. For example, [1, 2]", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + }, + "defaultValue": null + }, + { "name": "labelName", "description": "Labels applied to this issue", "type": { @@ -27770,7 +32795,7 @@ }, { "name": "milestoneTitle", - "description": "Milestones applied to this issue", + "description": "Milestone applied to this issue", "type": { "kind": "LIST", "name": null, @@ -27784,7 +32809,7 @@ }, { "name": "assigneeUsername", - "description": "Username of a user assigned to the issues", + "description": "Username of a user assigned to the issue", "type": { "kind": "SCALAR", "name": "String", @@ -27873,6 +32898,34 @@ "defaultValue": null }, { + "name": "types", + "description": "Filter issues by the given issue types", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "ENUM", + "name": "IssueType", + "ofType": null + } + } + }, + "defaultValue": null + }, + { + "name": "state", + "description": "Current state of this issue", + "type": { + "kind": "ENUM", + "name": "IssuableState", + "ofType": null + }, + "defaultValue": null + }, + { "name": "sort", "description": "Sort issues by this criteria", "type": { @@ -27960,6 +33013,129 @@ "deprecationReason": null }, { + "name": "iterations", + "description": "Find iterations", + "args": [ + { + "name": "startDate", + "description": "List items within a time frame where items.start_date is between startDate and endDate parameters (endDate parameter must be present)", + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "endDate", + "description": "List items within a time frame where items.end_date is between startDate and endDate parameters (startDate parameter must be present)", + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "state", + "description": "Filter iterations by state", + "type": { + "kind": "ENUM", + "name": "IterationState", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "title", + "description": "Fuzzy search by title", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "id", + "description": "The ID of the Iteration to look up", + "type": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "iid", + "description": "The internal ID of the Iteration to look up", + "type": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "includeAncestors", + "description": "Whether to include ancestor iterations. Defaults to true", + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "IterationConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "jiraImportStatus", "description": "Status of Jira import background job of the project", "args": [ @@ -28272,6 +33448,26 @@ "defaultValue": null }, { + "name": "mergedAfter", + "description": "Merge requests merged after this date", + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "mergedBefore", + "description": "Merge requests merged before this date", + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "defaultValue": null + }, + { "name": "after", "description": "Returns the elements in the list that come after the specified cursor.", "type": { @@ -28349,6 +33545,117 @@ "deprecationReason": null }, { + "name": "milestones", + "description": "Milestones of the project", + "args": [ + { + "name": "startDate", + "description": "List items within a time frame where items.start_date is between startDate and endDate parameters (endDate parameter must be present)", + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "endDate", + "description": "List items within a time frame where items.end_date is between startDate and endDate parameters (startDate parameter must be present)", + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "ids", + "description": "Array of global milestone IDs, e.g., \"gid://gitlab/Milestone/1\"", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + } + }, + "defaultValue": null + }, + { + "name": "state", + "description": "Filter milestones by state", + "type": { + "kind": "ENUM", + "name": "MilestoneStateEnum", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "includeAncestors", + "description": "Also return milestones in the project's parent group and its ancestors", + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "MilestoneConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "name", "description": "Name of the project (without namespace)", "args": [ @@ -29081,6 +34388,20 @@ "deprecationReason": null }, { + "name": "securityDashboardPath", + "description": "Path to project's security dashboard", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "securityScanners", "description": "Information about security analyzers used in the project", "args": [ @@ -31037,6 +36358,55 @@ }, { "kind": "OBJECT", + "name": "PrometheusAlert", + "description": "The alert condition for Prometheus", + "fields": [ + { + "name": "humanizedText", + "description": "The human-readable text of the alert condition", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "id", + "description": "ID of the alert condition", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", "name": "Query", "description": null, "fields": [ @@ -31168,6 +36538,33 @@ "deprecationReason": null }, { + "name": "iteration", + "description": "Find an iteration", + "args": [ + { + "name": "id", + "description": "Find an iteration by its ID", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "IterationID", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "Iteration", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "metadata", "description": "Metadata about GitLab", "args": [ @@ -31182,6 +36579,33 @@ "deprecationReason": null }, { + "name": "milestone", + "description": "Find a milestone", + "args": [ + { + "name": "id", + "description": "Find a milestone by its ID", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "MilestoneID", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "Milestone", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "namespace", "description": "Find a namespace", "args": [ @@ -31705,8 +37129,8 @@ "deprecationReason": null }, { - "name": "vulnerabilitiesCountByDayAndSeverity", - "description": "Number of vulnerabilities per severity level, per day, for the projects on the current user's instance security dashboard", + "name": "vulnerabilitiesCountByDay", + "description": "Number of vulnerabilities per day for the projects on the current user's instance security dashboard", "args": [ { "name": "startDate", @@ -31779,11 +37203,92 @@ ], "type": { "kind": "OBJECT", - "name": "VulnerabilitiesCountByDayAndSeverityConnection", + "name": "VulnerabilitiesCountByDayConnection", "ofType": null }, "isDeprecated": false, "deprecationReason": null + }, + { + "name": "vulnerabilitiesCountByDayAndSeverity", + "description": "Number of vulnerabilities per severity level, per day, for the projects on the current user's instance security dashboard. Deprecated in 13.3: Use `vulnerabilitiesCountByDay`", + "args": [ + { + "name": "startDate", + "description": "First day for which to fetch vulnerability history", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ISO8601Date", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "endDate", + "description": "Last day for which to fetch vulnerability history", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ISO8601Date", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "VulnerabilitiesCountByDayAndSeverityConnection", + "ofType": null + }, + "isDeprecated": true, + "deprecationReason": "Use `vulnerabilitiesCountByDay`. Deprecated in 13.3" } ], "inputFields": null, @@ -34647,6 +40152,20 @@ "deprecationReason": null }, { + "name": "size", + "description": "Size of the UI component.", + "args": [ + + ], + "type": { + "kind": "ENUM", + "name": "SastUiComponentSize", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "type", "description": "Type of the field value.", "args": [ @@ -34948,6 +40467,35 @@ "possibleTypes": null }, { + "kind": "ENUM", + "name": "SastUiComponentSize", + "description": "Size of UI component in SAST configuration page", + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": [ + { + "name": "SMALL", + "description": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "MEDIUM", + "description": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "LARGE", + "description": null, + "isDeprecated": false, + "deprecationReason": null + } + ], + "possibleTypes": null + }, + { "kind": "OBJECT", "name": "ScannedResource", "description": "Represents a resource scanned by a security scan", @@ -35342,6 +40890,12 @@ "description": null, "isDeprecated": false, "deprecationReason": null + }, + { + "name": "COVERAGE_FUZZING", + "description": null, + "isDeprecated": false, + "deprecationReason": null } ], "possibleTypes": null @@ -37243,7 +42797,7 @@ }, { "name": "blob", - "description": "Snippet blob", + "description": "Snippet blob. Deprecated in 13.3: Use `blobs`", "args": [ ], @@ -37256,8 +42810,8 @@ "ofType": null } }, - "isDeprecated": false, - "deprecationReason": null + "isDeprecated": true, + "deprecationReason": "Use `blobs`. Deprecated in 13.3" }, { "name": "blobs", @@ -37841,6 +43395,100 @@ "possibleTypes": null }, { + "kind": "ENUM", + "name": "SnippetBlobActionEnum", + "description": "Type of a snippet blob input action", + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": [ + { + "name": "create", + "description": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "update", + "description": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "delete", + "description": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "move", + "description": null, + "isDeprecated": false, + "deprecationReason": null + } + ], + "possibleTypes": null + }, + { + "kind": "INPUT_OBJECT", + "name": "SnippetBlobActionInputType", + "description": "Represents an action to perform over a snippet file", + "fields": null, + "inputFields": [ + { + "name": "action", + "description": "Type of input action", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "ENUM", + "name": "SnippetBlobActionEnum", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "previousPath", + "description": "Previous path of the snippet file", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "filePath", + "description": "Path of the snippet file", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "content", + "description": "Snippet file content", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { "kind": "OBJECT", "name": "SnippetBlobViewer", "description": "Represents how the blob content should be displayed", @@ -38088,100 +43736,6 @@ "possibleTypes": null }, { - "kind": "ENUM", - "name": "SnippetFileInputActionEnum", - "description": "Type of a snippet file input action", - "fields": null, - "inputFields": null, - "interfaces": null, - "enumValues": [ - { - "name": "create", - "description": null, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "update", - "description": null, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "delete", - "description": null, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "move", - "description": null, - "isDeprecated": false, - "deprecationReason": null - } - ], - "possibleTypes": null - }, - { - "kind": "INPUT_OBJECT", - "name": "SnippetFileInputType", - "description": "Represents an action to perform over a snippet file", - "fields": null, - "inputFields": [ - { - "name": "action", - "description": "Type of input action", - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "ENUM", - "name": "SnippetFileInputActionEnum", - "ofType": null - } - }, - "defaultValue": null - }, - { - "name": "previousPath", - "description": "Previous path of the snippet file", - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "filePath", - "description": "Path of the snippet file", - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "defaultValue": null - }, - { - "name": "content", - "description": "Snippet file content", - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "defaultValue": null - } - ], - "interfaces": null, - "enumValues": null, - "possibleTypes": null - }, - { "kind": "OBJECT", "name": "SnippetPermissions", "description": null, @@ -40428,6 +45982,20 @@ "deprecationReason": null }, { + "name": "webPath", + "description": "Web path for the tree entry (directory)", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "webUrl", "description": "Web URL for the tree entry (directory)", "args": [ @@ -40758,6 +46326,290 @@ }, { "kind": "INPUT_OBJECT", + "name": "UpdateBoardInput", + "description": "Autogenerated input type of UpdateBoard", + "fields": null, + "inputFields": [ + { + "name": "id", + "description": "The board global id.", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "name", + "description": "Name of the board", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "hideBacklogList", + "description": "Whether or not backlog list is hidden.", + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "hideClosedList", + "description": "Whether or not closed list is hidden.", + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "assigneeId", + "description": "The id of user to be assigned to the board.", + "type": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "milestoneId", + "description": "The id of milestone to be assigned to the board.", + "type": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "weight", + "description": "The weight value to be assigned to the board.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "INPUT_OBJECT", + "name": "UpdateBoardListInput", + "description": "Autogenerated input type of UpdateBoardList", + "fields": null, + "inputFields": [ + { + "name": "listId", + "description": "Global ID of the list.", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "position", + "description": "Position of list within the board", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "collapsed", + "description": "Indicates if list is collapsed for this user", + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "UpdateBoardListPayload", + "description": "Autogenerated return type of UpdateBoardList", + "fields": [ + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "errors", + "description": "Errors encountered during execution of the mutation.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "list", + "description": "Mutated list", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "BoardList", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "UpdateBoardPayload", + "description": "Autogenerated return type of UpdateBoard", + "fields": [ + { + "name": "board", + "description": "The board after mutation.", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "Board", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "errors", + "description": "Errors encountered during execution of the mutation.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "INPUT_OBJECT", "name": "UpdateContainerExpirationPolicyInput", "description": "Autogenerated input type of UpdateContainerExpirationPolicy", "fields": null, @@ -41345,7 +47197,7 @@ }, { "name": "iid", - "description": "The iid of the issue to mutate", + "description": "The IID of the issue to mutate", "type": { "kind": "NON_NULL", "name": null, @@ -41398,6 +47250,62 @@ "defaultValue": null }, { + "name": "locked", + "description": "Indicates discussion is locked on the issue", + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "addLabelIds", + "description": "The IDs of labels to be added to the issue.", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + } + }, + "defaultValue": null + }, + { + "name": "removeLabelIds", + "description": "The IDs of labels to be removed from the issue.", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + } + }, + "defaultValue": null + }, + { + "name": "milestoneId", + "description": "The ID of the milestone to be assigned, milestone will be removed if set to null.", + "type": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + }, + "defaultValue": null + }, + { "name": "healthStatus", "description": "The desired health status", "type": { @@ -41669,13 +47577,19 @@ "name": "body", "description": "Content of the note", "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "confidential", + "description": "The confidentiality flag of a note. Default is false.", + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null }, "defaultValue": null }, @@ -41968,8 +47882,8 @@ "defaultValue": null }, { - "name": "files", - "description": "The snippet files to update", + "name": "blobActions", + "description": "Actions to perform over the snippet repository and blobs", "type": { "kind": "LIST", "name": null, @@ -41978,7 +47892,7 @@ "name": null, "ofType": { "kind": "INPUT_OBJECT", - "name": "SnippetFileInputType", + "name": "SnippetBlobActionInputType", "ofType": null } } @@ -42169,6 +48083,26 @@ "defaultValue": null }, { + "name": "mergedAfter", + "description": "Merge requests merged after this date", + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "mergedBefore", + "description": "Merge requests merged before this date", + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "defaultValue": null + }, + { "name": "projectPath", "description": "The full-path of the project the authored merge requests should be in. Incompatible with projectId.", "type": { @@ -42324,6 +48258,26 @@ "defaultValue": null }, { + "name": "mergedAfter", + "description": "Merge requests merged after this date", + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "mergedBefore", + "description": "Merge requests merged before this date", + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "defaultValue": null + }, + { "name": "projectPath", "description": "The full-path of the project the authored merge requests should be in. Incompatible with projectId.", "type": { @@ -42407,6 +48361,20 @@ "deprecationReason": null }, { + "name": "email", + "description": "User email", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "groupMemberships", "description": "Group memberships of the user", "args": [ @@ -42658,6 +48626,20 @@ "deprecationReason": null }, { + "name": "status", + "description": "User status", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "UserStatus", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "todos", "description": "Todos of the user", "args": [ @@ -42859,6 +48841,24 @@ "deprecationReason": null }, { + "name": "webPath", + "description": "Web path of the user", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "webUrl", "description": "Web URL of the user", "args": [ @@ -43057,6 +49057,61 @@ "possibleTypes": null }, { + "kind": "OBJECT", + "name": "UserStatus", + "description": null, + "fields": [ + { + "name": "emoji", + "description": "String representation of emoji", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "message", + "description": "User status message", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "messageHtml", + "description": "HTML of the user status message", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { "kind": "ENUM", "name": "VisibilityLevelsEnum", "description": null, @@ -43116,6 +49171,163 @@ }, { "kind": "OBJECT", + "name": "VulnerabilitiesCountByDay", + "description": "Represents the count of vulnerabilities by severity on a particular day", + "fields": [ + { + "name": "critical", + "description": "Total number of vulnerabilities on a particular day with critical severity", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "date", + "description": "Date for the count", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ISO8601Date", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "high", + "description": "Total number of vulnerabilities on a particular day with high severity", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "info", + "description": "Total number of vulnerabilities on a particular day with info severity", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "low", + "description": "Total number of vulnerabilities on a particular day with low severity", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "medium", + "description": "Total number of vulnerabilities on a particular day with medium severity", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "total", + "description": "Total number of vulnerabilities on a particular day", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "unknown", + "description": "Total number of vulnerabilities on a particular day with unknown severity", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", "name": "VulnerabilitiesCountByDayAndSeverity", "description": "Represents the number of vulnerabilities for a particular severity on a particular day", "fields": [ @@ -43283,6 +49495,118 @@ }, { "kind": "OBJECT", + "name": "VulnerabilitiesCountByDayConnection", + "description": "The connection type for VulnerabilitiesCountByDay.", + "fields": [ + { + "name": "edges", + "description": "A list of edges.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "VulnerabilitiesCountByDayEdge", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "nodes", + "description": "A list of nodes.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "VulnerabilitiesCountByDay", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "pageInfo", + "description": "Information to aid in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "PageInfo", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "VulnerabilitiesCountByDayEdge", + "description": "An edge in a connection.", + "fields": [ + { + "name": "cursor", + "description": "A cursor for use in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "node", + "description": "The item at the end of the edge.", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "VulnerabilitiesCountByDay", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", "name": "Vulnerability", "description": "Represents a vulnerability.", "fields": [ @@ -43468,6 +49792,24 @@ "deprecationReason": null }, { + "name": "resolvedOnDefaultBranch", + "description": "Indicates whether the vulnerability is fixed on the default branch or not", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "scanner", "description": "Scanner metadata for the vulnerability.", "args": [ @@ -43694,6 +50036,47 @@ "possibleTypes": null }, { + "kind": "ENUM", + "name": "VulnerabilityGrade", + "description": "The grade of the vulnerable project", + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": [ + { + "name": "A", + "description": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "B", + "description": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "C", + "description": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "D", + "description": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "F", + "description": null, + "isDeprecated": false, + "deprecationReason": null + } + ], + "possibleTypes": null + }, + { "kind": "OBJECT", "name": "VulnerabilityIdentifier", "description": "Represents a vulnerability identifier.", @@ -43980,6 +50363,11 @@ }, { "kind": "OBJECT", + "name": "VulnerabilityLocationCoverageFuzzing", + "ofType": null + }, + { + "kind": "OBJECT", "name": "VulnerabilityLocationDast", "ofType": null }, @@ -44057,6 +50445,89 @@ }, { "kind": "OBJECT", + "name": "VulnerabilityLocationCoverageFuzzing", + "description": "Represents the location of a vulnerability found by a Coverage Fuzzing scan", + "fields": [ + { + "name": "endLine", + "description": "Number of the last relevant line in the vulnerable file", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "file", + "description": "Path to the vulnerable file", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "startLine", + "description": "Number of the first relevant line in the vulnerable file", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "vulnerableClass", + "description": "Class containing the vulnerability", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "vulnerableMethod", + "description": "Method containing the vulnerability", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", "name": "VulnerabilityLocationDast", "description": "Represents the location of a vulnerability found by a DAST scan", "fields": [ @@ -44965,6 +51436,112 @@ }, { "kind": "OBJECT", + "name": "VulnerableProjectsByGrade", + "description": "Represents vulnerability letter grades with associated projects", + "fields": [ + { + "name": "count", + "description": "Number of projects within this grade", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "grade", + "description": "Grade based on the highest severity vulnerability present", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "ENUM", + "name": "VulnerabilityGrade", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "projects", + "description": "Projects within this grade", + "args": [ + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "ProjectConnection", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", "name": "__Directive", "description": "A Directive provides a way to describe alternate runtime execution and type validation behavior in a GraphQL document.\n\nIn some cases, you need to provide options to alter GraphQL's execution behavior in ways field arguments will not suffice, such as conditionally including or skipping a field. Directives provide this by describing additional information to the executor.", "fields": [ diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md index 6df6632f3bd..8ba1862b009 100644 --- a/doc/api/graphql/reference/index.md +++ b/doc/api/graphql/reference/index.md @@ -64,6 +64,7 @@ Describes an alert from the project's Alert Management | `createdAt` | Time | Timestamp the alert was created | | `description` | String | Description of the alert | | `details` | JSON | Alert details | +| `detailsUrl` | String! | The URL of the alert detail page | | `endedAt` | Time | Timestamp the alert ended | | `eventCount` | Int | Number of events of this alert | | `hosts` | String! => Array | List of hosts the alert came from | @@ -71,6 +72,8 @@ Describes an alert from the project's Alert Management | `issueIid` | ID | Internal ID of the GitLab issue attached to the alert | | `metricsDashboardUrl` | String | URL for metrics embed for the alert | | `monitoringTool` | String | Monitoring tool the alert came from | +| `prometheusAlert` | PrometheusAlert | The alert condition for Prometheus | +| `runbook` | String | Runbook for the alert as defined in alert details | | `service` | String | Service the alert came from | | `severity` | AlertManagementSeverity | Severity of the alert | | `startedAt` | Time | Timestamp the alert was raised | @@ -178,6 +181,7 @@ Autogenerated return type of AwardEmojiToggle | `path` | String! | Path of the entry | | `sha` | String! | Last commit sha for the entry | | `type` | EntryType! | Type of tree entry | +| `webPath` | String | Web path of the blob | | `webUrl` | String | Web URL of the blob | ## Board @@ -186,9 +190,13 @@ Represents a project or group board | Name | Type | Description | | --- | ---- | ---------- | +| `assignee` | User | The board assignee. | +| `hideBacklogList` | Boolean | Whether or not backlog list is hidden. | +| `hideClosedList` | Boolean | Whether or not closed list is hidden. | | `id` | ID! | ID (global ID) of the board | +| `milestone` | Milestone | The board milestone. | | `name` | String | Name of the board | -| `weight` | Int | Weight of the board | +| `weight` | Int | Weight of the board. | ## BoardList @@ -199,6 +207,7 @@ Represents a list for an issue board | `assignee` | User | Assignee in the list | | `collapsed` | Boolean | Indicates if list is collapsed for this user | | `id` | ID! | ID (global ID) of the list | +| `issuesCount` | Int | Count of issues in the list | | `label` | Label | Label of the list | | `limitMetric` | ListLimitMetric | The current limit metric for the list | | `listType` | String! | Type of the list | @@ -207,6 +216,17 @@ Represents a list for an issue board | `milestone` | Milestone | Milestone of the list | | `position` | Int | Position of list within the board | | `title` | String! | Title of the list | +| `totalWeight` | Int | Total weight of all issues in the list | + +## BoardListCreatePayload + +Autogenerated return type of BoardListCreate + +| Name | Type | Description | +| --- | ---- | ---------- | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | +| `list` | BoardList | List of the issue board | ## BoardListUpdateLimitMetricsPayload @@ -225,6 +245,35 @@ Autogenerated return type of BoardListUpdateLimitMetrics | `commit` | Commit | Commit for the branch | | `name` | String! | Name of the branch | +## CiGroup + +| Name | Type | Description | +| --- | ---- | ---------- | +| `name` | String | Name of the job group | +| `size` | Int | Size of the group | + +## CiJob + +| Name | Type | Description | +| --- | ---- | ---------- | +| `name` | String | Name of the job | + +## CiStage + +| Name | Type | Description | +| --- | ---- | ---------- | +| `name` | String | Name of the stage | + +## ClusterAgent + +| Name | Type | Description | +| --- | ---- | ---------- | +| `createdAt` | Time | Timestamp the cluster agent was created | +| `id` | ID! | ID of the cluster agent | +| `name` | String | Name of the cluster agent | +| `project` | Project | The project this cluster agent is associated with | +| `updatedAt` | Time | Timestamp the cluster agent was updated | + ## Commit | Name | Type | Description | @@ -234,6 +283,7 @@ Autogenerated return type of BoardListUpdateLimitMetrics | `authorName` | String | Commit authors name | | `authoredDate` | Time | Timestamp of when the commit was authored | | `description` | String | Description of the commit message | +| `descriptionHtml` | String | The GitLab Flavored Markdown rendering of `description` | | `id` | ID! | ID (global ID) of the commit | | `latestPipeline` **{warning-solid}** | Pipeline | **Deprecated:** Use `pipelines`. Deprecated in 12.5 | | `message` | String | Raw commit message | @@ -241,6 +291,7 @@ Autogenerated return type of BoardListUpdateLimitMetrics | `signatureHtml` | String | Rendered HTML of the commit signature | | `title` | String | Title of the commit message | | `titleHtml` | String | The GitLab Flavored Markdown rendering of `title` | +| `webPath` | String! | Web path of the commit | | `webUrl` | String! | Web URL of the commit | ## CommitCreatePayload @@ -261,6 +312,16 @@ Represents a ComplianceFramework associated with a Project | --- | ---- | ---------- | | `name` | ProjectSettingEnum! | Name of the compliance framework | +## ConfigureSastPayload + +Autogenerated return type of ConfigureSast + +| Name | Type | Description | +| --- | ---- | ---------- | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | +| `result` | JSON | JSON containing the status of MR creation. | + ## ContainerExpirationPolicy A tag expiration policy designed to keep only the images that matter most @@ -309,6 +370,16 @@ Autogenerated return type of CreateBranch | `clientMutationId` | String | A unique identifier for the client performing the mutation. | | `errors` | String! => Array | Errors encountered during execution of the mutation. | +## CreateClusterAgentPayload + +Autogenerated return type of CreateClusterAgent + +| Name | Type | Description | +| --- | ---- | ---------- | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `clusterAgent` | ClusterAgent | Cluster agent created after mutation | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | + ## CreateDiffNotePayload Autogenerated return type of CreateDiffNote @@ -379,6 +450,50 @@ Autogenerated return type of CreateSnippet | `errors` | String! => Array | Errors encountered during execution of the mutation. | | `snippet` | Snippet | The snippet after mutation | +## DastOnDemandScanCreatePayload + +Autogenerated return type of DastOnDemandScanCreate + +| Name | Type | Description | +| --- | ---- | ---------- | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | +| `pipelineUrl` | String | URL of the pipeline that was created. | + +## DastScannerProfile + +Represents a DAST scanner profile. + +| Name | Type | Description | +| --- | ---- | ---------- | +| `id` | ID! | ID of the DAST scanner profile | +| `profileName` | String | Name of the DAST scanner profile | +| `spiderTimeout` | Int | The maximum number of seconds allowed for the spider to traverse the site | +| `targetTimeout` | Int | The maximum number of seconds allowed for the site under test to respond to a request | + +## DastScannerProfileCreatePayload + +Autogenerated return type of DastScannerProfileCreate + +| Name | Type | Description | +| --- | ---- | ---------- | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | +| `id` | ID | ID of the scanner profile. | + +## DastSiteProfile + +Represents a DAST Site Profile. + +| Name | Type | Description | +| --- | ---- | ---------- | +| `editPath` | String | Relative web path to the edit page of a site profile | +| `id` | DastSiteProfileID! | ID of the site profile | +| `profileName` | String | The name of the site profile | +| `targetUrl` | String | The URL of the target to be scanned | +| `userPermissions` | DastSiteProfilePermissions! | Permissions for the current user on the resource | +| `validationStatus` | DastSiteProfileValidationStatusEnum | The current validation status of the site profile | + ## DastSiteProfileCreatePayload Autogenerated return type of DastSiteProfileCreate @@ -387,7 +502,34 @@ Autogenerated return type of DastSiteProfileCreate | --- | ---- | ---------- | | `clientMutationId` | String | A unique identifier for the client performing the mutation. | | `errors` | String! => Array | Errors encountered during execution of the mutation. | -| `id` | ID | ID of the site profile. | +| `id` | DastSiteProfileID | ID of the site profile. | + +## DastSiteProfileDeletePayload + +Autogenerated return type of DastSiteProfileDelete + +| Name | Type | Description | +| --- | ---- | ---------- | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | + +## DastSiteProfilePermissions + +Check permissions for the current user on site profile + +| Name | Type | Description | +| --- | ---- | ---------- | +| `createOnDemandDastScan` | Boolean! | Indicates the user can perform `create_on_demand_dast_scan` on this resource | + +## DastSiteProfileUpdatePayload + +Autogenerated return type of DastSiteProfileUpdate + +| Name | Type | Description | +| --- | ---- | ---------- | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | +| `id` | DastSiteProfileID | ID of the site profile. | ## DeleteAnnotationPayload @@ -473,6 +615,16 @@ Autogenerated return type of DesignManagementDelete | `errors` | String! => Array | Errors encountered during execution of the mutation. | | `version` | DesignVersion | The new version in which the designs are deleted | +## DesignManagementMovePayload + +Autogenerated return type of DesignManagementMove + +| Name | Type | Description | +| --- | ---- | ---------- | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `designCollection` | DesignCollection | The current state of the collection | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | + ## DesignManagementUploadPayload Autogenerated return type of DesignManagementUpload @@ -611,6 +763,7 @@ Describes where code is deployed for a project | Name | Type | Description | | --- | ---- | ---------- | | `id` | ID! | ID of the environment | +| `latestOpenedMostSevereAlert` | AlertManagementAlert | The most severe open alert for the environment. If multiple alerts have equal severity, the most recent is returned. | | `metricsDashboard` | MetricsDashboard | Metrics dashboard schema for the environment | | `name` | String! | Human-readable name of the environment | | `state` | String! | State of the environment, for example: available/stopped | @@ -674,9 +827,9 @@ Counts of descendent epics. | Name | Type | Description | | --- | ---- | ---------- | -| `closedEpics` | Int | Number of closed sub-epics | +| `closedEpics` | Int | Number of closed child epics | | `closedIssues` | Int | Number of closed epic issues | -| `openedEpics` | Int | Number of opened sub-epics | +| `openedEpics` | Int | Number of opened child epics | | `openedIssues` | Int | Number of opened epic issues | ## EpicDescendantWeights @@ -705,6 +858,7 @@ Relationship between an epic and an issue | Name | Type | Description | | --- | ---- | ---------- | | `author` | User! | User that created the issue | +| `blocked` | Boolean! | Indicates the issue is blocked | | `closedAt` | Time | Timestamp of when the issue was closed | | `confidential` | Boolean! | Indicates the issue is confidential | | `createdAt` | Time! | Timestamp of when the issue was created | @@ -726,12 +880,14 @@ Relationship between an epic and an issue | `relationPath` | String | URI path of the epic-issue relation | | `relativePosition` | Int | Relative position of the issue (used for positioning in epic tree and issue boards) | | `state` | IssueState! | State of the issue | +| `statusPagePublishedIncident` | Boolean | Indicates whether an issue is published to the status page | | `subscribed` | Boolean! | Indicates the currently logged in user is subscribed to the issue | | `taskCompletionStatus` | TaskCompletionStatus! | Task completion status of the issue | | `timeEstimate` | Int! | Time estimate of the issue | | `title` | String! | Title of the issue | | `titleHtml` | String | The GitLab Flavored Markdown rendering of `title` | | `totalTimeSpent` | Int! | Total time reported as spent on the issue | +| `type` | IssueType | Type of the issue | | `updatedAt` | Time! | Timestamp of when the issue was last updated | | `upvotes` | Int! | Number of upvotes the issue has received | | `userNotesCount` | Int! | Number of user notes of the issue | @@ -820,6 +976,7 @@ Autogenerated return type of EpicTreeReorder | `fullPath` | ID! | Full path of the namespace | | `groupTimelogsEnabled` | Boolean | Indicates if Group timelogs are enabled for namespace | | `id` | ID! | ID of the namespace | +| `isTemporaryStorageIncreaseEnabled` | Boolean! | Status of the temporary storage increase | | `label` | Label | A label available on this group | | `lfsEnabled` | Boolean | Indicates if Large File Storage (LFS) is enabled for namespace | | `mentionsDisabled` | Boolean | Indicates if a group is disabled from getting mentioned | @@ -837,6 +994,7 @@ Autogenerated return type of EpicTreeReorder | `twoFactorGracePeriod` | Int | Time before two-factor authentication is enforced | | `userPermissions` | GroupPermissions! | Permissions for the current user on the resource | | `visibility` | String | Visibility of the namespace | +| `vulnerabilityGrades` | VulnerableProjectsByGrade! => Array | Represents vulnerable project counts for each grade | | `webUrl` | String! | Web URL of the group | ## GroupMember @@ -859,11 +1017,18 @@ Represents a Group Member | --- | ---- | ---------- | | `readGroup` | Boolean! | Indicates the user can perform `read_group` on this resource | +## InstanceSecurityDashboard + +| Name | Type | Description | +| --- | ---- | ---------- | +| `vulnerabilityGrades` | VulnerableProjectsByGrade! => Array | Represents vulnerable project counts for each grade | + ## Issue | Name | Type | Description | | --- | ---- | ---------- | | `author` | User! | User that created the issue | +| `blocked` | Boolean! | Indicates the issue is blocked | | `closedAt` | Time | Timestamp of when the issue was closed | | `confidential` | Boolean! | Indicates the issue is confidential | | `createdAt` | Time! | Timestamp of when the issue was created | @@ -883,12 +1048,14 @@ Represents a Group Member | `reference` | String! | Internal reference of the issue. Returned in shortened format by default | | `relativePosition` | Int | Relative position of the issue (used for positioning in epic tree and issue boards) | | `state` | IssueState! | State of the issue | +| `statusPagePublishedIncident` | Boolean | Indicates whether an issue is published to the status page | | `subscribed` | Boolean! | Indicates the currently logged in user is subscribed to the issue | | `taskCompletionStatus` | TaskCompletionStatus! | Task completion status of the issue | | `timeEstimate` | Int! | Time estimate of the issue | | `title` | String! | Title of the issue | | `titleHtml` | String | The GitLab Flavored Markdown rendering of `title` | | `totalTimeSpent` | Int! | Total time reported as spent on the issue | +| `type` | IssueType | Type of the issue | | `updatedAt` | Time! | Timestamp of when the issue was last updated | | `upvotes` | Int! | Number of upvotes the issue has received | | `userNotesCount` | Int! | Number of user notes of the issue | @@ -897,6 +1064,16 @@ Represents a Group Member | `webUrl` | String! | Web URL of the issue | | `weight` | Int | Weight of the issue | +## IssueMoveListPayload + +Autogenerated return type of IssueMoveList + +| Name | Type | Description | +| --- | ---- | ---------- | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | +| `issue` | Issue | The issue after mutation | + ## IssuePermissions Check permissions for the current user on a issue @@ -912,6 +1089,16 @@ Check permissions for the current user on a issue | `reopenIssue` | Boolean! | Indicates the user can perform `reopen_issue` on this resource | | `updateIssue` | Boolean! | Indicates the user can perform `update_issue` on this resource | +## IssueSetAssigneesPayload + +Autogenerated return type of IssueSetAssignees + +| Name | Type | Description | +| --- | ---- | ---------- | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | +| `issue` | Issue | The issue after mutation | + ## IssueSetConfidentialPayload Autogenerated return type of IssueSetConfidential @@ -932,6 +1119,16 @@ Autogenerated return type of IssueSetDueDate | `errors` | String! => Array | Errors encountered during execution of the mutation. | | `issue` | Issue | The issue after mutation | +## IssueSetEpicPayload + +Autogenerated return type of IssueSetEpic + +| Name | Type | Description | +| --- | ---- | ---------- | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | +| `issue` | Issue | The issue after mutation | + ## IssueSetIterationPayload Autogenerated return type of IssueSetIteration @@ -952,6 +1149,16 @@ Autogenerated return type of IssueSetLocked | `errors` | String! => Array | Errors encountered during execution of the mutation. | | `issue` | Issue | The issue after mutation | +## IssueSetSubscriptionPayload + +Autogenerated return type of IssueSetSubscription + +| Name | Type | Description | +| --- | ---- | ---------- | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | +| `issue` | Issue | The issue after mutation | + ## IssueSetWeightPayload Autogenerated return type of IssueSetWeight @@ -962,6 +1169,16 @@ Autogenerated return type of IssueSetWeight | `errors` | String! => Array | Errors encountered during execution of the mutation. | | `issue` | Issue | The issue after mutation | +## IssueStatusCountsType + +Represents total number of issues for the represented statuses. + +| Name | Type | Description | +| --- | ---- | ---------- | +| `all` | Int | Number of issues with status ALL for the project | +| `closed` | Int | Number of issues with status CLOSED for the project | +| `opened` | Int | Number of issues with status OPENED for the project | + ## Iteration Represents an iteration object. @@ -970,9 +1187,12 @@ Represents an iteration object. | --- | ---- | ---------- | | `createdAt` | Time! | Timestamp of iteration creation | | `description` | String | Description of the iteration | +| `descriptionHtml` | String | The GitLab Flavored Markdown rendering of `description` | | `dueDate` | Time | Timestamp of the iteration due date | | `id` | ID! | ID of the iteration | | `iid` | ID! | Internal ID of the iteration | +| `scopedPath` | String | Web path of the iteration, scoped to the query parent. Only valid for Project parents. Returns null in other contexts | +| `scopedUrl` | String | Web URL of the iteration, scoped to the query parent. Only valid for Project parents. Returns null in other contexts | | `startDate` | Time | Timestamp of the iteration start date | | `state` | IterationState! | State of the iteration | | `title` | String! | Title of the iteration | @@ -1025,7 +1245,6 @@ Autogenerated return type of JiraImportUsers | Name | Type | Description | | --- | ---- | ---------- | | `active` | Boolean | Indicates if the service is active | -| `projects` | JiraProjectConnection | List of all Jira projects fetched through Jira REST API | | `type` | String | Class name of the service | ## JiraUser @@ -1066,6 +1285,7 @@ Autogenerated return type of MarkAsSpamSnippet | --- | ---- | ---------- | | `allowCollaboration` | Boolean | Indicates if members of the target project can push to the fork | | `author` | User | User who created this merge request | +| `commitCount` | Int | Number of commits in the merge request | | `createdAt` | Time! | Timestamp of when the merge request was created | | `defaultMergeCommitMessage` | String | Default merge commit message of the merge request | | `description` | String | Description of the merge request (Markdown rendered as HTML for caching) | @@ -1276,6 +1496,7 @@ Contains statistics about a milestone | `fullName` | String! | Full name of the namespace | | `fullPath` | ID! | Full path of the namespace | | `id` | ID! | ID of the namespace | +| `isTemporaryStorageIncreaseEnabled` | Boolean! | Status of the temporary storage increase | | `lfsEnabled` | Boolean | Indicates if Large File Storage (LFS) is enabled for namespace | | `name` | String! | Name of the namespace | | `path` | String! | Path of the namespace | @@ -1285,6 +1506,16 @@ Contains statistics about a milestone | `temporaryStorageIncreaseEndsOn` | Time | Date until the temporary storage increase is active | | `visibility` | String | Visibility of the namespace | +## NamespaceIncreaseStorageTemporarilyPayload + +Autogenerated return type of NamespaceIncreaseStorageTemporarily + +| Name | Type | Description | +| --- | ---- | ---------- | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | +| `namespace` | Namespace | The namespace after mutation | + ## Note | Name | Type | Description | @@ -1362,6 +1593,7 @@ Information about pagination in a connection. | --- | ---- | ---------- | | `beforeSha` | String | Base SHA of the source branch | | `committedAt` | Time | Timestamp of the pipeline's commit | +| `configSource` | PipelineConfigSourceEnum | Config source of the pipeline (UNKNOWN_SOURCE, REPOSITORY_SOURCE, AUTO_DEVOPS_SOURCE, WEBIDE_SOURCE, REMOTE_SOURCE, EXTERNAL_PROJECT_SOURCE, BRIDGE_SOURCE, PARAMETER_SOURCE) | | `coverage` | Float | Coverage percentage | | `createdAt` | Time! | Timestamp of the pipeline's creation | | `detailedStatus` | DetailedStatus! | Detailed status of the pipeline | @@ -1374,6 +1606,7 @@ Information about pagination in a connection. | `startedAt` | Time | Timestamp when the pipeline was started | | `status` | PipelineStatusEnum! | Status of the pipeline (CREATED, WAITING_FOR_RESOURCE, PREPARING, PENDING, RUNNING, FAILED, SUCCESS, CANCELED, SKIPPED, MANUAL, SCHEDULED) | | `updatedAt` | Time! | Timestamp of the pipeline's last activity | +| `user` | User | Pipeline user | | `userPermissions` | PipelinePermissions! | Permissions for the current user on the resource | ## PipelinePermissions @@ -1400,6 +1633,7 @@ Information about pagination in a connection. | `createdAt` | Time | Timestamp of the project creation | | `description` | String | Short description of the project | | `descriptionHtml` | String | The GitLab Flavored Markdown rendering of `description` | +| `environment` | Environment | A single environment of the project | | `forksCount` | Int! | Number of times the project has been forked | | `fullPath` | ID! | Full path of the project | | `grafanaIntegration` | GrafanaIntegration | Grafana integration details for the project | @@ -1408,6 +1642,7 @@ Information about pagination in a connection. | `id` | ID! | ID of the project | | `importStatus` | String | Status of import background job of the project | | `issue` | Issue | A single issue of the project | +| `issueStatusCounts` | IssueStatusCountsType | Counts of issues by status for the project | | `issuesEnabled` | Boolean | Indicates if Issues are enabled for the current user | | `jiraImportStatus` | String | Status of Jira import background job of the project | | `jobsEnabled` | Boolean | Indicates if CI/CD pipeline jobs are enabled for the current user | @@ -1434,6 +1669,7 @@ Information about pagination in a connection. | `requirement` | Requirement | Find a single requirement. Available only when feature flag `requirements_management` is enabled. | | `requirementStatesCount` | RequirementStatesCount | Number of requirements for the project by their state | | `sastCiConfiguration` | SastCiConfiguration | SAST CI configuration for the project | +| `securityDashboardPath` | String | Path to project's security dashboard | | `securityScanners` | SecurityScanners | Information about security analyzers used in the project | | `sentryDetailedError` | SentryDetailedError | Detailed version of a Sentry error on the project | | `sentryErrors` | SentryErrorCollection | Paginated collection of Sentry errors on the project | @@ -1528,6 +1764,15 @@ Represents a Project Member | `storageSize` | Float! | Storage size of the project | | `wikiSize` | Float | Wiki size of the project | +## PrometheusAlert + +The alert condition for Prometheus + +| Name | Type | Description | +| --- | ---- | ---------- | +| `humanizedText` | String! | The human-readable text of the alert condition | +| `id` | ID! | ID of the alert condition | + ## Release Represents a release @@ -1703,6 +1948,7 @@ Represents an entity in SAST CI configuration | `description` | String | Entity description that is displayed on the form. | | `field` | String | CI keyword of entity. | | `label` | String | Label for entity used in the form. | +| `size` | SastUiComponentSize | Size of the UI component. | | `type` | String | Type of the field value. | | `value` | String | Current value of the entity. | @@ -1881,7 +2127,7 @@ Represents a snippet entry | Name | Type | Description | | --- | ---- | ---------- | | `author` | User | The owner of the snippet | -| `blob` | SnippetBlob! | Snippet blob | +| `blob` **{warning-solid}** | SnippetBlob! | **Deprecated:** Use `blobs`. Deprecated in 13.3 | | `blobs` | SnippetBlob! => Array | Snippet blobs | | `createdAt` | Time! | Timestamp this snippet was created | | `description` | String | Description of the snippet | @@ -2072,6 +2318,7 @@ Represents a directory | `path` | String! | Path of the entry | | `sha` | String! | Last commit sha for the entry | | `type` | EntryType! | Type of tree entry | +| `webPath` | String | Web path for the tree entry (directory) | | `webUrl` | String | Web URL for the tree entry (directory) | ## UpdateAlertStatusPayload @@ -2086,6 +2333,26 @@ Autogenerated return type of UpdateAlertStatus | `issue` | Issue | The issue created after mutation | | `todo` | Todo | The todo after mutation | +## UpdateBoardListPayload + +Autogenerated return type of UpdateBoardList + +| Name | Type | Description | +| --- | ---- | ---------- | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | +| `list` | BoardList | Mutated list | + +## UpdateBoardPayload + +Autogenerated return type of UpdateBoard + +| Name | Type | Description | +| --- | ---- | ---------- | +| `board` | Board | The board after mutation. | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | + ## UpdateContainerExpirationPolicyPayload Autogenerated return type of UpdateContainerExpirationPolicy @@ -2171,11 +2438,14 @@ Autogenerated return type of UpdateSnippet | Name | Type | Description | | --- | ---- | ---------- | | `avatarUrl` | String | URL of the user's avatar | +| `email` | String | User email | | `id` | ID! | ID of the user | | `name` | String! | Human-readable name of the user | | `state` | UserState! | State of the user | +| `status` | UserStatus | User status | | `userPermissions` | UserPermissions! | Permissions for the current user on the resource | | `username` | String! | Username of the user. Unique within this instance of GitLab | +| `webPath` | String! | Web path of the user | | `webUrl` | String! | Web URL of the user | ## UserPermissions @@ -2184,6 +2454,29 @@ Autogenerated return type of UpdateSnippet | --- | ---- | ---------- | | `createSnippet` | Boolean! | Indicates the user can perform `create_snippet` on this resource | +## UserStatus + +| Name | Type | Description | +| --- | ---- | ---------- | +| `emoji` | String | String representation of emoji | +| `message` | String | User status message | +| `messageHtml` | String | HTML of the user status message | + +## VulnerabilitiesCountByDay + +Represents the count of vulnerabilities by severity on a particular day + +| Name | Type | Description | +| --- | ---- | ---------- | +| `critical` | Int! | Total number of vulnerabilities on a particular day with critical severity | +| `date` | ISO8601Date! | Date for the count | +| `high` | Int! | Total number of vulnerabilities on a particular day with high severity | +| `info` | Int! | Total number of vulnerabilities on a particular day with info severity | +| `low` | Int! | Total number of vulnerabilities on a particular day with low severity | +| `medium` | Int! | Total number of vulnerabilities on a particular day with medium severity | +| `total` | Int! | Total number of vulnerabilities on a particular day | +| `unknown` | Int! | Total number of vulnerabilities on a particular day with unknown severity | + ## VulnerabilitiesCountByDayAndSeverity Represents the number of vulnerabilities for a particular severity on a particular day @@ -2207,6 +2500,7 @@ Represents a vulnerability. | `primaryIdentifier` | VulnerabilityIdentifier | Primary identifier of the vulnerability. | | `project` | Project | The project on which the vulnerability was found | | `reportType` | VulnerabilityReportType | Type of the security report that found the vulnerability (SAST, DEPENDENCY_SCANNING, CONTAINER_SCANNING, DAST, SECRET_DETECTION, COVERAGE_FUZZING) | +| `resolvedOnDefaultBranch` | Boolean! | Indicates whether the vulnerability is fixed on the default branch or not | | `scanner` | VulnerabilityScanner | Scanner metadata for the vulnerability. | | `severity` | VulnerabilitySeverity | Severity of the vulnerability (INFO, UNKNOWN, LOW, MEDIUM, HIGH, CRITICAL) | | `state` | VulnerabilityState | State of the vulnerability (DETECTED, DISMISSED, RESOLVED, CONFIRMED) | @@ -2246,6 +2540,18 @@ Represents the location of a vulnerability found by a container security scan | `image` | String | Name of the vulnerable container image | | `operatingSystem` | String | Operating system that runs on the vulnerable container image | +## VulnerabilityLocationCoverageFuzzing + +Represents the location of a vulnerability found by a Coverage Fuzzing scan + +| Name | Type | Description | +| --- | ---- | ---------- | +| `endLine` | String | Number of the last relevant line in the vulnerable file | +| `file` | String | Path to the vulnerable file | +| `startLine` | String | Number of the first relevant line in the vulnerable file | +| `vulnerableClass` | String | Class containing the vulnerability | +| `vulnerableMethod` | String | Method containing the vulnerability | + ## VulnerabilityLocationDast Represents the location of a vulnerability found by a DAST scan @@ -2345,3 +2651,12 @@ Represents a vulnerable package. Used in vulnerability dependency data | Name | Type | Description | | --- | ---- | ---------- | | `name` | String | The name of the vulnerable package | + +## VulnerableProjectsByGrade + +Represents vulnerability letter grades with associated projects + +| Name | Type | Description | +| --- | ---- | ---------- | +| `count` | Int! | Number of projects within this grade | +| `grade` | VulnerabilityGrade! | Grade based on the highest severity vulnerability present | diff --git a/doc/api/graphql/sample_issue_boards.md b/doc/api/graphql/sample_issue_boards.md new file mode 100644 index 00000000000..4ac9317b01a --- /dev/null +++ b/doc/api/graphql/sample_issue_boards.md @@ -0,0 +1,44 @@ +# Identify issue boards with GraphQL + +This page describes how you can use the GraphiQL explorer to identify +existing issue boards in the `gitlab-docs` documentation repository. + +## Set up the GraphiQL explorer + +This procedure presents a substantive example that you can copy and paste into your own +instance of the [GraphiQL explorer](https://gitlab.com/-/graphql-explorer): + +1. Copy the following code excerpt: + + ```graphql + query { + project(fullPath: "gitlab-org/gitlab-docs") { + name + forksCount + statistics { + wikiSize + } + issuesEnabled + boards { + nodes { + id + name + } + } + } + } + ``` + +1. Open the [GraphiQL Explorer](https://gitlab.com/-/graphql-explorer) page. +1. Paste the `query` listed above into the left window of your GraphiQL explorer tool. +1. Click Play to get the result shown here: + + + +If you want to view one of these boards, take one of the numeric identifiers shown in the output. From the screenshot, the first identifier is `105011`. Navigate to the following URL, which includes the identifier: + +```markdown +https://gitlab.com/gitlab-org/gitlab-docs/-/boards/105011 +``` + +For more information on each attribute, see the [GraphQL API Resources](reference/index.md). diff --git a/doc/api/group_activity_analytics.md b/doc/api/group_activity_analytics.md index 302c7703669..90920d1b25c 100644 --- a/doc/api/group_activity_analytics.md +++ b/doc/api/group_activity_analytics.md @@ -1,6 +1,6 @@ # Group Activity Analytics API -> **Note:** This feature was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/26460) in GitLab 12.9. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/26460) in GitLab 12.9. ## Get count of recently created issues for group diff --git a/doc/api/group_milestones.md b/doc/api/group_milestones.md index e157655a713..e992637f4f0 100644 --- a/doc/api/group_milestones.md +++ b/doc/api/group_milestones.md @@ -27,13 +27,14 @@ GET /groups/:id/milestones?search=version Parameters: -| Attribute | Type | Required | Description | -| --------- | ------ | -------- | ----------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) owned by the authenticated user | -| `iids[]` | integer array | no | Return only the milestones having the given `iid` | -| `state` | string | no | Return only `active` or `closed` milestones | -| `title` | string | no | Return only the milestones having the given `title` | -| `search` | string | no | Return only milestones with a title or description matching the provided string | +| Attribute | Type | Required | Description | +| --------- | ------ | -------- | ----------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) owned by the authenticated user | +| `iids[]` | integer array | no | Return only the milestones having the given `iid` (Note: ignored if `include_parent_milestones` is set as `true`) | +| `state` | string | no | Return only `active` or `closed` milestones | +| `title` | string | no | Return only the milestones having the given `title` | +| `search` | string | no | Return only milestones with a title or description matching the provided string | +| `include_parent_milestones` | boolean | optional | Include milestones from parent group and its ancestors. Introduced in [GitLab 13.4](https://gitlab.com/gitlab-org/gitlab/-/issues/196066) | ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/5/milestones" diff --git a/doc/api/group_wikis.md b/doc/api/group_wikis.md index 62094ffc940..414c795e092 100644 --- a/doc/api/group_wikis.md +++ b/doc/api/group_wikis.md @@ -1,3 +1,10 @@ +--- +stage: Create +group: Knowledge +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference, api +--- + # Wikis API > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/212199) in GitLab 13.2. diff --git a/doc/api/groups.md b/doc/api/groups.md index a5a0c210540..07b2738f2d3 100644 --- a/doc/api/groups.md +++ b/doc/api/groups.md @@ -184,7 +184,7 @@ Parameters: | `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) owned by the authenticated user | | `archived` | boolean | no | Limit by archived status | | `visibility` | string | no | Limit by visibility `public`, `internal`, or `private` | -| `order_by` | string | no | Return projects ordered by `id`, `name`, `path`, `created_at`, `updated_at`, or `last_activity_at` fields. Default is `created_at` | +| `order_by` | string | no | Return projects ordered by `id`, `name`, `path`, `created_at`, `updated_at`, `similarity` (1), or `last_activity_at` fields. Default is `created_at` | | `sort` | string | no | Return projects sorted in `asc` or `desc` order. Default is `desc` | | `search` | string | no | Return list of authorized projects matching the search criteria | | `simple` | boolean | no | Return only the ID, URL, name, and path of each project | @@ -198,6 +198,13 @@ Parameters: | `with_custom_attributes` | boolean | no | Include [custom attributes](custom_attributes.md) in response (admins only) | | `with_security_reports` | boolean | no | **(ULTIMATE)** Return only projects that have security reports artifacts present in any of their builds. This means "projects with security reports enabled". Default is `false` | +1. Order by similarity: Orders the results by a similarity score calculated from the provided `search` +URL parameter. This is an [alpha](https://about.gitlab.com/handbook/product/gitlab-the-product/#alpha) feature [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/221043) in GitLab 13.3. + + The feature is behind a feature flag, you can [enable it](../administration/feature_flags.md#enable-or-disable-the-feature) +with the `similarity_search` flag. When using `order_by=similarity` the `sort` parameter is +ignored. When the `search` parameter is not provided, the API returns the projects ordered by `name`. + Example response: ```json @@ -743,6 +750,7 @@ PUT /groups/:id | `file_template_project_id` | integer | no | **(PREMIUM)** The ID of a project to load custom file templates from. | | `shared_runners_minutes_limit` | integer | no | **(STARTER ONLY)** Pipeline minutes quota for this group (included in plan). Can be `nil` (default; inherit system default), `0` (unlimited) or `> 0` | | `extra_shared_runners_minutes_limit` | integer | no | **(STARTER ONLY)** Extra pipeline minutes quota for this group (purchased in addition to the minutes included in the plan). | +| `prevent_forking_outside_group` | boolean | no | **(PREMIUM)** When enabled, users can **not** fork projects from this group to external namespaces NOTE: **Note:** The `projects` and `shared_projects` attributes in the response are deprecated and will be [removed in API v5](https://gitlab.com/gitlab-org/gitlab/-/issues/213797). @@ -839,7 +847,7 @@ Only available to group owners and administrators. This endpoint either: - Removes group, and queues a background job to delete all projects in the group as well. -- Since [GitLab 12.8](https://gitlab.com/gitlab-org/gitlab/-/issues/33257), on [Premium or Silver](https://about.gitlab.com/pricing/) or higher tiers, marks a group for deletion. The deletion will happen 7 days later by default, but this can be changed in the [instance settings](../user/admin_area/settings/visibility_and_access_controls.md#default-deletion-adjourned-period-premium-only). +- Since [GitLab 12.8](https://gitlab.com/gitlab-org/gitlab/-/issues/33257), on [Premium or Silver](https://about.gitlab.com/pricing/) or higher tiers, marks a group for deletion. The deletion will happen 7 days later by default, but this can be changed in the [instance settings](../user/admin_area/settings/visibility_and_access_controls.md#default-deletion-delay-premium-only). ```plaintext DELETE /groups/:id @@ -1156,3 +1164,46 @@ DELETE /groups/:id/share/:group_id | --------- | -------------- | -------- | ----------- | | `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) | | `group_id` | integer | yes | The ID of the group to share with | + +## Push Rules **(STARTER)** + +### Get group push rules + +Get the [push rules](../user/group/index.md#group-push-rules-starter) of a group. + +```plaintext +GET /groups/:id/push_rule +``` + +| Attribute | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `id` | integer/string | yes | The ID of the group or [URL-encoded path of the group](README.md#namespaced-path-encoding) | + +```json +{ + "id": 2, + "created_at": "2020-08-17T19:09:19.580Z", + "commit_message_regex": "[a-zA-Z]", + "commit_message_negative_regex": "[x+]", + "branch_name_regex": "[a-z]", + "deny_delete_tag": true, + "member_check": true, + "prevent_secrets": true, + "author_email_regex": "^[A-Za-z0-9.]+@gitlab.com$", + "file_name_regex": "(exe)$", + "max_file_size": 100 +} +``` + +Users on GitLab [Premium, Silver, or higher](https://about.gitlab.com/pricing/) will also see +the `commit_committer_check` and `reject_unsigned_commits` parameters: + +```json +{ + "id": 2, + "created_at": "2020-08-17T19:09:19.580Z", + "commit_committer_check": true, + "reject_unsigned_commits": false, + ... +} +``` diff --git a/doc/api/instance_level_ci_variables.md b/doc/api/instance_level_ci_variables.md index ceaf7e30c48..d8f306a822c 100644 --- a/doc/api/instance_level_ci_variables.md +++ b/doc/api/instance_level_ci_variables.md @@ -1,3 +1,9 @@ +--- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Instance-level CI/CD variables API > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/14108) in GitLab 13.0 @@ -73,7 +79,7 @@ POST /admin/ci/variables | Attribute | Type | required | Description | |-----------------|---------|----------|-----------------------| | `key` | string | yes | The `key` of a variable. Max 255 characters, only `A-Z`, `a-z`, `0-9`, and `_` are allowed. | -| `value` | string | yes | The `value` of a variable. Around 700 characters allowed. | +| `value` | string | yes | The `value` of a variable. 10,000 characters allowed. [Since GitLab 13.3](https://gitlab.com/gitlab-org/gitlab/-/issues/220028) | | `variable_type` | string | no | The type of a variable. Available types are: `env_var` (default) and `file`. | | `protected` | boolean | no | Whether the variable is protected. | | `masked` | boolean | no | Whether the variable is masked. | @@ -103,7 +109,7 @@ PUT /admin/ci/variables/:key | Attribute | Type | required | Description | |-----------------|---------|----------|-------------------------| | `key` | string | yes | The `key` of a variable. | -| `value` | string | yes | The `value` of a variable. | +| `value` | string | yes | The `value` of a variable. [Since GitLab 13.3](https://gitlab.com/gitlab-org/gitlab/-/issues/220028), around 10,000 characters allowed. Previously 700 characters. | | `variable_type` | string | no | The type of a variable. Available types are: `env_var` (default) and `file`. | | `protected` | boolean | no | Whether the variable is protected. | | `masked` | boolean | no | Whether the variable is masked. | diff --git a/doc/api/issues.md b/doc/api/issues.md index 22f5d994f85..478557e1cd1 100644 --- a/doc/api/issues.md +++ b/doc/api/issues.md @@ -72,6 +72,7 @@ GET /issues?confidential=true | `confidential` | boolean | no | Filter confidential or public issues. | | `not` | Hash | no | Return issues that do not match the parameters supplied. Accepts: `labels`, `milestone`, `author_id`, `author_username`, `assignee_id`, `assignee_username`, `my_reaction_emoji` | | `non_archived` | boolean | no | Return issues only from non-archived projects. If `false`, response will return issues from both archived and non-archived projects. Default is `true`. _(Introduced in [GitLab 13.0](https://gitlab.com/gitlab-org/gitlab/-/issues/197170))_ | +| `due_date` | string | no | Return issues that have no due date (`0`) or whose due date is this week, this month, between two weeks ago and next month, or which are overdue. Accepts: `0` (no due date), `overdue`, `week`, `month`, `next_month_and_previous_two_weeks`. _(Introduced in [GitLab 13.3](https://gitlab.com/gitlab-org/gitlab/-/issues/233420))_ | ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/issues" @@ -134,7 +135,7 @@ Example response: "merge_requests_count": 0, "user_notes_count": 1, "due_date": "2016-07-22", - "web_url": "http://example.com/my-group/my-project/issues/6", + "web_url": "http://gitlab.example.com/my-group/my-project/issues/6", "references": { "short": "#6", "relative": "my-group/my-project#6", @@ -151,10 +152,10 @@ Example response: "confidential": false, "discussion_locked": false, "_links":{ - "self":"http://example.com/api/v4/projects/1/issues/76", - "notes":"`http://example.com/`api/v4/projects/1/issues/76/notes", - "award_emoji":"http://example.com/api/v4/projects/1/issues/76/award_emoji", - "project":"http://example.com/api/v4/projects/1" + "self":"http://gitlab.example.com/api/v4/projects/1/issues/76", + "notes":"http://gitlab.example.com/api/v4/projects/1/issues/76/notes", + "award_emoji":"http://gitlab.example.com/api/v4/projects/1/issues/76/award_emoji", + "project":"http://gitlab.example.com/api/v4/projects/1" }, "task_completion_status":{ "count":0, @@ -178,6 +179,20 @@ the `weight` parameter: ] ``` +Users on GitLab [Ultimate](https://about.gitlab.com/pricing/) will also see +the `health_status` parameter: + +```json +[ + { + "state" : "opened", + "description" : "Ratione dolores corrupti mollitia soluta quia.", + "health_status": "on_track", + ... + } +] +``` + **Note**: `assignee` column is deprecated, now we show it as a single-sized array `assignees` to conform to the GitLab EE API. **Note**: The `closed_by` attribute was [introduced in GitLab 10.6](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/17042). This value will only be present for issues which were closed after GitLab 10.6 and when the user account that closed the issue still exists. @@ -231,6 +246,7 @@ GET /groups/:id/issues?confidential=true | `confidential` | boolean | no | Filter confidential or public issues. | | `not` | Hash | no | Return issues that do not match the parameters supplied. Accepts: `labels`, `milestone`, `author_id`, `author_username`, `assignee_id`, `assignee_username`, `my_reaction_emoji`, `search`, `in` | | `non_archived` | boolean | no | Return issues from non archived projects. Default is true. _(Introduced in [GitLab 12.8](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/23785))_ | +| `due_date` | string | no | Return issues that have no due date (`0`) or whose due date is this week, this month, between two weeks ago and next month, or which are overdue. Accepts: `0` (no due date), `overdue`, `week`, `month`, `next_month_and_previous_two_weeks`. _(Introduced in [GitLab 13.3](https://gitlab.com/gitlab-org/gitlab/-/issues/233420))_ | ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/4/issues" @@ -292,7 +308,7 @@ Example response: "closed_by" : null, "user_notes_count": 1, "due_date": null, - "web_url": "http://example.com/my-group/my-project/issues/1", + "web_url": "http://gitlab.example.com/my-group/my-project/issues/1", "references": { "short": "#1", "relative": "my-project#1", @@ -309,10 +325,10 @@ Example response: "confidential": false, "discussion_locked": false, "_links":{ - "self":"http://example.com/api/v4/projects/4/issues/41", - "notes":"`http://example.com/`api/v4/projects/4/issues/41/notes", - "award_emoji":"http://example.com/api/v4/projects/4/issues/41/award_emoji", - "project":"http://example.com/api/v4/projects/4" + "self":"http://gitlab.example.com/api/v4/projects/4/issues/41", + "notes":"http://gitlab.example.com/api/v4/projects/4/issues/41/notes", + "award_emoji":"http://gitlab.example.com/api/v4/projects/4/issues/41/award_emoji", + "project":"http://gitlab.example.com/api/v4/projects/4" }, "task_completion_status":{ "count":0, @@ -336,6 +352,20 @@ the `weight` parameter: ] ``` +Users on GitLab [Ultimate](https://about.gitlab.com/pricing/) will also see +the `health_status` parameter: + +```json +[ + { + "project_id" : 4, + "description" : "Omnis vero earum sunt corporis dolor et placeat.", + "health_status": "at_risk", + ... + } +] +``` + **Note**: `assignee` column is deprecated, now we show it as a single-sized array `assignees` to conform to the GitLab EE API. **Note**: The `closed_by` attribute was [introduced in GitLab 10.6](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/17042). This value will only be present for issues which were closed after GitLab 10.6 and when the user account that closed the issue still exists. @@ -388,6 +418,7 @@ GET /projects/:id/issues?confidential=true | `updated_before` | datetime | no | Return issues updated on or before the given time | | `confidential` | boolean | no | Filter confidential or public issues. | | `not` | Hash | no | Return issues that do not match the parameters supplied. Accepts: `labels`, `milestone`, `author_id`, `author_username`, `assignee_id`, `assignee_username`, `my_reaction_emoji`, `search`, `in` | +| `due_date` | string | no | Return issues that have no due date (`0`) or whose due date is this week, this month, between two weeks ago and next month, or which are overdue. Accepts: `0` (no due date), `overdue`, `week`, `month`, `next_month_and_previous_two_weeks`. _(Introduced in [GitLab 13.3](https://gitlab.com/gitlab-org/gitlab/-/issues/233420))_ | ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/4/issues" @@ -456,7 +487,7 @@ Example response: }, "user_notes_count": 1, "due_date": "2016-07-22", - "web_url": "http://example.com/my-group/my-project/issues/1", + "web_url": "http://gitlab.example.com/my-group/my-project/issues/1", "references": { "short": "#1", "relative": "#1", @@ -473,10 +504,10 @@ Example response: "confidential": false, "discussion_locked": false, "_links":{ - "self":"http://example.com/api/v4/projects/4/issues/41", - "notes":"`http://example.com/`api/v4/projects/4/issues/41/notes", - "award_emoji":"http://example.com/api/v4/projects/4/issues/41/award_emoji", - "project":"http://example.com/api/v4/projects/4" + "self":"http://gitlab.example.com/api/v4/projects/4/issues/41", + "notes":"http://gitlab.example.com/api/v4/projects/4/issues/41/notes", + "award_emoji":"http://gitlab.example.com/api/v4/projects/4/issues/41/award_emoji", + "project":"http://gitlab.example.com/api/v4/projects/4" }, "task_completion_status":{ "count":0, @@ -500,6 +531,20 @@ the `weight` parameter: ] ``` +Users on GitLab [Ultimate](https://about.gitlab.com/pricing/) will also see +the `health_status` parameter: + +```json +[ + { + "project_id" : 4, + "description" : "Omnis vero earum sunt corporis dolor et placeat.", + "health_status": "at_risk", + ... + } +] +``` + **Note**: `assignee` column is deprecated, now we show it as a single-sized array `assignees` to conform to the GitLab EE API. **Note**: The `closed_by` attribute was [introduced in GitLab 10.6](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/17042). This value will only be present for issues which were closed after GitLab 10.6 and when the user account that closed the issue still exists. @@ -581,7 +626,7 @@ Example response: "subscribed": false, "user_notes_count": 1, "due_date": null, - "web_url": "http://example.com/my-group/my-project/issues/1", + "web_url": "http://gitlab.example.com/my-group/my-project/issues/1", "references": { "short": "#1", "relative": "#1", @@ -596,10 +641,10 @@ Example response: "confidential": false, "discussion_locked": false, "_links": { - "self": "http://example.com/api/v4/projects/1/issues/2", - "notes": "http://example.com/api/v4/projects/1/issues/2/notes", - "award_emoji": "http://example.com/api/v4/projects/1/issues/2/award_emoji", - "project": "http://example.com/api/v4/projects/1" + "self": "http://gitlab.example.com/api/v4/projects/1/issues/2", + "notes": "http://gitlab.example.com/api/v4/projects/1/issues/2/notes", + "award_emoji": "http://gitlab.example.com/api/v4/projects/1/issues/2/award_emoji", + "project": "http://gitlab.example.com/api/v4/projects/1" }, "task_completion_status":{ "count":0, @@ -639,6 +684,20 @@ the `epic` property: } ``` +Users on GitLab [Ultimate](https://about.gitlab.com/pricing/) will also additionally see +the `health_status` property: + +```json +[ + { + "project_id" : 4, + "description" : "Omnis vero earum sunt corporis dolor et placeat.", + "health_status": "on_track", + ... + } +] +``` + **Note**: `assignee` column is deprecated, now we show it as a single-sized array `assignees` to conform to the GitLab EE API. **Note**: The `closed_by` attribute was [introduced in GitLab 10.6](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/17042). This value will only be present for issues which were closed after GitLab 10.6 and when the user account that closed the issue still exists. @@ -661,7 +720,7 @@ POST /projects/:id/issues | `title` | string | yes | The title of an issue | | `description` | string | no | The description of an issue. Limited to 1,048,576 characters. | | `confidential` | boolean | no | Set an issue to be confidential. Default is `false`. | -| `assignee_ids` | integer array | no | The ID of a user to assign issue | +| `assignee_ids` | integer array | no | The ID of the user(s) to assign the issue to. | | `milestone_id` | integer | no | The global ID of a milestone to assign issue | | `labels` | string | no | Comma-separated label names for an issue | | `created_at` | string | no | Date time string, ISO 8601 formatted, for example `2016-03-11T03:45:40Z` (requires admin or project/group owner rights) | @@ -710,7 +769,7 @@ Example response: "subscribed" : true, "user_notes_count": 0, "due_date": null, - "web_url": "http://example.com/my-group/my-project/issues/14", + "web_url": "http://gitlab.example.com/my-group/my-project/issues/14", "references": { "short": "#14", "relative": "#14", @@ -725,10 +784,10 @@ Example response: "confidential": false, "discussion_locked": false, "_links": { - "self": "http://example.com/api/v4/projects/1/issues/2", - "notes": "http://example.com/api/v4/projects/1/issues/2/notes", - "award_emoji": "http://example.com/api/v4/projects/1/issues/2/award_emoji", - "project": "http://example.com/api/v4/projects/1" + "self": "http://gitlab.example.com/api/v4/projects/1/issues/2", + "notes": "http://gitlab.example.com/api/v4/projects/1/issues/2/notes", + "award_emoji": "http://gitlab.example.com/api/v4/projects/1/issues/2/award_emoji", + "project": "http://gitlab.example.com/api/v4/projects/1" }, "task_completion_status":{ "count":0, @@ -749,17 +808,28 @@ the `weight` parameter: } ``` +Users on GitLab [Ultimate](https://about.gitlab.com/pricing/) will also see +the `health_status` parameter: + +```json +[ + { + "project_id" : 4, + "description" : "Omnis vero earum sunt corporis dolor et placeat.", + "health_status": "on_track", + ... + } +] +``` + **Note**: `assignee` column is deprecated, now we show it as a single-sized array `assignees` to conform to the GitLab EE API. **Note**: The `closed_by` attribute was [introduced in GitLab 10.6](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/17042). This value will only be present for issues which were closed after GitLab 10.6 and when the user account that closed the issue still exists. ## Rate limits -To help avoid abuse, users are limited to: - -| Request Type | Limit | -| ---------------- | --------------------------- | -| Create | 300 issues per minute | +To help avoid abuse, users can be limited to a specific number of `Create` requests per minute. +See [Issues rate limits](../user/admin_area/settings/rate_limit_on_issues_creation.md). ## Edit issue @@ -835,7 +905,7 @@ Example response: "subscribed" : true, "user_notes_count": 0, "due_date": "2016-07-22", - "web_url": "http://example.com/my-group/my-project/issues/15", + "web_url": "http://gitlab.example.com/my-group/my-project/issues/15", "references": { "short": "#15", "relative": "#15", @@ -850,10 +920,10 @@ Example response: "confidential": false, "discussion_locked": false, "_links": { - "self": "http://example.com/api/v4/projects/1/issues/2", - "notes": "http://example.com/api/v4/projects/1/issues/2/notes", - "award_emoji": "http://example.com/api/v4/projects/1/issues/2/award_emoji", - "project": "http://example.com/api/v4/projects/1" + "self": "http://gitlab.example.com/api/v4/projects/1/issues/2", + "notes": "http://gitlab.example.com/api/v4/projects/1/issues/2/notes", + "award_emoji": "http://gitlab.example.com/api/v4/projects/1/issues/2/award_emoji", + "project": "http://gitlab.example.com/api/v4/projects/1" }, "task_completion_status":{ "count":0, @@ -874,6 +944,20 @@ the `weight` parameter: } ``` +Users on GitLab [Ultimate](https://about.gitlab.com/pricing/) will also see +the `health_status` parameter: + +```json +[ + { + "project_id" : 4, + "description" : "Omnis vero earum sunt corporis dolor et placeat.", + "health_status": "on_track", + ... + } +] +``` + NOTE: **Note:** At least one of following parameters is required to be passed for the request to be successful: `:assignee_id`, `:assignee_ids`, `:confidential`, `:created_at`, `:description`, `:discussion_locked`, `:due_date`, `:labels`, `:milestone_id`, `:state_event`, or `:title`. @@ -902,7 +986,7 @@ curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://git ## Reorder an issue -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/211864) as a [community contribution](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/35349) in GitLab 13.2. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/211864) in GitLab 13.2. Reorders an issue, you can see the results when sorting issues manually @@ -988,7 +1072,7 @@ Example response: "web_url": "https://gitlab.example.com/solon.cremin" }, "due_date": null, - "web_url": "http://example.com/my-group/my-project/issues/11", + "web_url": "http://gitlab.example.com/my-group/my-project/issues/11", "references": { "short": "#11", "relative": "#11", @@ -1003,10 +1087,10 @@ Example response: "confidential": false, "discussion_locked": false, "_links": { - "self": "http://example.com/api/v4/projects/1/issues/2", - "notes": "http://example.com/api/v4/projects/1/issues/2/notes", - "award_emoji": "http://example.com/api/v4/projects/1/issues/2/award_emoji", - "project": "http://example.com/api/v4/projects/1" + "self": "http://gitlab.example.com/api/v4/projects/1/issues/2", + "notes": "http://gitlab.example.com/api/v4/projects/1/issues/2/notes", + "award_emoji": "http://gitlab.example.com/api/v4/projects/1/issues/2/award_emoji", + "project": "http://gitlab.example.com/api/v4/projects/1" }, "task_completion_status":{ "count":0, @@ -1027,6 +1111,20 @@ the `weight` parameter: } ``` +Users on GitLab [Ultimate](https://about.gitlab.com/pricing/) will also see +the `health_status` parameter: + +```json +[ + { + "project_id" : 4, + "description" : "Omnis vero earum sunt corporis dolor et placeat.", + "health_status": "on_track", + ... + } +] +``` + **Note**: `assignee` column is deprecated, now we show it as a single-sized array `assignees` to conform to the GitLab EE API. **Note**: The `closed_by` attribute was [introduced in GitLab 10.6](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/17042). This value will only be present for issues which were closed after GitLab 10.6 and when the user account that closed the issue still exists. @@ -1094,7 +1192,7 @@ Example response: "web_url": "https://gitlab.example.com/solon.cremin" }, "due_date": null, - "web_url": "http://example.com/my-group/my-project/issues/11", + "web_url": "http://gitlab.example.com/my-group/my-project/issues/11", "references": { "short": "#11", "relative": "#11", @@ -1109,10 +1207,10 @@ Example response: "confidential": false, "discussion_locked": false, "_links": { - "self": "http://example.com/api/v4/projects/1/issues/2", - "notes": "http://example.com/api/v4/projects/1/issues/2/notes", - "award_emoji": "http://example.com/api/v4/projects/1/issues/2/award_emoji", - "project": "http://example.com/api/v4/projects/1" + "self": "http://gitlab.example.com/api/v4/projects/1/issues/2", + "notes": "http://gitlab.example.com/api/v4/projects/1/issues/2/notes", + "award_emoji": "http://gitlab.example.com/api/v4/projects/1/issues/2/award_emoji", + "project": "http://gitlab.example.com/api/v4/projects/1" }, "task_completion_status":{ "count":0, @@ -1193,7 +1291,7 @@ Example response: }, "subscribed": false, "due_date": null, - "web_url": "http://example.com/my-group/my-project/issues/12", + "web_url": "http://gitlab.example.com/my-group/my-project/issues/12", "references": { "short": "#12", "relative": "#12", @@ -1300,7 +1398,7 @@ Example response: "downvotes": 0, "merge_requests_count": 0, "due_date": null, - "web_url": "http://example.com/my-group/my-project/issues/10", + "web_url": "http://gitlab.example.com/my-group/my-project/issues/10", "references": { "short": "#10", "relative": "#10", @@ -1732,7 +1830,7 @@ Example response: "username": "user1", "state": "active", "avatar_url": "http://www.gravatar.com/avatar/c922747a93b40d1ea88262bf1aebee62?s=80&d=identicon", - "web_url": "http://localhost/user1" + "web_url": "http://gitlab.example.com/user1" }, { "id": 5, @@ -1740,7 +1838,7 @@ Example response: "username": "user5", "state": "active", "avatar_url": "http://www.gravatar.com/avatar/4aea8cf834ed91844a2da4ff7ae6b491?s=80&d=identicon", - "web_url": "http://localhost/user5" + "web_url": "http://gitlab.example.com/user5" } ] ``` @@ -1775,3 +1873,8 @@ Example response: "akismet_submitted": false } ``` + +## List issue state events + +To track which state was set, who did it, and when it happened, check out +[Resource state events API](./resource_state_events.md#issues). diff --git a/doc/api/job_artifacts.md b/doc/api/job_artifacts.md new file mode 100644 index 00000000000..5df7915ad5c --- /dev/null +++ b/doc/api/job_artifacts.md @@ -0,0 +1,265 @@ +# Job Artifacts API + +## Get job artifacts + +> The use of `CI_JOB_TOKEN` in the artifacts download API was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/2346) in [GitLab Premium](https://about.gitlab.com/pricing/) 9.5. + +Get the job's artifacts zipped archive of a project. + +```plaintext +GET /projects/:id/jobs/:job_id/artifacts +``` + +| Attribute | Type | Required | Description | +|-------------|----------------|----------|-------------------------------------------------------------------------------------------------------------------------------------------------| +| `id` | integer/string | yes | ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user. | +| `job_id` | integer | yes | ID of a job. | +| `job_token` **(PREMIUM)** | string | no | To be used with [triggers](../ci/triggers/README.md#when-a-pipeline-depends-on-the-artifacts-of-another-pipeline-premium) for multi-project pipelines. It should be invoked only inside `.gitlab-ci.yml`. Its value is always `$CI_JOB_TOKEN`. | + +Example request using the `PRIVATE-TOKEN` header: + +```shell +curl --output artifacts.zip --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/jobs/42/artifacts" +``` + +To use this in a [`script` definition](../ci/yaml/README.md#script) inside +`.gitlab-ci.yml` **(PREMIUM)**, you can use either: + +- The `JOB-TOKEN` header with the GitLab-provided `CI_JOB_TOKEN` variable. + For example, the following job will download the artifacts of the job with ID + `42`. Note that the command is wrapped into single quotes since it contains a + colon (`:`): + + ```yaml + artifact_download: + stage: test + script: + - 'curl --location --output artifacts.zip --header "JOB-TOKEN: $CI_JOB_TOKEN" "https://gitlab.example.com/api/v4/projects/1/jobs/42/artifacts"' + ``` + +- Or the `job_token` attribute with the GitLab-provided `CI_JOB_TOKEN` variable. + For example, the following job will download the artifacts of the job with ID `42`: + + ```yaml + artifact_download: + stage: test + script: + - 'curl --location --output artifacts.zip "https://gitlab.example.com/api/v4/projects/1/jobs/42/artifacts?job_token=$CI_JOB_TOKEN"' + ``` + +Possible response status codes: + +| Status | Description | +|-----------|---------------------------------| +| 200 | Serves the artifacts file. | +| 404 | Build not found or no artifacts.| + +## Download the artifacts archive + +> The use of `CI_JOB_TOKEN` in the artifacts download API was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/2346) in [GitLab Premium](https://about.gitlab.com/pricing/) 9.5. + +Download the artifacts zipped archive from the latest successful pipeline for +the given reference name and job, provided the job finished successfully. This +is the same as [getting the job's artifacts](#get-job-artifacts), but by +defining the job's name instead of its ID. + +```plaintext +GET /projects/:id/jobs/artifacts/:ref_name/download?job=name +``` + +Parameters + +| Attribute | Type | Required | Description | +|-------------|----------------|----------|-------------------------------------------------------------------------------------------------------------------------------------------------| +| `id` | integer/string | yes | ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user. | +| `ref_name` | string | yes | Branch or tag name in repository. HEAD or SHA references are not supported. | +| `job` | string | yes | The name of the job. | +| `job_token` **(PREMIUM)** | string | no | To be used with [triggers](../ci/triggers/README.md#when-a-pipeline-depends-on-the-artifacts-of-another-pipeline-premium) for multi-project pipelines. It should be invoked only inside `.gitlab-ci.yml`. Its value is always `$CI_JOB_TOKEN`. | + +Example request using the `PRIVATE-TOKEN` header: + +```shell +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/jobs/artifacts/master/download?job=test" +``` + +To use this in a [`script` definition](../ci/yaml/README.md#script) inside +`.gitlab-ci.yml` **(PREMIUM)**, you can use either: + +- The `JOB-TOKEN` header with the GitLab-provided `CI_JOB_TOKEN` variable. + For example, the following job will download the artifacts of the `test` job + of the `master` branch. Note that the command is wrapped into single quotes + since it contains a colon (`:`): + + ```yaml + artifact_download: + stage: test + script: + - 'curl --location --output artifacts.zip --header "JOB-TOKEN: $CI_JOB_TOKEN" "https://gitlab.example.com/api/v4/projects/$CI_PROJECT_ID/jobs/artifacts/master/download?job=test"' + ``` + +- Or the `job_token` attribute with the GitLab-provided `CI_JOB_TOKEN` variable. + For example, the following job will download the artifacts of the `test` job + of the `master` branch: + + ```yaml + artifact_download: + stage: test + script: + - 'curl --location --output artifacts.zip "https://gitlab.example.com/api/v4/projects/$CI_PROJECT_ID/jobs/artifacts/master/download?job=test&job_token=$CI_JOB_TOKEN"' + ``` + +Possible response status codes: + +| Status | Description | +|-----------|---------------------------------| +| 200 | Serves the artifacts file. | +| 404 | Build not found or no artifacts.| + +## Download a single artifact file by job ID + +> Introduced in GitLab 10.0 + +Download a single artifact file from a job with a specified ID from within +the job's artifacts zipped archive. The file is extracted from the archive and +streamed to the client. + +```plaintext +GET /projects/:id/jobs/:job_id/artifacts/*artifact_path +``` + +Parameters + +| Attribute | Type | Required | Description | +|-----------------|----------------|----------|------------------------------------------------------------------------------------------------------------------| +| `id` | integer/string | yes | ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user. | +| `job_id` | integer | yes | The unique job identifier. | +| `artifact_path` | string | yes | Path to a file inside the artifacts archive. | + +Example request: + +```shell +curl --location --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/jobs/5/artifacts/some/release/file.pdf" +``` + +Possible response status codes: + +| Status | Description | +|-----------|--------------------------------------| +| 200 | Sends a single artifact file | +| 400 | Invalid path provided | +| 404 | Build not found or no file/artifacts | + +## Download a single artifact file from specific tag or branch + +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/23538) in GitLab 11.5. + +Download a single artifact file for a specific job of the latest successful +pipeline for the given reference name from within the job's artifacts archive. +The file is extracted from the archive and streamed to the client. + +```plaintext +GET /projects/:id/jobs/artifacts/:ref_name/raw/*artifact_path?job=name +``` + +Parameters: + +| Attribute | Type | Required | Description | +|-----------------|----------------|----------|------------------------------------------------------------------------------------------------------------------| +| `id` | integer/string | yes | ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user. | +| `ref_name` | string | yes | Branch or tag name in repository. HEAD or SHA references are not supported. | +| `artifact_path` | string | yes | Path to a file inside the artifacts archive. | +| `job` | string | yes | The name of the job. | + +Example request: + +```shell +curl --location --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/jobs/artifacts/master/raw/some/release/file.pdf?job=pdf" +``` + +Possible response status codes: + +| Status | Description | +|-----------|--------------------------------------| +| 200 | Sends a single artifact file | +| 400 | Invalid path provided | +| 404 | Build not found or no file/artifacts | + +## Keep artifacts + +Prevents artifacts from being deleted when expiration is set. + +```plaintext +POST /projects/:id/jobs/:job_id/artifacts/keep +``` + +Parameters + +| Attribute | Type | Required | Description | +|-----------|----------------|----------|------------------------------------------------------------------------------------------------------------------| +| `id` | integer/string | yes | ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user. | +| `job_id` | integer | yes | ID of a job. | + +Example request: + +```shell +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/jobs/1/artifacts/keep" +``` + +Example response: + +```json +{ + "commit": { + "author_email": "admin@example.com", + "author_name": "Administrator", + "created_at": "2015-12-24T16:51:14.000+01:00", + "id": "0ff3ae198f8601a285adcf5c0fff204ee6fba5fd", + "message": "Test the CI integration.", + "short_id": "0ff3ae19", + "title": "Test the CI integration." + }, + "coverage": null, + "allow_failure": false, + "download_url": null, + "id": 42, + "name": "rubocop", + "ref": "master", + "artifacts": [], + "runner": null, + "stage": "test", + "created_at": "2016-01-11T10:13:33.506Z", + "started_at": "2016-01-11T10:13:33.506Z", + "finished_at": "2016-01-11T10:15:10.506Z", + "duration": 97.0, + "status": "failed", + "tag": false, + "web_url": "https://example.com/foo/bar/-/jobs/42", + "user": null +} +``` + +## Delete artifacts + +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/25522) in GitLab 11.9. + +Delete artifacts of a job. + +```plaintext +DELETE /projects/:id/jobs/:job_id/artifacts +``` + +| Attribute | Type | Required | Description | +|-----------|----------------|----------|------------------------------------------------------------------------------------------------------------------| +| `id` | integer/string | yes | ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) | +| `job_id` | integer | yes | ID of a job. | + +Example request: + +```shell +curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/jobs/1/artifacts" +``` + +NOTE: **Note:** +At least Maintainer role is required to delete artifacts. + +If the artifacts were deleted successfully, a response with status `204 No Content` is returned. diff --git a/doc/api/jobs.md b/doc/api/jobs.md index 4dc29fc897d..054260794c7 100644 --- a/doc/api/jobs.md +++ b/doc/api/jobs.md @@ -1,3 +1,9 @@ +--- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Jobs API ## List project jobs @@ -269,6 +275,9 @@ Example of response ] ``` +In GitLab 13.3 and later, this endpoint [returns data for any pipeline](pipelines.md#single-pipeline-requests) +including [child pipelines](../ci/parent_child_pipelines.md). + ## List pipeline bridges Get a list of bridge jobs for a pipeline. @@ -425,198 +434,6 @@ Example of response } ``` -## Get job artifacts - -> **Notes**: -> -> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/2893) in GitLab 8.5. -> - The use of `CI_JOB_TOKEN` in the artifacts download API was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/2346) -> in [GitLab Premium](https://about.gitlab.com/pricing/) 9.5. - -Get the job's artifacts zipped archive of a project. - -```plaintext -GET /projects/:id/jobs/:job_id/artifacts -``` - -| Attribute | Type | Required | Description | -|-------------|----------------|----------|-------------------------------------------------------------------------------------------------------------------------------------------------| -| `id` | integer/string | yes | ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user. | -| `job_id` | integer | yes | ID of a job. | -| `job_token` **(PREMIUM)** | string | no | To be used with [triggers](../ci/triggers/README.md#when-a-pipeline-depends-on-the-artifacts-of-another-pipeline-premium) for multi-project pipelines. It should be invoked only inside `.gitlab-ci.yml`. Its value is always `$CI_JOB_TOKEN`. | - -Example request using the `PRIVATE-TOKEN` header: - -```shell -curl --output artifacts.zip --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/jobs/42/artifacts" -``` - -To use this in a [`script` definition](../ci/yaml/README.md#script) inside -`.gitlab-ci.yml` **(PREMIUM)**, you can use either: - -- The `JOB-TOKEN` header with the GitLab-provided `CI_JOB_TOKEN` variable. - For example, the following job will download the artifacts of the job with ID - `42`. Note that the command is wrapped into single quotes since it contains a - colon (`:`): - - ```yaml - artifact_download: - stage: test - script: - - 'curl --location --output artifacts.zip --header "JOB-TOKEN: $CI_JOB_TOKEN" "https://gitlab.example.com/api/v4/projects/1/jobs/42/artifacts"' - ``` - -- Or the `job_token` attribute with the GitLab-provided `CI_JOB_TOKEN` variable. - For example, the following job will download the artifacts of the job with ID `42`: - - ```yaml - artifact_download: - stage: test - script: - - 'curl --location --output artifacts.zip "https://gitlab.example.com/api/v4/projects/1/jobs/42/artifacts?job_token=$CI_JOB_TOKEN"' - ``` - -Possible response status codes: - -| Status | Description | -|-----------|---------------------------------| -| 200 | Serves the artifacts file. | -| 404 | Build not found or no artifacts.| - -## Download the artifacts archive - -> **Notes**: -> -> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/5347) in GitLab 8.10. -> - The use of `CI_JOB_TOKEN` in the artifacts download API was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/2346) -> in [GitLab Premium](https://about.gitlab.com/pricing/) 9.5. - -Download the artifacts zipped archive from the latest successful pipeline for -the given reference name and job, provided the job finished successfully. This -is the same as [getting the job's artifacts](#get-job-artifacts), but by -defining the job's name instead of its ID. - -```plaintext -GET /projects/:id/jobs/artifacts/:ref_name/download?job=name -``` - -Parameters - -| Attribute | Type | Required | Description | -|-------------|----------------|----------|-------------------------------------------------------------------------------------------------------------------------------------------------| -| `id` | integer/string | yes | ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user. | -| `ref_name` | string | yes | Branch or tag name in repository. HEAD or SHA references are not supported. | -| `job` | string | yes | The name of the job. | -| `job_token` **(PREMIUM)** | string | no | To be used with [triggers](../ci/triggers/README.md#when-a-pipeline-depends-on-the-artifacts-of-another-pipeline-premium) for multi-project pipelines. It should be invoked only inside `.gitlab-ci.yml`. Its value is always `$CI_JOB_TOKEN`. | - -Example request using the `PRIVATE-TOKEN` header: - -```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/jobs/artifacts/master/download?job=test" -``` - -To use this in a [`script` definition](../ci/yaml/README.md#script) inside -`.gitlab-ci.yml` **(PREMIUM)**, you can use either: - -- The `JOB-TOKEN` header with the GitLab-provided `CI_JOB_TOKEN` variable. - For example, the following job will download the artifacts of the `test` job - of the `master` branch. Note that the command is wrapped into single quotes - since it contains a colon (`:`): - - ```yaml - artifact_download: - stage: test - script: - - 'curl --location --output artifacts.zip --header "JOB-TOKEN: $CI_JOB_TOKEN" "https://gitlab.example.com/api/v4/projects/$CI_PROJECT_ID/jobs/artifacts/master/download?job=test"' - ``` - -- Or the `job_token` attribute with the GitLab-provided `CI_JOB_TOKEN` variable. - For example, the following job will download the artifacts of the `test` job - of the `master` branch: - - ```yaml - artifact_download: - stage: test - script: - - 'curl --location --output artifacts.zip "https://gitlab.example.com/api/v4/projects/$CI_PROJECT_ID/jobs/artifacts/master/download?job=test&job_token=$CI_JOB_TOKEN"' - ``` - -Possible response status codes: - -| Status | Description | -|-----------|---------------------------------| -| 200 | Serves the artifacts file. | -| 404 | Build not found or no artifacts.| - -## Download a single artifact file by job ID - -> Introduced in GitLab 10.0 - -Download a single artifact file from a job with a specified ID from within -the job's artifacts zipped archive. The file is extracted from the archive and -streamed to the client. - -```plaintext -GET /projects/:id/jobs/:job_id/artifacts/*artifact_path -``` - -Parameters - -| Attribute | Type | Required | Description | -|-----------------|----------------|----------|------------------------------------------------------------------------------------------------------------------| -| `id` | integer/string | yes | ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user. | -| `job_id` | integer | yes | The unique job identifier. | -| `artifact_path` | string | yes | Path to a file inside the artifacts archive. | - -Example request: - -```shell -curl --location --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/jobs/5/artifacts/some/release/file.pdf" -``` - -Possible response status codes: - -| Status | Description | -|-----------|--------------------------------------| -| 200 | Sends a single artifact file | -| 400 | Invalid path provided | -| 404 | Build not found or no file/artifacts | - -## Download a single artifact file from specific tag or branch - -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/23538) in GitLab 11.5. - -Download a single artifact file for a specific job of the latest successful -pipeline for the given reference name from within the job's artifacts archive. -The file is extracted from the archive and streamed to the client. - -```plaintext -GET /projects/:id/jobs/artifacts/:ref_name/raw/*artifact_path?job=name -``` - -Parameters: - -| Attribute | Type | Required | Description | -|-----------------|----------------|----------|------------------------------------------------------------------------------------------------------------------| -| `id` | integer/string | yes | ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user. | -| `ref_name` | string | yes | Branch or tag name in repository. HEAD or SHA references are not supported. | -| `artifact_path` | string | yes | Path to a file inside the artifacts archive. | -| `job` | string | yes | The name of the job. | - -Example request: - -```shell -curl --location --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/jobs/artifacts/master/raw/some/release/file.pdf?job=pdf" -``` - -Possible response status codes: - -| Status | Description | -|-----------|--------------------------------------| -| 200 | Sends a single artifact file | -| 400 | Invalid path provided | -| 404 | Build not found or no file/artifacts | - ## Get a log file Get a log (trace) of a specific job of a project: @@ -793,86 +610,6 @@ Example of response } ``` -## Keep artifacts - -Prevents artifacts from being deleted when expiration is set. - -```plaintext -POST /projects/:id/jobs/:job_id/artifacts/keep -``` - -Parameters - -| Attribute | Type | Required | Description | -|-----------|----------------|----------|------------------------------------------------------------------------------------------------------------------| -| `id` | integer/string | yes | ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user. | -| `job_id` | integer | yes | ID of a job. | - -Example request: - -```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/jobs/1/artifacts/keep" -``` - -Example response: - -```json -{ - "commit": { - "author_email": "admin@example.com", - "author_name": "Administrator", - "created_at": "2015-12-24T16:51:14.000+01:00", - "id": "0ff3ae198f8601a285adcf5c0fff204ee6fba5fd", - "message": "Test the CI integration.", - "short_id": "0ff3ae19", - "title": "Test the CI integration." - }, - "coverage": null, - "allow_failure": false, - "download_url": null, - "id": 42, - "name": "rubocop", - "ref": "master", - "artifacts": [], - "runner": null, - "stage": "test", - "created_at": "2016-01-11T10:13:33.506Z", - "started_at": "2016-01-11T10:13:33.506Z", - "finished_at": "2016-01-11T10:15:10.506Z", - "duration": 97.0, - "status": "failed", - "tag": false, - "web_url": "https://example.com/foo/bar/-/jobs/42", - "user": null -} -``` - -## Delete artifacts - -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/25522) in GitLab 11.9. - -Delete artifacts of a job. - -```plaintext -DELETE /projects/:id/jobs/:job_id/artifacts -``` - -| Attribute | Type | Required | Description | -|-----------|----------------|----------|------------------------------------------------------------------------------------------------------------------| -| `id` | integer/string | yes | ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) | -| `job_id` | integer | yes | ID of a job. | - -Example request: - -```shell -curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/jobs/1/artifacts" -``` - -NOTE: **Note:** -At least Maintainer role is required to delete artifacts. - -If the artifacts were deleted successfully, a response with status `204 No Content` is returned. - ## Play a job Triggers a manual action to start a job. diff --git a/doc/api/keys.md b/doc/api/keys.md index 6294ac300ce..b9e45c23e77 100644 --- a/doc/api/keys.md +++ b/doc/api/keys.md @@ -1,3 +1,10 @@ +--- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference, api +--- + # Keys API ## Get SSH key with user by ID of an SSH key diff --git a/doc/api/lint.md b/doc/api/lint.md index b5889884e48..f4d8a0bc011 100644 --- a/doc/api/lint.md +++ b/doc/api/lint.md @@ -1,3 +1,9 @@ +--- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Validate the `.gitlab-ci.yml` (API) > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/5953) in GitLab 8.12. diff --git a/doc/api/markdown.md b/doc/api/markdown.md index 32810ee349e..4e5c8515126 100644 --- a/doc/api/markdown.md +++ b/doc/api/markdown.md @@ -1,7 +1,8 @@ --- -stage: Plan -group: Project Management -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference, api --- # Markdown API diff --git a/doc/api/members.md b/doc/api/members.md index 8cd7bafdd77..90c36a0b822 100644 --- a/doc/api/members.md +++ b/doc/api/members.md @@ -16,6 +16,12 @@ Due to [an issue](https://gitlab.com/gitlab-org/gitlab/-/issues/219299), projects in personal namespaces will not show owner (`50`) permission for owner. +## Limitations + +The `group_saml_identity` attribute is only visible to a group owner for [SSO enabled groups](../user/group/saml_sso/index.md). + +The `email` attribute is only visible to a group owner who manages the user through [Group Managed Accounts](../user/group/saml_sso/group_managed_accounts.md). + ## List all members of a group or project Gets a list of group or project members viewable by the authenticated user. @@ -172,6 +178,7 @@ Example response: "avatar_url": "https://www.gravatar.com/avatar/c2525a7f58ae3776070e44c106c48e15?s=80&d=identicon", "web_url": "http://192.168.1.8:3000/root", "access_level": 30, + "email": "john@example.com", "expires_at": null, "group_saml_identity": null } @@ -209,6 +216,7 @@ Example response: "avatar_url": "https://www.gravatar.com/avatar/c2525a7f58ae3776070e44c106c48e15?s=80&d=identicon", "web_url": "http://192.168.1.8:3000/root", "access_level": 30, + "email": "john@example.com", "expires_at": null, "group_saml_identity": null } @@ -247,6 +255,7 @@ Example response: "web_url": "http://192.168.1.8:3000/root", "expires_at": "2012-10-22T14:13:35Z", "access_level": 30, + "email": "john@example.com", "group_saml_identity": null } ``` @@ -284,6 +293,7 @@ Example response: "web_url": "http://192.168.1.8:3000/root", "expires_at": "2012-10-22T14:13:35Z", "access_level": 40, + "email": "john@example.com", "group_saml_identity": null } ``` @@ -320,6 +330,7 @@ Example response: "web_url": "http://192.168.1.8:3000/root", "expires_at": "2012-10-22T14:13:35Z", "access_level": 40, + "email": "john@example.com", "override": true } ``` @@ -356,6 +367,7 @@ Example response: "web_url": "http://192.168.1.8:3000/root", "expires_at": "2012-10-22T14:13:35Z", "access_level": 40, + "email": "john@example.com", "override": false } ``` diff --git a/doc/api/merge_request_approvals.md b/doc/api/merge_request_approvals.md index 746a79e1b8e..643d03b6fb8 100644 --- a/doc/api/merge_request_approvals.md +++ b/doc/api/merge_request_approvals.md @@ -1,3 +1,10 @@ +--- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference, api +--- + # Merge request approvals API **(STARTER)** Configuration for approvals on all Merge Requests (MR) in the project. Must be authenticated for all endpoints. @@ -392,9 +399,11 @@ DELETE /projects/:id/approval_rules/:approval_rule_id ### Change allowed approvers ->**Note:** This API endpoint has been deprecated. Please use Approval Rule API instead. > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/183) in [GitLab Starter](https://about.gitlab.com/pricing/) 10.6. +NOTE: **Note:** +This API endpoint has been deprecated. Please use Approval Rule API instead. + If you are allowed to, you can change approvers and approver groups using the following endpoint: @@ -541,9 +550,11 @@ POST /projects/:id/merge_requests/:merge_request_iid/approvals ### Change allowed approvers for Merge Request ->**Note:** This API endpoint has been deprecated. Please use Approval Rule API instead. > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/183) in [GitLab Starter](https://about.gitlab.com/pricing/) 10.6. +NOTE: **Note:** +This API endpoint has been deprecated. Please use Approval Rule API instead. + If you are allowed to, you can change approvers and approver groups using the following endpoint: diff --git a/doc/api/merge_request_context_commits.md b/doc/api/merge_request_context_commits.md index e9ba31401ef..9b4697390d1 100644 --- a/doc/api/merge_request_context_commits.md +++ b/doc/api/merge_request_context_commits.md @@ -1,3 +1,10 @@ +--- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference, api +--- + # Merge request context commits API ## List MR context commits diff --git a/doc/api/merge_requests.md b/doc/api/merge_requests.md index 959cf87ba62..4798145e837 100644 --- a/doc/api/merge_requests.md +++ b/doc/api/merge_requests.md @@ -1,3 +1,10 @@ +--- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference, api +--- + # Merge requests API Every API call to merge requests must be authenticated. @@ -2449,3 +2456,8 @@ Example response: ## Approvals **(STARTER)** For approvals, please see [Merge Request Approvals](merge_request_approvals.md) + +## List merge request state events + +To track which state was set, who did it, and when it happened, check out +[Resource state events API](./resource_state_events.md#merge-requests). diff --git a/doc/api/milestones.md b/doc/api/milestones.md index b5702c7d6e0..7b4d1cc331d 100644 --- a/doc/api/milestones.md +++ b/doc/api/milestones.md @@ -25,13 +25,14 @@ GET /projects/:id/milestones?search=version Parameters: -| Attribute | Type | Required | Description | -| --------- | ------ | -------- | ----------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | -| `iids[]` | integer array | optional | Return only the milestones having the given `iid` | -| `state` | string | optional | Return only `active` or `closed` milestones | -| `title` | string | optional | Return only the milestones having the given `title` | -| `search` | string | optional | Return only milestones with a title or description matching the provided string | +| Attribute | Type | Required | Description | +| ---------------------------- | ------ | -------- | ----------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | +| `iids[]` | integer array | optional | Return only the milestones having the given `iid` (Note: ignored if `include_parent_milestones` is set as `true`) | +| `state` | string | optional | Return only `active` or `closed` milestones | +| `title` | string | optional | Return only the milestones having the given `title` | +| `search` | string | optional | Return only milestones with a title or description matching the provided string | +| `include_parent_milestones` | boolean | optional | Include group milestones from parent group and its ancestors. Introduced in [GitLab 13.4](https://gitlab.com/gitlab-org/gitlab/-/issues/196066) | ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/milestones" diff --git a/doc/api/namespaces.md b/doc/api/namespaces.md index e38e725fb97..ba59d467bc8 100644 --- a/doc/api/namespaces.md +++ b/doc/api/namespaces.md @@ -31,7 +31,14 @@ Example response: "name": "user1", "path": "user1", "kind": "user", - "full_path": "user1" + "full_path": "user1", + "parent_id": null, + "avatar_url": "https://secure.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon", + "web_url": "https://gitlab.example.com/user1", + "billable_members_count": 1, + "plan": "default", + "trial_ends_on": null, + "trial": false }, { "id": 2, @@ -40,7 +47,13 @@ Example response: "kind": "group", "full_path": "group1", "parent_id": null, - "members_count_with_descendants": 2 + "avatar_url": null, + "web_url": "https://gitlab.example.com/groups/group1", + "members_count_with_descendants": 2, + "billable_members_count": 2, + "plan": "default", + "trial_ends_on": null, + "trial": false }, { "id": 3, @@ -49,7 +62,13 @@ Example response: "kind": "group", "full_path": "foo/bar", "parent_id": 9, - "members_count_with_descendants": 5 + "avatar_url": null, + "web_url": "https://gitlab.example.com/groups/foo/bar", + "members_count_with_descendants": 5, + "billable_members_count": 5, + "plan": "default", + "trial_ends_on": null, + "trial": false } ] ``` @@ -100,7 +119,13 @@ Example response: "kind": "group", "full_path": "twitter", "parent_id": null, - "members_count_with_descendants": 2 + "avatar_url": null, + "web_url": "https://gitlab.example.com/groups/twitter", + "members_count_with_descendants": 2, + "billable_members_count": 2, + "plan": "default", + "trial_ends_on": null, + "trial": false } ] ``` @@ -133,7 +158,13 @@ Example response: "kind": "group", "full_path": "group1", "parent_id": null, - "members_count_with_descendants": 2 + "avatar_url": null, + "web_url": "https://gitlab.example.com/groups/group1", + "members_count_with_descendants": 2, + "billable_members_count": 2, + "plan": "default", + "trial_ends_on": null, + "trial": false } ``` @@ -153,6 +184,12 @@ Example response: "kind": "group", "full_path": "group1", "parent_id": null, - "members_count_with_descendants": 2 + "avatar_url": null, + "web_url": "https://gitlab.example.com/groups/group1", + "members_count_with_descendants": 2, + "billable_members_count": 2, + "plan": "default", + "trial_ends_on": null, + "trial": false } ``` diff --git a/doc/api/notes.md b/doc/api/notes.md index 9a75b950f28..3a68454507a 100644 --- a/doc/api/notes.md +++ b/doc/api/notes.md @@ -7,7 +7,19 @@ Notes are comments on: - Merge requests - Epics **(ULTIMATE)** -This includes system notes, which are notes about changes to the object (for example, when a milestone changes, there will be a corresponding system note). Label notes are not part of this API, but recorded as separate events in [resource label events](resource_label_events.md). +This includes system notes, which are notes about changes to the object (for example, when an +assignee changes, there will be a corresponding system note). + +## Resource events + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/38096) in GitLab 13.3 for state, milestone, and weight events. + +Some system notes are not part of this API, but are recorded as separate events: + +- [Resource label events](resource_label_events.md) +- [Resource state events](resource_state_events.md) +- [Resource milestone events](resource_milestone_events.md) +- [Resource weight events](resource_weight_events.md) **(STARTER)** ## Notes pagination @@ -133,10 +145,11 @@ PUT /projects/:id/issues/:issue_iid/notes/:note_id Parameters: -- `id` (required) - The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) -- `issue_iid` (required) - The IID of an issue -- `note_id` (required) - The ID of a note -- `body` (required) - The content of a note. Limited to 1,000,000 characters. +- `id` (required) - The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). +- `issue_iid` (required) - The IID of an issue. +- `note_id` (required) - The ID of a note. +- `body` (optional) - The content of a note. Limited to 1,000,000 characters. +- `confidential` (optional) - The confidential flag of a note. ```shell curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/issues/11/notes?body=note" diff --git a/doc/api/notification_settings.md b/doc/api/notification_settings.md index ffdf6d34832..8442e371a56 100644 --- a/doc/api/notification_settings.md +++ b/doc/api/notification_settings.md @@ -32,6 +32,7 @@ If the `custom` level is used, specific email events can be controlled. Availabl - `failed_pipeline` - `fixed_pipeline` - `success_pipeline` +- `moved_project` - `new_epic` **(ULTIMATE)** ## Global notification settings @@ -86,7 +87,8 @@ curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab | `failed_pipeline` | boolean | no | Enable/disable this notification | | `fixed_pipeline` | boolean | no | Enable/disable this notification | | `success_pipeline` | boolean | no | Enable/disable this notification | -| `new_epic` | boolean | no | Enable/disable this notification ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/6626) in 11.3) **(ULTIMATE)** | +| `moved_project` | boolean | no | Enable/disable this notification ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/30371) in GitLab 13.3) | +| `new_epic` | boolean | no | Enable/disable this notification ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/5863) in GitLab 11.3) **(ULTIMATE)** | Example response: @@ -156,7 +158,8 @@ curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab | `failed_pipeline` | boolean | no | Enable/disable this notification | | `fixed_pipeline` | boolean | no | Enable/disable this notification | | `success_pipeline` | boolean | no | Enable/disable this notification | -| `new_epic` | boolean | no | Enable/disable this notification ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/6626) in 11.3) **(ULTIMATE)** | +| `moved_project` | boolean | no | Enable/disable this notification ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/30371) in GitLab 13.3) | +| `new_epic` | boolean | no | Enable/disable this notification ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/5863) in GitLab 11.3) **(ULTIMATE)** | Example responses: @@ -187,8 +190,8 @@ Example responses: } ``` -Users on GitLab [Ultimate or Gold](https://about.gitlab.com/pricing/) will also see -the `new_epic` parameter: +Users on GitLab [Ultimate or Gold](https://about.gitlab.com/pricing/) also see the `new_epic` +parameter: ```json { diff --git a/doc/api/packages.md b/doc/api/packages.md index 19828208a26..cf65b518844 100644 --- a/doc/api/packages.md +++ b/doc/api/packages.md @@ -4,7 +4,7 @@ group: Package info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# Packages API **(PREMIUM)** +# Packages API This is the API docs of [GitLab Packages](../administration/packages/index.md). diff --git a/doc/api/personal_access_tokens.md b/doc/api/personal_access_tokens.md new file mode 100644 index 00000000000..517e26f3d85 --- /dev/null +++ b/doc/api/personal_access_tokens.md @@ -0,0 +1,88 @@ +# Personal access tokens API **(ULTIMATE)** + +You can read more about [personal access tokens](../user/profile/personal_access_tokens.md#personal-access-tokens). + +## List personal access tokens + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/227264) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.3. + +Get a list of personal access tokens. + +```plaintext +GET /personal_access_tokens +``` + +| Attribute | Type | required | Description | +|-----------|---------|----------|---------------------| +| `user_id` | integer/string | no | The ID of the user to filter by | + +NOTE: **Note:** +Administrators can use the `user_id` parameter to filter by a user. Non-administrators cannot filter by any user except themselves. Attempting to do so will result in a `401 Unauthorized` response. + +```shell +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/personal_access_tokens" +``` + +```json +[ + { + "id": 4, + "name": "Test Token", + "revoked": false, + "created_at": "2020-07-23T14:31:47.729Z", + "scopes": [ + "api" + ], + "active": true, + "user_id": 24, + "expires_at": null + } +] +``` + +```shell +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/personal_access_tokens?user_id=3" +``` + +```json +[ + { + "id": 4, + "name": "Test Token", + "revoked": false, + "created_at": "2020-07-23T14:31:47.729Z", + "scopes": [ + "api" + ], + "active": true, + "user_id": 3, + "expires_at": null + } +] +``` + +## Revoke a personal access token + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/216004) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.3. + +Revoke a personal access token. + +```plaintext +DELETE /personal_access_tokens/:id +``` + +| Attribute | Type | required | Description | +|-----------|---------|----------|---------------------| +| `id` | integer/string | yes | ID of personal access token | + +NOTE: **Note:** +Non-administrators can revoke their own tokens. Administrators can revoke tokens of any user. + +```shell +curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/personal_access_tokens/<personal_access_token_id>" +``` + +### Responses + +- `204: No Content` if successfully revoked. +- `400 Bad Request` if not revoked successfully. diff --git a/doc/api/pipeline_schedules.md b/doc/api/pipeline_schedules.md index dc16157ef4b..1faa6ef56db 100644 --- a/doc/api/pipeline_schedules.md +++ b/doc/api/pipeline_schedules.md @@ -1,3 +1,9 @@ +--- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Pipeline schedules API You can read more about [pipeline schedules](../ci/pipelines/schedules.md). diff --git a/doc/api/pipeline_triggers.md b/doc/api/pipeline_triggers.md index 1a63a04be71..c46992de51b 100644 --- a/doc/api/pipeline_triggers.md +++ b/doc/api/pipeline_triggers.md @@ -1,3 +1,9 @@ +--- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Pipeline triggers API You can read more about [triggering pipelines through the API](../ci/triggers/README.md). diff --git a/doc/api/pipelines.md b/doc/api/pipelines.md index 563829b8192..dc81ef0e25e 100644 --- a/doc/api/pipelines.md +++ b/doc/api/pipelines.md @@ -1,5 +1,19 @@ +--- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Pipelines API +## Single Pipeline Requests + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/36494) in GitLab 13.3. + +Endpoints that request information about a single pipeline return data for any pipeline. +Before 13.3, requests for [child pipelines](../ci/parent_child_pipelines.md) returned +a 404 error. + ## Pipelines pagination By default, `GET` requests return 20 results at a time because the API results @@ -19,7 +33,7 @@ GET /projects/:id/pipelines |-----------|---------|----------|---------------------| | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | | `scope` | string | no | The scope of pipelines, one of: `running`, `pending`, `finished`, `branches`, `tags` | -| `status` | string | no | The status of pipelines, one of: `running`, `pending`, `success`, `failed`, `canceled`, `skipped`, `created`, `manual` | +| `status` | string | no | The status of pipelines, one of: `created`, `waiting_for_resource`, `preparing`, `pending`, `running`, `success`, `failed`, `canceled`, `skipped`, `manual`, `scheduled` | | `ref` | string | no | The ref of pipelines | | `sha` | string | no | The SHA of pipelines | | `yaml_errors`| boolean | no | Returns pipelines with invalid configurations | @@ -205,7 +219,7 @@ POST /projects/:id/pipeline |-------------|---------|----------|---------------------| | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | | `ref` | string | yes | Reference to commit | -| `variables` | array | no | An array containing the variables available in the pipeline, matching the structure `[{ 'key' => 'UPLOAD_TO_S3', 'variable_type' => 'file', 'value' => 'true' }]` | +| `variables` | array | no | An array containing the variables available in the pipeline, matching the structure `[{ 'key': 'UPLOAD_TO_S3', 'variable_type': 'file', 'value': 'true' }]` | ```shell curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/pipeline?ref=master" diff --git a/doc/api/project_aliases.md b/doc/api/project_aliases.md index d80decfe53c..cfd225639f9 100644 --- a/doc/api/project_aliases.md +++ b/doc/api/project_aliases.md @@ -1,3 +1,10 @@ +--- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference, api +--- + # Project Aliases API **(PREMIUM ONLY)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/3264) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.1. diff --git a/doc/api/project_badges.md b/doc/api/project_badges.md index 86b1ba6ce19..936bd40d1ee 100644 --- a/doc/api/project_badges.md +++ b/doc/api/project_badges.md @@ -1,3 +1,10 @@ +--- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference, api +--- + # Project badges API > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/17082) in GitLab 10.6. diff --git a/doc/api/project_import_export.md b/doc/api/project_import_export.md index 74902a22594..5565eaa97f7 100644 --- a/doc/api/project_import_export.md +++ b/doc/api/project_import_export.md @@ -1,3 +1,10 @@ +--- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference, api +--- + # Project import/export API > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/41899) in GitLab 10.6. diff --git a/doc/api/project_level_variables.md b/doc/api/project_level_variables.md index 407e506e082..4760816f5d0 100644 --- a/doc/api/project_level_variables.md +++ b/doc/api/project_level_variables.md @@ -1,3 +1,10 @@ +--- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: reference, api +--- + # Project-level Variables API ## List project variables @@ -148,8 +155,10 @@ curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://git > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/34490) in GitLab 13.2. > - It's deployed behind a feature flag, disabled by default. -> - It's disabled on GitLab.com. -> - To use it in GitLab self-managed instances, ask a GitLab administrator to enable it. +> - [Became enabled by default](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/39209) on GitLab 13.3. +> - It's enabled on GitLab.com. +> - It's recommended for production use. +> - For GitLab self-managed instances, GitLab administrators can opt to [disable it](#enable-or-disable). This parameter is used for filtering by attributes, such as `environment_scope`. @@ -161,17 +170,18 @@ curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://git ### Enable or disable +It is deployed behind a feature flag that is **enabled by default**. [GitLab administrators with access to the GitLab Rails console](../administration/feature_flags.md) -can enable it for your instance. +can opt to disable it for your instance. -To enable it: +To disable it: ```ruby -Feature.enable(:ci_variables_api_filter_environment_scope) +Feature.disable(:ci_variables_api_filter_environment_scope) ``` -To disable it: +To enable it: ```ruby -Feature.disable(:ci_variables_api_filter_environment_scope) +Feature.enable(:ci_variables_api_filter_environment_scope) ``` diff --git a/doc/api/project_snippets.md b/doc/api/project_snippets.md index fd8cbd6e256..eccc8b4212d 100644 --- a/doc/api/project_snippets.md +++ b/doc/api/project_snippets.md @@ -1,3 +1,10 @@ +--- +stage: Create +group: Editor +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference, api +--- + # Project snippets ## Snippet visibility level diff --git a/doc/api/project_statistics.md b/doc/api/project_statistics.md index d96d3de6a73..344aeaa588f 100644 --- a/doc/api/project_statistics.md +++ b/doc/api/project_statistics.md @@ -1,3 +1,10 @@ +--- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference, api +--- + # Project statistics API Every API call to [project](../user/project/index.md) statistics must be authenticated. diff --git a/doc/api/project_templates.md b/doc/api/project_templates.md index e9658423b63..e08ff56925e 100644 --- a/doc/api/project_templates.md +++ b/doc/api/project_templates.md @@ -1,3 +1,10 @@ +--- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference, api +--- + # Project templates API This API is a project-specific version of these endpoints: @@ -6,15 +13,14 @@ This API is a project-specific version of these endpoints: - [Gitignore templates](templates/gitignores.md) - [GitLab CI/CD Configuration templates](templates/gitlab_ci_ymls.md) - [Open source license templates](templates/licenses.md) +- [Issue and merge request templates](../user/project/description_templates.md) + ([introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/37890) in GitLab 13.3) It deprecates these endpoints, which will be removed for API version 5. In addition to templates common to the entire instance, project-specific templates are also available from this API endpoint. -Support will be added for [Issue and Merge Request templates](../user/project/description_templates.md) -in a future release. - Support for [Group-level file templates](../user/group/index.md#group-file-templates-premium) **(PREMIUM)** was [added](https://gitlab.com/gitlab-org/gitlab/-/issues/5987) in GitLab 11.5 @@ -28,7 +34,7 @@ GET /projects/:id/templates/:type | Attribute | Type | Required | Description | | ---------- | ------ | -------- | ----------- | | `id` | integer / string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) | -| `type` | string | yes| The type `(dockerfiles|gitignores|gitlab_ci_ymls|licenses)` of the template | +| `type` | string | yes| The type `(dockerfiles|gitignores|gitlab_ci_ymls|licenses|issues|merge_requests)` of the template | Example response (licenses): @@ -94,7 +100,7 @@ GET /projects/:id/templates/:type/:key | Attribute | Type | Required | Description | | ---------- | ------ | -------- | ----------- | | `id` | integer / string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) | -| `type` | string | yes| The type `(dockerfiles|gitignores|gitlab_ci_ymls|licenses)` of the template | +| `type` | string | yes| The type `(dockerfiles|gitignores|gitlab_ci_ymls|licenses|issues|merge_requests)` of the template | | `key` | string | yes | The key of the template, as obtained from the collection endpoint | | `project` | string | no | The project name to use when expanding placeholders in the template. Only affects licenses | | `fullname` | string | no | The full name of the copyright holder to use when expanding placeholders in the template. Only affects licenses | diff --git a/doc/api/project_vulnerabilities.md b/doc/api/project_vulnerabilities.md index c4f1adccd3c..8ab02ed9ea6 100644 --- a/doc/api/project_vulnerabilities.md +++ b/doc/api/project_vulnerabilities.md @@ -1,3 +1,10 @@ +--- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference, api +--- + # Project Vulnerabilities API **(ULTIMATE)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/10242) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.6. diff --git a/doc/api/projects.md b/doc/api/projects.md index 6257f37f0e6..ee9779b54e0 100644 --- a/doc/api/projects.md +++ b/doc/api/projects.md @@ -1085,7 +1085,7 @@ POST /projects | `template_project_id` | integer | no | **(PREMIUM)** When used with `use_custom_template`, project ID of a custom project template. This is preferable to using `template_name` since `template_name` may be ambiguous. | | `use_custom_template` | boolean | no | **(PREMIUM)** Use either custom [instance](../user/admin_area/custom_project_templates.md) or [group](../user/group/custom_project_templates.md) (with `group_with_project_templates_id`) project template | | `group_with_project_templates_id` | integer | no | **(PREMIUM)** For group-level custom templates, specifies ID of group from which all the custom project templates are sourced. Leave empty for instance-level templates. Requires `use_custom_template` to be true | -| `packages_enabled` | boolean | no | **(PREMIUM ONLY)** Enable or disable packages repository feature | +| `packages_enabled` | boolean | no | Enable or disable packages repository feature | NOTE: **Note:** If your HTTP repository is not publicly accessible, @@ -1156,7 +1156,7 @@ POST /projects/user/:user_id | `template_name` | string | no | When used without `use_custom_template`, name of a [built-in project template](../gitlab-basics/create-project.md#built-in-templates). When used with `use_custom_template`, name of a custom project template | | `use_custom_template` | boolean | no | **(PREMIUM)** Use either custom [instance](../user/admin_area/custom_project_templates.md) or [group](../user/group/custom_project_templates.md) (with `group_with_project_templates_id`) project template | | `group_with_project_templates_id` | integer | no | **(PREMIUM)** For group-level custom templates, specifies ID of group from which all the custom project templates are sourced. Leave empty for instance-level templates. Requires `use_custom_template` to be true | -| `packages_enabled` | boolean | no | **(PREMIUM ONLY)** Enable or disable packages repository feature | +| `packages_enabled` | boolean | no | Enable or disable packages repository feature | NOTE: **Note:** If your HTTP repository is not publicly accessible, @@ -1227,8 +1227,8 @@ PUT /projects/:id | `mirror_trigger_builds` | boolean | no | **(STARTER)** Pull mirroring triggers builds | | `only_mirror_protected_branches` | boolean | no | **(STARTER)** Only mirror protected branches | | `mirror_overwrites_diverged_branches` | boolean | no | **(STARTER)** Pull mirror overwrites diverged branches | -| `packages_enabled` | boolean | no | **(PREMIUM ONLY)** Enable or disable packages repository feature | -| `service_desk_enabled` | boolean | no | Enable or disable service desk feature | +| `packages_enabled` | boolean | no | Enable or disable packages repository feature | +| `service_desk_enabled` | boolean | no | Enable or disable Service Desk feature | NOTE: **Note:** If your HTTP repository is not publicly accessible, @@ -1258,7 +1258,7 @@ POST /projects/:id/fork ## List Forks of a project ->**Note:** This feature was introduced in GitLab 10.1 +> Introduced in GitLab 10.1. List the projects accessible to the calling user that have an established, forked relationship with the specified project @@ -1832,16 +1832,16 @@ Example response: } ``` -## Remove project +## Delete project This endpoint: -- Removes a project including all associated resources (issues, merge requests etc). +- Deletes a project including all associated resources (issues, merge requests etc). - From [GitLab 13.2](https://gitlab.com/gitlab-org/gitlab/-/issues/220382) on [Premium or Silver](https://about.gitlab.com/pricing/) or higher tiers, group admins can [configure](../user/group/index.md#enabling-delayed-project-removal-premium) projects within a group to be deleted after a delayed period. When enabled, actual deletion happens after the number of days -specified in the [default deletion period](../user/admin_area/settings/visibility_and_access_controls.md#default-deletion-adjourned-period-premium-only). +specified in the [default deletion delay](../user/admin_area/settings/visibility_and_access_controls.md#default-deletion-delay-premium-only). CAUTION: **Warning:** The default behavior of [Delayed Project deletion](https://gitlab.com/gitlab-org/gitlab/-/issues/32935) in GitLab 12.6 @@ -1985,6 +1985,7 @@ GET /projects/:id/hooks/:hook_id "job_events": true, "pipeline_events": true, "wiki_page_events": true, + "deployment_events": true, "enable_ssl_verification": true, "created_at": "2012-10-12T17:04:47Z" } @@ -2013,6 +2014,7 @@ POST /projects/:id/hooks | `job_events` | boolean | no | Trigger hook on job events | | `pipeline_events` | boolean | no | Trigger hook on pipeline events | | `wiki_page_events` | boolean | no | Trigger hook on wiki events | +| `deployment_events` | boolean | no | Trigger hook on deployment events | | `enable_ssl_verification` | boolean | no | Do SSL verification when triggering the hook | | `token` | string | no | Secret token to validate received payloads; this will not be returned in the response | @@ -2040,6 +2042,7 @@ PUT /projects/:id/hooks/:hook_id | `job_events` | boolean | no | Trigger hook on job events | | `pipeline_events` | boolean | no | Trigger hook on pipeline events | | `wiki_events` | boolean | no | Trigger hook on wiki events | +| `deployment_events` | boolean | no | Trigger hook on deployment events | | `enable_ssl_verification` | boolean | no | Do SSL verification when triggering the hook | | `token` | string | no | Secret token to validate received payloads; this will not be returned in the response | @@ -2121,7 +2124,7 @@ POST /projects/:id/housekeeping ### Get project push rules -Get the push rules of a project. +Get the [push rules](../push_rules/push_rules.md#enabling-push-rules) of a project. ```plaintext GET /projects/:id/push_rule diff --git a/doc/api/protected_branches.md b/doc/api/protected_branches.md index 4206fe6a565..1f4f1eb933a 100644 --- a/doc/api/protected_branches.md +++ b/doc/api/protected_branches.md @@ -1,6 +1,13 @@ +--- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference, api +--- + # Protected branches API ->**Note:** This feature was introduced in GitLab 9.5 +> Introduced in GitLab 9.5. **Valid access levels** diff --git a/doc/api/protected_tags.md b/doc/api/protected_tags.md index 01de19f54ea..9f9c1ad8b5d 100644 --- a/doc/api/protected_tags.md +++ b/doc/api/protected_tags.md @@ -1,6 +1,13 @@ +--- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference, api +--- + # Protected tags API ->**Note:** This feature was introduced in GitLab 11.3 +> Introduced in GitLab 11.3. **Valid access levels** diff --git a/doc/api/releases/index.md b/doc/api/releases/index.md index 2c933061c37..357f7e7a125 100644 --- a/doc/api/releases/index.md +++ b/doc/api/releases/index.md @@ -360,7 +360,7 @@ POST /projects/:id/releases | `name` | string | no | The release name. | | `tag_name` | string | yes | The tag where the release will be created from. | | `description` | string | no | The description of the release. You can use [Markdown](../../user/markdown.md). | -| `ref` | string | yes, if `tag_name` doesn't exist | If `tag_name` doesn't exist, the release will be created from `ref`. It can be a commit SHA, another tag name, or a branch name. | +| `ref` | string | yes, if `tag_name` doesn't exist | If a tag specified in `tag_name` doesn't exist, the release will be created from `ref` and tagged with `tag_name`. It can be a commit SHA, another tag name, or a branch name. | | `milestones` | array of string | no | The title of each milestone the release is associated with. | | `assets:links` | array of hash | no | An array of assets links. | | `assets:links:name`| string | required by: `assets:links` | The name of the link. | diff --git a/doc/api/remote_mirrors.md b/doc/api/remote_mirrors.md index 6495f6d8383..a8355fb9009 100644 --- a/doc/api/remote_mirrors.md +++ b/doc/api/remote_mirrors.md @@ -1,3 +1,10 @@ +--- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference, api +--- + # Project remote mirrors API [Push mirrors](../user/project/repository/repository_mirroring.md#pushing-to-a-remote-repository-core) diff --git a/doc/api/repositories.md b/doc/api/repositories.md index 7e601ec96ec..305216f853a 100644 --- a/doc/api/repositories.md +++ b/doc/api/repositories.md @@ -1,3 +1,10 @@ +--- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference, api +--- + # Repositories API ## List repository tree @@ -192,6 +199,9 @@ authentication if the repository is publicly accessible. GET /projects/:id/repository/contributors ``` +CAUTION: **Deprecation:** +The `additions` and `deletions` attributes are deprecated [as of GitLab 13.4](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/39653) because they [always return `0`](https://gitlab.com/gitlab-org/gitlab/-/issues/233119). + Parameters: - `id` (required) - The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user @@ -205,14 +215,14 @@ Response: "name": "Example User", "email": "example@example.com", "commits": 117, - "additions": 2097, - "deletions": 517 + "additions": 0, + "deletions": 0 }, { "name": "Sample User", "email": "sample@example.com", "commits": 33, - "additions": 338, - "deletions": 244 + "additions": 0, + "deletions": 0 }] ``` diff --git a/doc/api/repository_files.md b/doc/api/repository_files.md index 9d934a0f855..cc1f0aa970c 100644 --- a/doc/api/repository_files.md +++ b/doc/api/repository_files.md @@ -1,3 +1,10 @@ +--- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference, api +--- + # Repository files API **CRUD for repository files** diff --git a/doc/api/repository_submodules.md b/doc/api/repository_submodules.md index 40708f5bcb0..9a5dcacbc2f 100644 --- a/doc/api/repository_submodules.md +++ b/doc/api/repository_submodules.md @@ -1,3 +1,10 @@ +--- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference, api +--- + # Repository submodules API > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/41213) in GitLab 11.5 diff --git a/doc/api/resource_milestone_events.md b/doc/api/resource_milestone_events.md index 8a81615857c..146f67527b7 100644 --- a/doc/api/resource_milestone_events.md +++ b/doc/api/resource_milestone_events.md @@ -6,6 +6,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Resource milestone events API +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/31720) in GitLab 13.1. + Resource milestone events keep track of what happens to GitLab [issues](../user/project/issues/) and [merge requests](../user/project/merge_requests/). diff --git a/doc/api/resource_state_events.md b/doc/api/resource_state_events.md index 6b257f10c6e..f93e0408300 100644 --- a/doc/api/resource_state_events.md +++ b/doc/api/resource_state_events.md @@ -6,6 +6,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Resource state events API +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/35210/) in GitLab 13.2. + Resource state events keep track of what happens to GitLab [issues](../user/project/issues/) and [merge requests](../user/project/merge_requests/). diff --git a/doc/api/resource_weight_events.md b/doc/api/resource_weight_events.md index 700ef288440..028878874d2 100644 --- a/doc/api/resource_weight_events.md +++ b/doc/api/resource_weight_events.md @@ -6,6 +6,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Resource weight events API +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/32542) in GitLab 13.2. + Resource weight events keep track of what happens to GitLab [issues](../user/project/issues/). Use them to track which weight was set, who did it, and when it happened. diff --git a/doc/api/scim.md b/doc/api/scim.md index 7c8da37a949..0a5703ad8db 100644 --- a/doc/api/scim.md +++ b/doc/api/scim.md @@ -35,7 +35,7 @@ Pagination follows the [SCIM spec](https://tools.ietf.org/html/rfc7644#section-3 Example request: ```shell -curl 'https://example.gitlab.com/api/scim/v2/groups/test_group/Users?filter=id%20eq%20"0b1d561c-21ff-4092-beab-8154b17f82f2"' --header "Authorization: Bearer <your_scim_token>" --header "Content-Type: application/scim+json" +curl 'https://gitlab.example.com/api/scim/v2/groups/test_group/Users?filter=id%20eq%20"0b1d561c-21ff-4092-beab-8154b17f82f2"' --header "Authorization: Bearer <your_scim_token>" --header "Content-Type: application/scim+json" ``` Example response: @@ -86,7 +86,7 @@ Parameters: Example request: ```shell -curl "https://example.gitlab.com/api/scim/v2/groups/test_group/Users/f0b1d561c-21ff-4092-beab-8154b17f82f2" --header "Authorization: Bearer <your_scim_token>" --header "Content-Type: application/scim+json" +curl "https://gitlab.example.com/api/scim/v2/groups/test_group/Users/f0b1d561c-21ff-4092-beab-8154b17f82f2" --header "Authorization: Bearer <your_scim_token>" --header "Content-Type: application/scim+json" ``` Example response: @@ -130,7 +130,7 @@ Parameters: Example request: ```shell -curl --verbose --request POST "https://example.gitlab.com/api/scim/v2/groups/test_group/Users" --data '{"externalId":"test_uid","active":null,"userName":"username","emails":[{"primary":true,"type":"work","value":"name@example.com"}],"name":{"formatted":"Test User","familyName":"User","givenName":"Test"},"schemas":["urn:ietf:params:scim:schemas:core:2.0:User"],"meta":{"resourceType":"User"}}' --header "Authorization: Bearer <your_scim_token>" --header "Content-Type: application/scim+json" +curl --verbose --request POST "https://gitlab.example.com/api/scim/v2/groups/test_group/Users" --data '{"externalId":"test_uid","active":null,"userName":"username","emails":[{"primary":true,"type":"work","value":"name@example.com"}],"name":{"formatted":"Test User","familyName":"User","givenName":"Test"},"schemas":["urn:ietf:params:scim:schemas:core:2.0:User"],"meta":{"resourceType":"User"}}' --header "Authorization: Bearer <your_scim_token>" --header "Content-Type: application/scim+json" ``` Example response: @@ -184,7 +184,7 @@ Parameters: Example request: ```shell -curl --verbose --request PATCH "https://example.gitlab.com/api/scim/v2/groups/test_group/Users/f0b1d561c-21ff-4092-beab-8154b17f82f2" --data '{ "Operations": [{"op":"Add","path":"name.formatted","value":"New Name"}] }' --header "Authorization: Bearer <your_scim_token>" --header "Content-Type: application/scim+json" +curl --verbose --request PATCH "https://gitlab.example.com/api/scim/v2/groups/test_group/Users/f0b1d561c-21ff-4092-beab-8154b17f82f2" --data '{ "Operations": [{"op":"Add","path":"name.formatted","value":"New Name"}] }' --header "Authorization: Bearer <your_scim_token>" --header "Content-Type: application/scim+json" ``` Returns an empty response with a `204` status code if successful. @@ -207,7 +207,7 @@ Parameters: Example request: ```shell -curl --verbose --request DELETE "https://example.gitlab.com/api/scim/v2/groups/test_group/Users/f0b1d561c-21ff-4092-beab-8154b17f82f2" --header "Authorization: Bearer <your_scim_token>" --header "Content-Type: application/scim+json" +curl --verbose --request DELETE "https://gitlab.example.com/api/scim/v2/groups/test_group/Users/f0b1d561c-21ff-4092-beab-8154b17f82f2" --header "Authorization: Bearer <your_scim_token>" --header "Content-Type: application/scim+json" ``` Returns an empty response with a `204` status code if successful. diff --git a/doc/api/search.md b/doc/api/search.md index 9c4eef7dfe8..4c87a826ca8 100644 --- a/doc/api/search.md +++ b/doc/api/search.md @@ -1,3 +1,10 @@ +--- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference, api +--- + # Search API > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/41763) in GitLab 10.5. @@ -279,7 +286,8 @@ Example response: ] ``` -**Note:** `filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` will be only the file name and not the full path. For details, see [issue 34521](https://gitlab.com/gitlab-org/gitlab/-/issues/34521). +NOTE: **Note:** +`filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` will be only the file name and not the full path. For details, see [issue 34521](https://gitlab.com/gitlab-org/gitlab/-/issues/34521). ### Scope: commits **(STARTER)** @@ -350,7 +358,8 @@ Example response: ] ``` -**Note:** `filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` will be only the file name and not the full path. For details, see [issue 34521](https://gitlab.com/gitlab-org/gitlab/-/issues/34521). +NOTE: **Note:** +`filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` will be only the file name and not the full path. For details, see [issue 34521](https://gitlab.com/gitlab-org/gitlab/-/issues/34521). ### Scope: users @@ -620,7 +629,8 @@ Example response: ] ``` -**Note:** `filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` will be only the file name and not the full path. For details, see [issue 34521](https://gitlab.com/gitlab-org/gitlab/-/issues/34521). +NOTE **Note:** +`filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` will be only the file name and not the full path. For details, see [issue 34521](https://gitlab.com/gitlab-org/gitlab/-/issues/34521). ### Scope: commits **(STARTER)** @@ -691,7 +701,8 @@ Example response: ] ``` -**Note:** `filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` will be only the file name and not the full path. For details, see [issue 34521](https://gitlab.com/gitlab-org/gitlab/-/issues/34521). +NOTE **Note:** +`filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` will be only the file name and not the full path. For details, see [issue 34521](https://gitlab.com/gitlab-org/gitlab/-/issues/34521). ### Scope: users @@ -976,7 +987,8 @@ Example response: ] ``` -**Note:** `filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` will be only the file name and not the full path. For details, see [issue 34521](https://gitlab.com/gitlab-org/gitlab/-/issues/34521). +NOTE: **Note:** +`filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` will be only the file name and not the full path. For details, see [issue 34521](https://gitlab.com/gitlab-org/gitlab/-/issues/34521). ### Scope: commits @@ -1049,7 +1061,8 @@ Example response: ] ``` -**Note:** `filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` will be only the file name and not the full path. For details, see [issue 34521](https://gitlab.com/gitlab-org/gitlab/-/issues/34521). +NOTE: **Note:** +`filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` will be only the file name and not the full path. For details, see [issue 34521](https://gitlab.com/gitlab-org/gitlab/-/issues/34521). ### Scope: users diff --git a/doc/api/services.md b/doc/api/services.md index 4052fd22641..25ad025027a 100644 --- a/doc/api/services.md +++ b/doc/api/services.md @@ -1,6 +1,7 @@ # Services API ->**Note:** This API requires an access token with Maintainer or Owner permissions +NOTE: **Note:** +This API requires an access token with Maintainer or Owner permissions ## List all active services @@ -229,8 +230,8 @@ Parameters: | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `token` | string | true | Buildkite project GitLab token | -| `project_url` | string | true | `https://buildkite.com/example/project` | -| `enable_ssl_verification` | boolean | false | Enable SSL verification | +| `project_url` | string | true | Pipeline URL. For example, `https://buildkite.com/example/pipeline` | +| `enable_ssl_verification` | boolean | false | DEPRECATED: This parameter has no effect since SSL verification will always be enabled | | `push_events` | boolean | false | Enable notifications for push events | ### Delete Buildkite service @@ -636,9 +637,9 @@ GET /projects/:id/services/github ## Hangouts Chat -Google GSuite team collaboration tool. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/20290) in GitLab 11.2. ->**Note:** This service was [introduced in v11.2](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/20290) +Google GSuite team collaboration tool. ### Create/Edit Hangouts Chat service @@ -648,7 +649,8 @@ Set Hangouts Chat service for a project. PUT /projects/:id/services/hangouts-chat ``` ->**Note:** Specific event parameters (for example, `push_events` flag) were [introduced in v10.4](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/11435) +NOTE: **Note:** +Specific event parameters (for example, `push_events` flag) were [introduced in v10.4](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/11435) Parameters: @@ -1151,7 +1153,8 @@ Set Slack service for a project. PUT /projects/:id/services/slack ``` ->**Note:** Specific event parameters (for example, `push_events` flag and `push_channel`) were [introduced in v10.4](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/11435) +NOTE: **Note:** +Specific event parameters (for example, `push_events` flag and `push_channel`) were [introduced in v10.4](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/11435) Parameters: @@ -1260,7 +1263,8 @@ Set Mattermost service for a project. PUT /projects/:id/services/mattermost ``` ->**Note:** Specific event parameters (for example, `push_events` flag and `push_channel`) were [introduced in v10.4](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/11435) +NOTE: **Note:** +Specific event parameters (for example, `push_events` flag and `push_channel`) were [introduced in v10.4](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/11435) Parameters: diff --git a/doc/api/settings.md b/doc/api/settings.md index d87a3c72a7e..64c529b0222 100644 --- a/doc/api/settings.md +++ b/doc/api/settings.md @@ -71,8 +71,10 @@ Example response: "asset_proxy_url": "https://assets.example.com", "asset_proxy_whitelist": ["example.com", "*.example.com", "your-instance.com"], "npm_package_requests_forwarding": true, + "snippet_size_limit": 52428800, "issues_create_limit": 300, - "raw_blob_request_limit": 300 + "raw_blob_request_limit": 300, + "wiki_page_max_content_bytes": 52428800 } ``` @@ -161,8 +163,10 @@ Example response: "allow_local_requests_from_web_hooks_and_services": true, "allow_local_requests_from_system_hooks": false, "npm_package_requests_forwarding": true, + "snippet_size_limit": 52428800, "issues_create_limit": 300, - "raw_blob_request_limit": 300 + "raw_blob_request_limit": 300, + "wiki_page_max_content_bytes": 52428800 } ``` @@ -214,27 +218,29 @@ are listed in the descriptions of the relevant settings. | `default_ci_config_path` | string | no | Default CI configuration path for new projects (`.gitlab-ci.yml` if not set). | | `default_group_visibility` | string | no | What visibility level new groups receive. Can take `private`, `internal` and `public` as a parameter. Default is `private`. | | `default_project_creation` | integer | no | Default project creation protection. Can take: `0` _(No one)_, `1` _(Maintainers)_ or `2` _(Developers + Maintainers)_| -| `default_projects_limit` | integer | no | Project limit per user. Default is `100000`. | | `default_project_visibility` | string | no | What visibility level new projects receive. Can take `private`, `internal` and `public` as a parameter. Default is `private`. | +| `default_projects_limit` | integer | no | Project limit per user. Default is `100000`. | | `default_snippet_visibility` | string | no | What visibility level new snippets receive. Can take `private`, `internal` and `public` as a parameter. Default is `private`. | +| `deletion_adjourned_period` | integer | no | **(PREMIUM ONLY)** The number of days to wait before deleting a project or group that is marked for deletion. Value must be between 0 and 90. | `diff_max_patch_bytes` | integer | no | Maximum diff patch size (Bytes). | | `disabled_oauth_sign_in_sources` | array of strings | no | Disabled OAuth sign-in sources. | | `dns_rebinding_protection_enabled` | boolean | no | Enforce DNS rebinding attack protection. | -| `domain_blacklist` | array of strings | no | Users with e-mail addresses that match these domain(s) will NOT be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: `domain.com`, `*.domain.com`. | | `domain_blacklist_enabled` | boolean | no | (**If enabled, requires:** `domain_blacklist`) Allows blocking sign-ups from emails from specific domains. | +| `domain_blacklist` | array of strings | no | Users with e-mail addresses that match these domain(s) will NOT be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: `domain.com`, `*.domain.com`. | | `domain_whitelist` | array of strings | no | Force people to use only corporate emails for sign-up. Default is `null`, meaning there is no restriction. | | `dsa_key_restriction` | integer | no | The minimum allowed bit length of an uploaded DSA key. Default is `0` (no restriction). `-1` disables DSA keys. | | `ecdsa_key_restriction` | integer | no | The minimum allowed curve size (in bits) of an uploaded ECDSA key. Default is `0` (no restriction). `-1` disables ECDSA keys. | | `ed25519_key_restriction` | integer | no | The minimum allowed curve size (in bits) of an uploaded ED25519 key. Default is `0` (no restriction). `-1` disables ED25519 keys. | -| `eks_integration_enabled` | boolean | no | Enable integration with Amazon EKS | -| `eks_account_id` | string | no | Amazon account ID | | `eks_access_key_id` | string | no | AWS IAM access key ID | +| `eks_account_id` | string | no | Amazon account ID | +| `eks_integration_enabled` | boolean | no | Enable integration with Amazon EKS | | `eks_secret_access_key` | string | no | AWS IAM secret access key | | `elasticsearch_aws_access_key` | string | no | **(PREMIUM)** AWS IAM access key | -| `elasticsearch_aws` | boolean | no | **(PREMIUM)** Enable the use of AWS hosted Elasticsearch | | `elasticsearch_aws_region` | string | no | **(PREMIUM)** The AWS region the Elasticsearch domain is configured | | `elasticsearch_aws_secret_access_key` | string | no | **(PREMIUM)** AWS IAM secret access key | +| `elasticsearch_aws` | boolean | no | **(PREMIUM)** Enable the use of AWS hosted Elasticsearch | | `elasticsearch_indexed_field_length_limit` | integer | no | **(PREMIUM)** Maximum size of text fields that will be indexed by Elasticsearch. 0 value means no limit. This does not apply to repository and wiki indexing. | +| `elasticsearch_indexed_file_size_limit_kb` | integer | no | **(PREMIUM)** Maximum size of repository and wiki files that will be indexed by Elasticsearch. | | `elasticsearch_indexing` | boolean | no | **(PREMIUM)** Enable Elasticsearch indexing | | `elasticsearch_limit_indexing` | boolean | no | **(PREMIUM)** Limit Elasticsearch to index certain namespaces and projects | | `elasticsearch_max_bulk_concurrency` | integer | no | **(PREMIUM)** Maximum concurrency of Elasticsearch bulk requests per indexing operation. This only applies to repository indexing operations. | @@ -246,6 +252,7 @@ are listed in the descriptions of the relevant settings. | `email_additional_text` | string | no | **(PREMIUM)** Additional text added to the bottom of every email for legal/auditing/compliance reasons | | `email_author_in_body` | boolean | no | Some email servers do not support overriding the email sender name. Enable this option to include the name of the author of the issue, merge request or comment in the email body instead. | | `enabled_git_access_protocol` | string | no | Enabled protocols for Git access. Allowed values are: `ssh`, `http`, and `nil` to allow both protocols. | +| `enforce_namespace_storage_limit` | boolean | no | Enabling this permits enforcement of namespace storage limits. | | `enforce_terms` | boolean | no | (**If enabled, requires:** `terms`) Enforce application ToS to all users. | | `external_auth_client_cert` | string | no | (**If enabled, requires:** `external_auth_client_key`) The certificate to use to authenticate with the external authorization service | | `external_auth_client_key_pass` | string | no | Passphrase to use for the private key when authenticating with the external service this is encrypted when stored | @@ -279,7 +286,10 @@ are listed in the descriptions of the relevant settings. | `html_emails_enabled` | boolean | no | Enable HTML emails. | | `import_sources` | array of strings | no | Sources to allow project import from, possible values: `github`, `bitbucket`, `bitbucket_server`, `gitlab`, `google_code`, `fogbugz`, `git`, `gitlab_project`, `gitea`, `manifest`, and `phabricator`. | | `instance_statistics_visibility_private` | boolean | no | When set to `true` Instance statistics will only be available to admins. | +| `issues_create_limit` | integer | no | Max number of issue creation requests per minute per user. Disabled by default.| | `local_markdown_version` | integer | no | Increase this value when any cached Markdown should be invalidated. | +| `maintenance_mode_message` | string | no | **(PREMIUM)** Message displayed when instance is in maintenance mode | +| `maintenance_mode` | boolean | no | **(PREMIUM)** When instance is in maintenance mode, non-admin users can sign in with read-only access and make read-only API requests | | `max_artifacts_size` | integer | no | Maximum artifacts size in MB | | `max_attachment_size` | integer | no | Limit attachment size in MB | | `max_import_size` | integer | no | Maximum import size in MB. 0 for unlimited. Default = 50 | @@ -291,8 +301,6 @@ are listed in the descriptions of the relevant settings. | `mirror_max_capacity` | integer | no | **(PREMIUM)** Maximum number of mirrors that can be synchronizing at the same time. | | `mirror_max_delay` | integer | no | **(PREMIUM)** Maximum time (in minutes) between updates that a mirror can have when scheduled to synchronize. | | `npm_package_requests_forwarding` | boolean | no | **(PREMIUM)** Use npmjs.org as a default remote repository when the package is not found in the GitLab NPM Registry | -| `maintenance_mode` | boolean | no | **(PREMIUM)** When instance is in maintenance mode, non-admin users can sign in with read-only access and make read-only API requests | -| `maintenance_mode_message` | string | no | **(PREMIUM)** Message displayed when instance is in maintenance mode | | `outbound_local_requests_whitelist` | array of strings | no | Define a list of trusted domains or ip addresses to which local requests are allowed when local requests for hooks and services are disabled. | `pages_domain_verification_enabled` | boolean | no | Require users to prove ownership of custom domains. Domain verification is an essential security measure for public GitLab sites. Users are required to demonstrate they control a domain before it is enabled. | | `password_authentication_enabled_for_git` | boolean | no | Enable authentication for Git over HTTP(S) via a GitLab account password. Default is `true`. | @@ -303,21 +311,21 @@ are listed in the descriptions of the relevant settings. | `plantuml_enabled` | boolean | no | (**If enabled, requires:** `plantuml_url`) Enable PlantUML integration. Default is `false`. | | `plantuml_url` | string | required by: `plantuml_enabled` | The PlantUML instance URL for integration. | | `polling_interval_multiplier` | decimal | no | Interval multiplier used by endpoints that perform polling. Set to `0` to disable polling. | -| `deletion_adjourned_period` | integer | no | **(PREMIUM ONLY)** The number of days to wait before removing a project or group that is marked for deletion. Value must be between 0 and 90. | `project_export_enabled` | boolean | no | Enable project export. | | `prometheus_metrics_enabled` | boolean | no | Enable Prometheus metrics. | | `protected_ci_variables` | boolean | no | Environment variables are protected by default. | | `pseudonymizer_enabled` | boolean | no | **(PREMIUM)** When enabled, GitLab will run a background job that will produce pseudonymized CSVs of the GitLab database that will be uploaded to your configured object storage directory. -| `push_event_hooks_limit` | integer | no | Number of changes (branches or tags) in a single push to determine whether webhooks and services will be fired or not. Webhooks and services won't be submitted if it surpasses that value. | | `push_event_activities_limit` | integer | no | Number of changes (branches or tags) in a single push to determine whether individual push events or bulk push events will be created. [Bulk push events will be created](../user/admin_area/settings/push_event_activities_limit.md) if it surpasses that value. | +| `push_event_hooks_limit` | integer | no | Number of changes (branches or tags) in a single push to determine whether webhooks and services will be fired or not. Webhooks and services won't be submitted if it surpasses that value. | +| `raw_blob_request_limit` | integer | no | Max number of requests per minute for each raw path. Default: 300. To disable throttling set to 0.| | `recaptcha_enabled` | boolean | no | (**If enabled, requires:** `recaptcha_private_key` and `recaptcha_site_key`) Enable reCAPTCHA. | | `recaptcha_private_key` | string | required by: `recaptcha_enabled` | Private key for reCAPTCHA. | | `recaptcha_site_key` | string | required by: `recaptcha_enabled` | Site key for reCAPTCHA. | | `receive_max_input_size` | integer | no | Maximum push size (MB). | | `repository_checks_enabled` | boolean | no | GitLab will periodically run `git fsck` in all project and wiki repositories to look for silent disk corruption issues. | | `repository_size_limit` | integer | no | **(PREMIUM)** Size limit per repository (MB) | -| `repository_storages` | array of strings | no | (GitLab 13.0 and earlier) List of names of enabled storage paths, taken from `gitlab.yml`. New projects are created in one of these stores, chosen at random. | | `repository_storages_weighted` | hash of strings to integers | no | (GitLab 13.1 and later) Hash of names of taken from `gitlab.yml` to weights. New projects are created in one of these stores, chosen by a weighted random selection. | +| `repository_storages` | array of strings | no | (GitLab 13.0 and earlier) List of names of enabled storage paths, taken from `gitlab.yml`. New projects are created in one of these stores, chosen at random. | | `require_two_factor_authentication` | boolean | no | (**If enabled, requires:** `two_factor_grace_period`) Require all users to set up Two-factor authentication. | | `restricted_visibility_levels` | array of strings | no | Selected levels cannot be used by non-admin users for groups, projects or snippets. Can take `private`, `internal` and `public` as a parameter. Default is `null` which means there is no restriction. | | `rsa_key_restriction` | integer | no | The minimum allowed bit length of an uploaded RSA key. Default is `0` (no restriction). `-1` disables RSA keys. | @@ -326,21 +334,22 @@ are listed in the descriptions of the relevant settings. | `shared_runners_enabled` | boolean | no | (**If enabled, requires:** `shared_runners_text` and `shared_runners_minutes`) Enable shared runners for new projects. | | `shared_runners_minutes` | integer | required by: `shared_runners_enabled` | **(PREMIUM)** Set the maximum number of pipeline minutes that a group can use on shared Runners per month. | | `shared_runners_text` | string | required by: `shared_runners_enabled` | Shared runners text. | -| `signin_enabled` | string | no | (Deprecated: Use `password_authentication_enabled_for_web` instead) Flag indicating if password authentication is enabled for the web interface. | | `sign_in_text` | string | no | Text on the login page. | +| `signin_enabled` | string | no | (Deprecated: Use `password_authentication_enabled_for_web` instead) Flag indicating if password authentication is enabled for the web interface. | | `signup_enabled` | boolean | no | Enable registration. Default is `true`. | | `slack_app_enabled` | boolean | no | **(PREMIUM)** (**If enabled, requires:** `slack_app_id`, `slack_app_secret` and `slack_app_secret`) Enable Slack app. | | `slack_app_id` | string | required by: `slack_app_enabled` | **(PREMIUM)** The app ID of the Slack-app. | | `slack_app_secret` | string | required by: `slack_app_enabled` | **(PREMIUM)** The app secret of the Slack-app. | | `slack_app_verification_token` | string | required by: `slack_app_enabled` | **(PREMIUM)** The verification token of the Slack-app. | +| `snippet_size_limit` | integer | no | Max snippet content size in **bytes**. Default: 52428800 Bytes (50MB).| +| `snowplow_app_id` | string | no | The Snowplow site name / application ID. (for example, `gitlab`) | | `snowplow_collector_hostname` | string | required by: `snowplow_enabled` | The Snowplow collector hostname. (for example, `snowplow.trx.gitlab.net`) | | `snowplow_cookie_domain` | string | no | The Snowplow cookie domain. (for example, `.gitlab.com`) | | `snowplow_enabled` | boolean | no | Enable snowplow tracking. | -| `snowplow_app_id` | string | no | The Snowplow site name / application ID. (for example, `gitlab`) | | `snowplow_iglu_registry_url` | string | no | The Snowplow base Iglu Schema Registry URL to use for custom context and self describing events'| | `sourcegraph_enabled` | boolean | no | Enables Sourcegraph integration. Default is `false`. **If enabled, requires** `sourcegraph_url`. | -| `sourcegraph_url` | string | required by: `sourcegraph_enabled` | The Sourcegraph instance URL for integration. | | `sourcegraph_public_only` | boolean | no | Blocks Sourcegraph from being loaded on private and internal projects. Default is `true`. | +| `sourcegraph_url` | string | required by: `sourcegraph_enabled` | The Sourcegraph instance URL for integration. | | `spam_check_endpoint_enabled` | boolean | no | Enables Spam Check via external API endpoint. Default is `false`. | | `spam_check_endpoint_url` | string | no | URL of the external Spam Check service endpoint. | | `terminal_max_session_time` | integer | no | Maximum time for web terminal websocket connection (in seconds). Set to `0` for unlimited time. | @@ -366,6 +375,4 @@ are listed in the descriptions of the relevant settings. | `user_show_add_ssh_key_message` | boolean | no | When set to `false` disable the "You won't be able to pull or push project code via SSH" warning shown to users with no uploaded SSH key. | | `version_check_enabled` | boolean | no | Let GitLab inform you when an update is available. | | `web_ide_clientside_preview_enabled` | boolean | no | Live Preview (allow live previews of JavaScript projects in the Web IDE using CodeSandbox Live Preview). | -| `snippet_size_limit` | integer | no | Max snippet content size in **bytes**. Default: 52428800 Bytes (50MB).| -| `issues_create_limit` | integer | no | Max number of issue creation requests per minute per user. Default: 300. To disable throttling set to 0.| -| `raw_blob_request_limit` | integer | no | Max number of requests per minute for each raw path. Default: 300. To disable throttling set to 0.| +| `wiki_page_max_content_bytes` | integer | no | Maximum wiki page content size in **bytes**. Default: 52428800 Bytes (50 MB). The minimum value is 1024 bytes. | diff --git a/doc/api/sidekiq_metrics.md b/doc/api/sidekiq_metrics.md index 8523ac88e00..95acc992789 100644 --- a/doc/api/sidekiq_metrics.md +++ b/doc/api/sidekiq_metrics.md @@ -1,6 +1,6 @@ # Sidekiq Metrics API ->**Note:** This endpoint is only available on GitLab 8.9 and above. +> Introduced in GitLab 8.9. This API endpoint allows you to retrieve some information about the current state of Sidekiq, its jobs, queues, and processes. diff --git a/doc/api/snippets.md b/doc/api/snippets.md index db94716c2d4..0cdc07b1f46 100644 --- a/doc/api/snippets.md +++ b/doc/api/snippets.md @@ -1,3 +1,10 @@ +--- +stage: Create +group: Editor +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference, api +--- + # Snippets API > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/6373) in GitLab 8.15. diff --git a/doc/api/suggestions.md b/doc/api/suggestions.md index e3bbcaa51c3..10528fe33b9 100644 --- a/doc/api/suggestions.md +++ b/doc/api/suggestions.md @@ -1,3 +1,10 @@ +--- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference, api +--- + # Suggest Changes API Every API call to suggestions must be authenticated. diff --git a/doc/api/tags.md b/doc/api/tags.md index 569271d6206..cf6cfd25dbb 100644 --- a/doc/api/tags.md +++ b/doc/api/tags.md @@ -1,3 +1,10 @@ +--- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference, api +--- + # Tags API ## List project repository tags diff --git a/doc/api/templates/dockerfiles.md b/doc/api/templates/dockerfiles.md index a2c21ada05b..2fa3d7b7fa1 100644 --- a/doc/api/templates/dockerfiles.md +++ b/doc/api/templates/dockerfiles.md @@ -93,6 +93,10 @@ Example response: "name": "Ruby-alpine" }, { + "key": "Rust", + "name": "Rust" + }, + { "key": "Swift", "name": "Swift" } diff --git a/doc/api/templates/gitignores.md b/doc/api/templates/gitignores.md index 6e2e3e2d07f..3acd666ad66 100644 --- a/doc/api/templates/gitignores.md +++ b/doc/api/templates/gitignores.md @@ -2,7 +2,7 @@ type: reference --- -# `.gitignore` API +# .gitignore API In GitLab, there is an API endpoint available for `.gitignore`. For more information on `gitignore`, see the diff --git a/doc/api/templates/gitlab_ci_ymls.md b/doc/api/templates/gitlab_ci_ymls.md index 816061c6235..dfe22fc453e 100644 --- a/doc/api/templates/gitlab_ci_ymls.md +++ b/doc/api/templates/gitlab_ci_ymls.md @@ -1,4 +1,7 @@ --- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference --- @@ -8,9 +11,9 @@ In GitLab, there is an API endpoint available to work with GitLab CI/CD YMLs. Fo information on CI/CD pipeline configuration in GitLab, see the [configuration reference documentation](../../ci/yaml/README.md). -## List GitLab CI YML templates +## List GitLab CI YAML templates -Get all GitLab CI/CD YML templates. +Get all GitLab CI/CD YAML templates. ```plaintext GET /templates/gitlab_ci_ymls @@ -19,7 +22,7 @@ GET /templates/gitlab_ci_ymls Example request: ```shell -curl https://gitlab.example.com/api/v4/templates/gitlab_ci_ymls +curl "https://gitlab.example.com/api/v4/templates/gitlab_ci_ymls" ``` Example response: @@ -109,9 +112,9 @@ Example response: ] ``` -## Single GitLab CI YML template +## Single GitLab CI YAML template -Get a single GitLab CI/CD YML template. +Get a single GitLab CI/CD YAML template. ```plaintext GET /templates/gitlab_ci_ymls/:key @@ -119,12 +122,12 @@ GET /templates/gitlab_ci_ymls/:key | Attribute | Type | Required | Description | | ---------- | ------ | -------- | ------------------------------------- | -| `key` | string | yes | The key of the GitLab CI/CD YML template | +| `key` | string | yes | The key of the GitLab CI/CD YAML template | Example request: ```shell -curl https://gitlab.example.com/api/v4/templates/gitlab_ci_ymls/Ruby +curl "https://gitlab.example.com/api/v4/templates/gitlab_ci_ymls/Ruby" ``` Example response: diff --git a/doc/api/templates/licenses.md b/doc/api/templates/licenses.md index f66fb70e108..4eb3c0f6111 100644 --- a/doc/api/templates/licenses.md +++ b/doc/api/templates/licenses.md @@ -120,7 +120,7 @@ GET /templates/licenses/:key | `project` | string | no | The copyrighted project name | | `fullname` | string | no | The full-name of the copyright holder | ->**Note:** +NOTE: **Note:** If you omit the `fullname` parameter but authenticate your request, the name of the authenticated user will be used to replace the copyright holder placeholder. diff --git a/doc/api/users.md b/doc/api/users.md index 505468945cb..76075e8b7be 100644 --- a/doc/api/users.md +++ b/doc/api/users.md @@ -314,7 +314,8 @@ Example Responses: "current_sign_in_ip": "196.165.1.102", "last_sign_in_ip": "172.127.2.22", "plan": "gold", - "trial": true + "trial": true, + "sign_in_count": 1337 } ``` @@ -1053,6 +1054,10 @@ curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://git Get a list of currently authenticated user's emails. +NOTE: **Note:** +Due to [a bug](https://gitlab.com/gitlab-org/gitlab/-/issues/25077) this endpoint currently +does not return the primary email address. + ```plaintext GET /user/emails ``` @@ -1078,6 +1083,10 @@ Parameters: Get a list of a specified user's emails. Available only for admin +NOTE: **Note:** +Due to [a bug](https://gitlab.com/gitlab-org/gitlab/-/issues/25077) this endpoint currently +does not return the primary email address. + ```plaintext GET /users/:id/emails ``` @@ -1284,6 +1293,7 @@ Example response: [ { "active" : true, + "user_id" : 2, "scopes" : [ "api" ], @@ -1296,6 +1306,7 @@ Example response: }, { "active" : false, + "user_id" : 2, "scopes" : [ "read_user" ], @@ -1335,6 +1346,7 @@ Example response: ```json { "active" : true, + "user_id" : 2, "scopes" : [ "api" ], @@ -1378,6 +1390,7 @@ Example response: { "id" : 2, "revoked" : false, + "user_id" : 2, "scopes" : [ "api" ], diff --git a/doc/api/version.md b/doc/api/version.md index 661fdfb4b03..3c6feaae071 100644 --- a/doc/api/version.md +++ b/doc/api/version.md @@ -1,6 +1,6 @@ # Version API ->**Note:** This feature was introduced in GitLab 8.13 +> Introduced in GitLab 8.13. Retrieve version information for this GitLab instance. Responds `200 OK` for authenticated users. diff --git a/doc/api/visual_review_discussions.md b/doc/api/visual_review_discussions.md index bcbfdbdc6d0..c9863784038 100644 --- a/doc/api/visual_review_discussions.md +++ b/doc/api/visual_review_discussions.md @@ -1,3 +1,10 @@ +--- +stage: Verify +group: Testing +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: reference, api +--- + # Visual Review discussions API **(STARTER)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/18710) in [GitLab Starter](https://about.gitlab.com/pricing/) 12.5. diff --git a/doc/api/vulnerabilities.md b/doc/api/vulnerabilities.md index 70f29d961e3..a0d871af127 100644 --- a/doc/api/vulnerabilities.md +++ b/doc/api/vulnerabilities.md @@ -6,7 +6,7 @@ NOTE: **Note:** The former Vulnerabilities API was renamed to Vulnerability Findings API and its documentation was moved to [a different location](vulnerability_findings.md). This document now describes the new Vulnerabilities API that provides access to -[Standalone Vulnerabilities](https://gitlab.com/groups/gitlab-org/-/epics/634). +[Vulnerabilities](https://gitlab.com/groups/gitlab-org/-/epics/634). CAUTION: **Caution:** This API is in an alpha stage and considered unstable. diff --git a/doc/api/vulnerability_findings.md b/doc/api/vulnerability_findings.md index e21d903e474..96171f0229d 100644 --- a/doc/api/vulnerability_findings.md +++ b/doc/api/vulnerability_findings.md @@ -4,7 +4,7 @@ NOTE: **Note:** This API resource is renamed from Vulnerabilities to Vulnerability Findings because the Vulnerabilities are reserved -for serving the upcoming [Standalone Vulnerability objects](https://gitlab.com/gitlab-org/gitlab/-/issues/13561). +for serving [Vulnerability objects](https://gitlab.com/gitlab-org/gitlab/-/issues/13561). To fix any broken integrations with the former Vulnerabilities API, change the `vulnerabilities` URL part to be `vulnerability_findings`. diff --git a/doc/api/wikis.md b/doc/api/wikis.md index f7595e7efe6..7d16a5a38ee 100644 --- a/doc/api/wikis.md +++ b/doc/api/wikis.md @@ -1,3 +1,10 @@ +--- +stage: Create +group: Knowledge +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference, api +--- + # Wikis API > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/13372) in GitLab 10.0. diff --git a/doc/articles/index.md b/doc/articles/index.md index 7fc4c18f771..4b965b0256f 100644 --- a/doc/articles/index.md +++ b/doc/articles/index.md @@ -1,18 +1,5 @@ --- -comments: false +redirect_to: '../topics/index.md' --- -# Technical articles list (deprecated) - -Technical articles are -topic-related documentation, written with a user-friendly approach and language, aiming -to provide the community with guidance on specific processes to achieve certain objectives. - -The list of technical articles was [deprecated](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/41138) in favor of having them linked from their topic-related documentation: - -- [Git](../topics/git/index.md) -- [GitLab administrator](../administration/index.md) -- [GitLab CI/CD](../ci/README.md) -- [GitLab Pages](../user/project/pages/index.md) -- [GitLab user](../user/index.md) -- [Install GitLab](../install/README.md) +This document was moved to [another location](../topics/index.md) diff --git a/doc/ci/README.md b/doc/ci/README.md index 43e66adbd35..6f15fa52550 100644 --- a/doc/ci/README.md +++ b/doc/ci/README.md @@ -63,7 +63,7 @@ the following documents: If you're migrating from another CI/CD tool, check out our handy references: - [Migrating from CircleCI](migration/circleci.md) -- [Migrating from Jenkins](jenkins/index.md) +- [Migrating from Jenkins](migration/jenkins.md) You can also get started by using one of the [`.gitlab-ci.yml` templates](https://gitlab.com/gitlab-org/gitlab-foss/tree/master/lib/gitlab/ci/templates) diff --git a/doc/ci/ci_cd_for_external_repos/github_integration.md b/doc/ci/ci_cd_for_external_repos/github_integration.md index 5365ef49944..dc1135742ea 100644 --- a/doc/ci/ci_cd_for_external_repos/github_integration.md +++ b/doc/ci/ci_cd_for_external_repos/github_integration.md @@ -23,7 +23,7 @@ cannot be used to authenticate with GitHub as an external CI/CD repository. NOTE: **Note:** Personal access tokens can only be used to connect GitHub.com -repositories to GitLab, and the GitHub user must have the [owner role](https://help.github.com/en/github/getting-started-with-github/access-permissions-on-github). +repositories to GitLab, and the GitHub user must have the [owner role](https://docs.github.com/en/github/getting-started-with-github/access-permissions-on-github). To perform a one-off authorization with GitHub to grant GitLab access your repositories: diff --git a/doc/ci/cloud_deployment/index.md b/doc/ci/cloud_deployment/index.md index 29ce8bdf625..355bc7813d9 100644 --- a/doc/ci/cloud_deployment/index.md +++ b/doc/ci/cloud_deployment/index.md @@ -81,11 +81,12 @@ path to point to your ECR image. ### Deploy your application to the AWS Elastic Container Service (ECS) -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/207962) in GitLab 12.9. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/207962) in GitLab 12.9. +> - The `Deploy-ECS.gitlab-ci.yml` template was [moved](https://gitlab.com/gitlab-org/gitlab/-/issues/220821) to `AWS/Deploy-ECS.gitlab-ci.yml` in GitLab 13.2. GitLab provides a series of [CI templates that you can include in your project](../yaml/README.md#include). To automate deployments of your application to your [Amazon Elastic Container Service](https://aws.amazon.com/ecs/) (AWS ECS) -cluster, you can `include` the `Deploy-ECS.gitlab-ci.yml` template in your `.gitlab-ci.yml` file. +cluster, you can `include` the `AWS/Deploy-ECS.gitlab-ci.yml` template in your `.gitlab-ci.yml` file. GitLab also provides [Docker images](https://gitlab.com/gitlab-org/cloud-deploy/-/tree/master/aws) that can be used in your `gitlab-ci.yml` file to simplify working with AWS: @@ -96,11 +97,25 @@ Before getting started with this process, you need a cluster on AWS ECS, as well components, like an ECS service, ECS task definition, a database on AWS RDS, etc. [Read more about AWS ECS](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/Welcome.html). -After you're all set up on AWS ECS, follow these steps: +The ECS task definition can be: + +- An existing task definition in AWS ECS +- A JSON file containing a task definition. Create the JSON file by using the template provided in + the [AWS documentation](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/create-task-definition.html#task-definition-template). + Copy the task definition into a new file in your project, for example `<project-root>/ci/aws/task-definition.json`. + [Available](https://gitlab.com/gitlab-org/gitlab/-/issues/222618) in GitLab 13.3 and later. + +After you have these prerequisites ready, follow these steps: 1. Make sure your AWS credentials are set up as environment variables for your project. You can follow [the steps above](#run-aws-commands-from-gitlab-cicd) to complete this setup. -1. Add these variables to your project's `.gitlab-ci.yml` file: +1. Add these variables to your project's `.gitlab-ci.yml` file, or in the project's + [CI/CD settings](../variables/README.md#create-a-custom-variable-in-the-ui): + + - `CI_AWS_ECS_CLUSTER`: The name of the AWS ECS cluster that you're targeting for your deployments. + - `CI_AWS_ECS_SERVICE`: The name of the targeted service tied to your AWS ECS cluster. + - `CI_AWS_ECS_TASK_DEFINITION`: The name of an existing task definition in ECS tied + to the service mentioned above. ```yaml variables: @@ -109,19 +124,37 @@ After you're all set up on AWS ECS, follow these steps: CI_AWS_ECS_TASK_DEFINITION: my-task-definition ``` - Three variables are defined in this snippet: - - - `CI_AWS_ECS_CLUSTER`: The name of your AWS ECS cluster that you're - targeting for your deployments. - - `CI_AWS_ECS_SERVICE`: The name of the targeted service tied to - your AWS ECS cluster. - - `CI_AWS_ECS_TASK_DEFINITION`: The name of the task definition tied - to the service mentioned above. - You can find these names after selecting the targeted cluster on your [AWS ECS dashboard](https://console.aws.amazon.com/ecs/home):  + Alternatively, if you want to use a task definition defined in a JSON file, use + `CI_AWS_ECS_TASK_DEFINITION_FILE` instead: + + ```yaml + variables: + CI_AWS_ECS_CLUSTER: my-cluster + CI_AWS_ECS_SERVICE: my-service + CI_AWS_ECS_TASK_DEFINITION_FILE: ci/aws/my_task_definition.json + ``` + + You can create your `CI_AWS_ECS_TASK_DEFINITION_FILE` variable as a + [file-typed environment variable](../variables/README.md#custom-environment-variables-of-type-file) instead of a + regular environment variable. If you choose to do so, set the variable value to be the full contents of + the JSON task definition. You can then remove the JSON file from your project. + + In both cases, make sure that the value for the `containerDefinitions[].name` attribute is + the same as the `Container name` defined in your targeted ECS service. + + CAUTION: **Warning:** + `CI_AWS_ECS_TASK_DEFINITION_FILE` takes precedence over `CI_AWS_ECS_TASK_DEFINITION` if both these environment + variables are defined within your project. + + NOTE: **Note:** + If the name of the task definition you wrote in your JSON file is the same name + as an existing task definition on AWS, then a new revision is created for it. + Otherwise, a brand new task definition is created, starting at revision 1. + 1. Include this template in `.gitlab-ci.yml`: ```yaml @@ -129,12 +162,15 @@ After you're all set up on AWS ECS, follow these steps: - template: AWS/Deploy-ECS.gitlab-ci.yml ``` - The `Deploy-ECS` template ships with GitLab and is available [on + The `AWS/Deploy-ECS` template ships with GitLab and is available [on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/AWS/Deploy-ECS.gitlab-ci.yml). 1. Commit and push your updated `.gitlab-ci.yml` to your project's repository, and you're done! Your application Docker image will be rebuilt and pushed to the GitLab registry. + If your image is located in a private registry, make sure your task definition is + [configured with a `repositoryCredentials` attribute](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/private-auth.html). + Then the targeted task definition will be updated with the location of the new Docker image, and a new revision will be created in ECS as result. @@ -143,17 +179,17 @@ After you're all set up on AWS ECS, follow these steps: application. CAUTION: **Warning:** -The [`Deploy-ECS.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/AWS/Deploy-ECS.gitlab-ci.yml) +The [`AWS/Deploy-ECS.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/AWS/Deploy-ECS.gitlab-ci.yml) template includes both the [`Jobs/Build.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml) and [`Jobs/Deploy/ECS.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml) "sub-templates". Do not include these "sub-templates" on their own, and only include the main -`Deploy-ECS.gitlab-ci.yml` template. The "sub-templates" are designed to only be +`AWS/Deploy-ECS.gitlab-ci.yml` template. The "sub-templates" are designed to only be used along with the main template. They may move or change unexpectedly causing your pipeline to fail if you didn't include the main template. Also, the job names within these templates may change. Do not override these jobs names in your own pipeline, as the override will stop working when the name changes. -Alternatively, if you don't wish to use the `Deploy-ECS.gitlab-ci.yml` template +Alternatively, if you don't wish to use the `AWS/Deploy-ECS.gitlab-ci.yml` template to deploy to AWS ECS, you can always use our `aws-base` Docker image to run your own [AWS CLI commands for ECS](https://docs.aws.amazon.com/cli/latest/reference/ecs/index.html#cli-aws-ecs). diff --git a/doc/ci/directed_acyclic_graph/index.md b/doc/ci/directed_acyclic_graph/index.md index b7dd74a0230..8fc58df51fe 100644 --- a/doc/ci/directed_acyclic_graph/index.md +++ b/doc/ci/directed_acyclic_graph/index.md @@ -84,6 +84,7 @@ are certain use cases that you may need to work around. For more information: > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/215517) in GitLab 13.1 as a [Beta feature](https://about.gitlab.com/handbook/product/#beta). > - It was deployed behind a feature flag, disabled by default. > - It became [enabled by default](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/36802) in 13.2. +> - It became a [standard feature](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/38517) in 13.3. > - It's enabled on GitLab.com. > - For GitLab self-managed instances, GitLab administrators can opt to [disable it](#enable-or-disable-dag-visualization-core-only). @@ -97,9 +98,7 @@ Clicking a node will highlight all the job paths it depends on. ### Enable or disable DAG Visualization **(CORE ONLY)** -DAG Visualization is under development, but is being made available as a beta feature so users can check its limitations and uses. - -It is deployed behind a feature flag that is **enabled by default**. +DAG Visualization is deployed behind a feature flag that is **enabled by default**. [GitLab administrators with access to the GitLab Rails console](../../administration/feature_flags.md) can opt to disable it for your instance: diff --git a/doc/ci/docker/using_docker_build.md b/doc/ci/docker/using_docker_build.md index 4bed6d9e323..88d6dc3aae4 100644 --- a/doc/ci/docker/using_docker_build.md +++ b/doc/ci/docker/using_docker_build.md @@ -32,7 +32,7 @@ during jobs. ## Runner Configuration There are three methods to enable the use of `docker build` and `docker run` -during jobs; each with their own tradeoffs. +during jobs, each with their own tradeoffs. An alternative to using `docker build` is to [use kaniko](using_kaniko.md). This avoids having to execute Runner in privileged mode. @@ -61,8 +61,8 @@ GitLab Runner then executes job scripts as the `gitlab-runner` user. 1. Install Docker Engine on server. - For more information how to install Docker Engine on different systems - checkout the [Supported installations](https://docs.docker.com/engine/installation/). + For more information how to install Docker Engine on different systems, + check out the [Supported installations](https://docs.docker.com/engine/installation/). 1. Add `gitlab-runner` user to `docker` group: @@ -118,13 +118,13 @@ not without its own challenges: - When using Docker-in-Docker, each job is in a clean environment without the past history. Concurrent jobs work fine because every build gets its own - instance of Docker engine so they won't conflict with each other. But this + instance of Docker engine so they don't conflict with each other. But this also means that jobs can be slower because there's no caching of layers. - By default, Docker 17.09 and higher uses `--storage-driver overlay2` which is the recommended storage driver. See [Using the overlayfs driver](#use-the-overlayfs-driver) for details. - Since the `docker:19.03.12-dind` container and the Runner container don't share their - root filesystem, the job's working directory can be used as a mount point for + root file system, the job's working directory can be used as a mount point for child containers. For example, if you have files you want to share with a child container, you may create a subdirectory under `/builds/$CI_PROJECT_PATH` and use it as your mount point (for a more thorough explanation, check [issue @@ -143,8 +143,8 @@ An example project using this approach can be found here: <https://gitlab.com/gi In the examples below, we are using Docker images tags to specify a specific version, such as `docker:19.03.12`. If tags like `docker:stable` -are used, you have no control over what version is going to be used and this -can lead to unpredictable behavior, especially when new versions are +are used, you have no control over what version is used. This can lead to +unpredictable behavior, especially when new versions are released. #### TLS enabled @@ -179,18 +179,18 @@ support this. --docker-volumes "/certs/client" ``` - The above command will register a new Runner to use the special + The above command registers a new Runner to use the special `docker:19.03.12` image, which is provided by Docker. **Notice that it's using the `privileged` mode to start the build and service containers.** If you want to use [Docker-in-Docker](https://www.docker.com/blog/docker-can-now-run-within-docker/) mode, you always have to use `privileged = true` in your Docker containers. - This will also mount `/certs/client` for the service and build + This also mounts `/certs/client` for the service and build container, which is needed for the Docker client to use the - certificates inside of that directory. For more information how - Docker with TLS works check <https://hub.docker.com/_/docker/#tls>. + certificates inside of that directory. For more information on how + Docker with TLS works, check <https://hub.docker.com/_/docker/#tls>. - The above command will create a `config.toml` entry similar to this: + The above command creates a `config.toml` entry similar to this: ```toml [[runners]] @@ -215,7 +215,7 @@ support this. image: docker:19.03.12 variables: - # When using dind service, we need to instruct docker, to talk with + # When using dind service, we need to instruct docker to talk with # the daemon started inside of the service. The daemon is available # with a network connection instead of the default # /var/run/docker.sock socket. Docker 19.03 does this automatically @@ -337,13 +337,13 @@ In order to do that, follow the steps: --docker-volumes /var/run/docker.sock:/var/run/docker.sock ``` - The above command will register a new Runner to use the special + The above command registers a new Runner to use the special `docker:19.03.12` image which is provided by Docker. **Notice that it's using the Docker daemon of the Runner itself, and any containers spawned by Docker - commands will be siblings of the Runner rather than children of the Runner.** + commands are siblings of the Runner rather than children of the Runner.** This may have complications and limitations that are unsuitable for your workflow. - The above command will create a `config.toml` entry similar to this: + The above command creates a `config.toml` entry similar to this: ```toml [[runners]] @@ -387,7 +387,7 @@ aware of the following implications: containers. - Concurrent jobs may not work; if your tests create containers with specific names, they may conflict with each other. -- Sharing files and directories from the source repo into containers may not +- Sharing files and directories from the source repository into containers may not work as expected since volume mounting is done in the context of the host machine, not the build container. For example: @@ -397,7 +397,7 @@ aware of the following implications: ## Making Docker-in-Docker builds faster with Docker layer caching -When using Docker-in-Docker, Docker will download all layers of your image every +When using Docker-in-Docker, Docker downloads all layers of your image every time you create a build. Recent versions of Docker (Docker 1.13 and above) can use a pre-existing image as a cache during the `docker build` step, considerably speeding up the build process. @@ -504,7 +504,7 @@ environment variable in the environment = ["DOCKER_DRIVER=overlay2"] ``` -If you're running multiple Runners you will have to modify all configuration files. +If you're running multiple Runners, you have to modify all configuration files. NOTE: **Note:** Read more about the [Runner configuration](https://docs.gitlab.com/runner/configuration/) @@ -523,7 +523,7 @@ This is a common error when you are using [Docker in Docker](#use-docker-in-docker-workflow-with-docker-executor) v19.03 or higher. -This occurs because Docker starts on TLS automatically, so you need to do some set up. +This occurs because Docker starts on TLS automatically, so you need to do some setup. If: - This is the first time setting it up, carefully read diff --git a/doc/ci/docker/using_docker_images.md b/doc/ci/docker/using_docker_images.md index 735cf35584f..db39532bbf2 100644 --- a/doc/ci/docker/using_docker_images.md +++ b/doc/ci/docker/using_docker_images.md @@ -16,12 +16,12 @@ instance. [Docker Hub](https://hub.docker.com/) has a rich database of pre-built used to test and build your applications. When used with GitLab CI/CD, Docker runs each job in a separate and isolated -container using the predefined image that is set up in +container using the predefined image that's set up in [`.gitlab-ci.yml`](../yaml/README.md). This makes it easier to have a simple and reproducible build environment that can also run on your workstation. The added benefit is that you can test all -the commands that we will explore later from your shell, rather than having to +the commands that we explore later from your shell, rather than having to test them on a dedicated CI server. ## Register Docker Runner @@ -54,16 +54,16 @@ sudo gitlab-runner register \ --docker-image ruby:2.6 ``` -The registered runner will use the `ruby:2.6` Docker image and will run two -services, `postgres:latest` and `mysql:latest`, both of which will be +The registered runner uses the `ruby:2.6` Docker image and runs two +services, `postgres:latest` and `mysql:latest`, both of which are accessible during the build process. ## What is an image The `image` keyword is the name of the Docker image the Docker executor -will run to perform the CI tasks. +runs to perform the CI tasks. -By default, the executor will only pull images from [Docker Hub](https://hub.docker.com/), +By default, the executor only pulls images from [Docker Hub](https://hub.docker.com/), however this can be configured in the `gitlab-runner/config.toml` by setting the [Docker pull policy](https://docs.gitlab.com/runner/executors/docker.html#how-pull-policies-work) to allow using local images. @@ -72,16 +72,16 @@ the [Docker Fundamentals](https://docs.docker.com/engine/understanding-docker/) ## What is a service -The `services` keyword defines just another Docker image that is run during +The `services` keyword defines just another Docker image that's run during your job and is linked to the Docker image that the `image` keyword defines. This allows you to access the service image during build time. The service image can run any application, but the most common use case is to -run a database container, e.g., `mysql`. It's easier and faster to use an +run a database container, for example, `mysql`. It's easier and faster to use an existing image and run it as an additional container than install `mysql` every time the project is built. -You are not limited to have only database services. You can add as many +You're not limited to have only database services. You can add as many services you need to `.gitlab-ci.yml` or manually modify `config.toml`. Any image found at [Docker Hub](https://hub.docker.com/) or your private Container Registry can be used as a service. @@ -97,10 +97,10 @@ You can see some widely used services examples in the relevant documentation of To better understand how the container linking works, read [Linking containers together](https://docs.docker.com/engine/userguide/networking/default_network/dockerlinks/). -To summarize, if you add `mysql` as service to your application, the image will -then be used to create a container that is linked to the job container. +To summarize, if you add `mysql` as service to your application, the image is +then used to create a container that's linked to the job container. -The service container for MySQL will be accessible under the hostname `mysql`. +The service container for MySQL is accessible under the hostname `mysql`. So, in order to access your database service you have to connect to the host named `mysql` instead of a socket or `localhost`. Read more in [accessing the services](#accessing-the-services). @@ -109,7 +109,7 @@ services](#accessing-the-services). Services are designed to provide additional functionality which is **network accessible**. It may be a database like MySQL, or Redis, and even `docker:stable-dind` which -allows you to use Docker in Docker. It can be practically anything that is +allows you to use Docker in Docker. It can be practically anything that's required for the CI/CD job to proceed and is accessed by network. To make sure this works, the Runner: @@ -121,8 +121,8 @@ When the second stage of the check fails, either because there is no opened port service, or the service was not started properly before the timeout and the port is not responding, it prints the warning: `*** WARNING: Service XYZ probably didn't start properly`. -In most cases it will affect the job, but there may be situations when the job -will still succeed even if that warning was printed. For example: +In most cases it affects the job, but there may be situations when the job +still succeeds even if that warning was printed. For example: - The service was started a little after the warning was raised, and the job is not using the linked service from the beginning. In that case, when the @@ -130,8 +130,8 @@ will still succeed even if that warning was printed. For example: connections. - The service container is not providing any networking service, but it's doing something with the job's directory (all services have the job directory mounted - as a volume under `/builds`). In that case, the service will do its job, and - since the job is not trying to connect to it, it won't fail. + as a volume under `/builds`). In that case, the service does its job, and + since the job is not trying to connect to it, it does not fail. ### What services are not for @@ -139,12 +139,12 @@ As it was mentioned before, this feature is designed to provide **network access services. A database is the simplest example of such a service. NOTE: **Note:** -The services feature is not designed to, and will not add any software from the +The services feature is not designed to, and does not add any software from the defined `services` image(s) to the job's container. For example, if you have the following `services` defined in your job, the `php`, -`node` or `go` commands will **not** be available for your script, and thus -the job will fail: +`node` or `go` commands are **not** available for your script, and thus +the job fails: ```yaml job: @@ -163,7 +163,7 @@ If you need to have `php`, `node` and `go` available for your script, you should either: - Choose an existing Docker image that contains all required tools. -- Create your own Docker image, which will have all the required tools included +- Create your own Docker image, with all the required tools included and use that in your job. ### Accessing the services @@ -180,7 +180,7 @@ services: ``` If you don't [specify a service alias](#available-settings-for-services), -when the job is run, `tutum/wordpress` will be started and you will have +when the job is run, `tutum/wordpress` is started and you have access to it from your build container under two hostnames to choose from: - `tutum-wordpress` @@ -204,7 +204,7 @@ To override the default behavior, you can ## Define `image` and `services` from `.gitlab-ci.yml` -You can simply define an image that will be used for all jobs and a list of +You can simply define an image that's used for all jobs and a list of services that you want to use during build time: ```yaml @@ -228,7 +228,7 @@ The image name must be in one of the following formats: - `image: <image-name>:<tag>` - `image: <image-name>@<digest>` -It is also possible to define different images and services per job: +It's also possible to define different images and services per job: ```yaml default: @@ -280,7 +280,7 @@ to fine tune your Docker `images` and `services` directly in the `.gitlab-ci.yml For more information, see [custom environment variables](../variables/README.md#gitlab-ciyml-defined-variables) ```yaml -# The following variables will automatically be passed down to the Postgres container +# The following variables are automatically passed down to the Postgres container # as well as the Ruby container and available within each. variables: HTTPS_PROXY: "https://10.1.1.1:8090" @@ -353,7 +353,7 @@ For example, the following two definitions are equal: | Setting | Required | GitLab version | Description | |------------|----------|----------------| ----------- | | `name` | yes, when used with any other option | 9.4 |Full name of the image that should be used. It should contain the Registry part if needed. | -| `entrypoint` | no | 9.4 |Command or script that should be executed as the container's entrypoint. It will be translated to Docker's `--entrypoint` option while creating the container. The syntax is similar to [`Dockerfile`'s `ENTRYPOINT`](https://docs.docker.com/engine/reference/builder/#entrypoint) directive, where each shell token is a separate string in the array. | +| `entrypoint` | no | 9.4 |Command or script that should be executed as the container's entrypoint. It's translated to Docker's `--entrypoint` option while creating the container. The syntax is similar to [`Dockerfile`'s `ENTRYPOINT`](https://docs.docker.com/engine/reference/builder/#entrypoint) directive, where each shell token is a separate string in the array. | ### Available settings for `services` @@ -362,8 +362,8 @@ For example, the following two definitions are equal: | Setting | Required | GitLab version | Description | |------------|----------|----------------| ----------- | | `name` | yes, when used with any other option | 9.4 | Full name of the image that should be used. It should contain the Registry part if needed. | -| `entrypoint` | no | 9.4 |Command or script that should be executed as the container's entrypoint. It will be translated to Docker's `--entrypoint` option while creating the container. The syntax is similar to [`Dockerfile`'s `ENTRYPOINT`](https://docs.docker.com/engine/reference/builder/#entrypoint) directive, where each shell token is a separate string in the array. | -| `command` | no | 9.4 |Command or script that should be used as the container's command. It will be translated to arguments passed to Docker after the image's name. The syntax is similar to [`Dockerfile`'s `CMD`](https://docs.docker.com/engine/reference/builder/#cmd) directive, where each shell token is a separate string in the array. | +| `entrypoint` | no | 9.4 |Command or script that should be executed as the container's entrypoint. It's translated to Docker's `--entrypoint` option while creating the container. The syntax is similar to [`Dockerfile`'s `ENTRYPOINT`](https://docs.docker.com/engine/reference/builder/#entrypoint) directive, where each shell token is a separate string in the array. | +| `command` | no | 9.4 |Command or script that should be used as the container's command. It's translated to arguments passed to Docker after the image's name. The syntax is similar to [`Dockerfile`'s `CMD`](https://docs.docker.com/engine/reference/builder/#cmd) directive, where each shell token is a separate string in the array. | | `alias` | no | 9.4 |Additional alias that can be used to access the service from the job's container. Read [Accessing the services](#accessing-the-services) for more information. | NOTE: **Note:** @@ -398,8 +398,8 @@ services: alias: mysql-2 ``` -The Runner will still start two containers using the `mysql:latest` image, -however now each of them will also be accessible with the alias configured +The Runner still starts two containers using the `mysql:latest` image, +however now each of them are also accessible with the alias configured in `.gitlab-ci.yml` file. ### Setting a command for the service @@ -408,7 +408,7 @@ in `.gitlab-ci.yml` file. Let's assume you have a `super/sql:latest` image with some SQL database inside it and you would like to use it as a service for your job. Let's also -assume that this image doesn't start the database process while starting +assume that this image does not start the database process while starting the container and the user needs to manually use `/usr/bin/super-sql run` as a command to start the database. @@ -462,8 +462,8 @@ CI jobs: output. To override the entrypoint of a Docker image, the recommended solution is to -define an empty `entrypoint` in `.gitlab-ci.yml`, so the Runner doesn't start -a useless shell layer. However, that will not work for all Docker versions, and +define an empty `entrypoint` in `.gitlab-ci.yml`, so the Runner does not start +a useless shell layer. However, that does not work for all Docker versions, and you should check which one your Runner is using. Specifically: - If Docker 17.06 or later is used, the `entrypoint` can be set to an empty value. @@ -476,8 +476,8 @@ Let's assume you have a `super/sql:experimental` image with some SQL database inside it and you would like to use it as a base image for your job because you want to execute some tests with this database binary. Let's also assume that this image is configured with `/usr/bin/super-sql run` as an entrypoint. That -means that when starting the container without additional options, it will run -the database's process, while Runner expects that the image will have no +means that when starting the container without additional options, it runs +the database's process, while Runner expects that the image has no entrypoint or that the entrypoint is prepared to start a shell command. With the extended Docker configuration options, instead of creating your @@ -511,7 +511,7 @@ Look for the `[runners.docker]` section: services = ["mysql:latest", "postgres:latest"] ``` -The image and services defined this way will be added to all job run by +The image and services defined this way are added to all job run by that runner. ## Define an image from a private Container Registry @@ -530,12 +530,12 @@ To define which should be used, the GitLab Runner process reads the configuratio - `DOCKER_AUTH_CONFIG` variable provided as environment variable in `config.toml` of the Runner. - `config.json` file placed in `$HOME/.docker` directory of the user running GitLab Runner process. If the `--user` flag is provided to run the GitLab Runner child processes as unprivileged user, - the home directory of the main GitLab Runner process user will be used. + the home directory of the main GitLab Runner process user is used. NOTE: **Note:** GitLab Runner reads this configuration **only** from `config.toml` and ignores it if it's provided as an environment variable. This is because GitLab Runner uses **only** -`config.toml` configuration and doesn't interpolate **ANY** environment variables at +`config.toml` configuration and does not interpolate **ANY** environment variables at runtime. ### Requirements and limitations @@ -593,9 +593,9 @@ There are two ways to determine the value of `DOCKER_AUTH_CONFIG`: ``` - **Second way -** In some setups, it's possible that Docker client - will use the available system key store to store the result of `docker + uses the available system key store to store the result of `docker login`. In that case, it's impossible to read `~/.docker/config.json`, - so you will need to prepare the required base64-encoded version of + so you need to prepare the required base64-encoded version of `${username}:${password}` and create the Docker configuration JSON manually. Open a terminal and execute the following command: @@ -644,7 +644,7 @@ follow these steps: image: registry.example.com:5000/namespace/image:tag ``` - In the example above, GitLab Runner will look at `registry.example.com:5000` for the + In the example above, GitLab Runner looks at `registry.example.com:5000` for the image `namespace/image:tag`. You can add configuration for as many registries as you want, adding more @@ -655,19 +655,19 @@ The full `hostname:port` combination is required everywhere for the Runner to match the `DOCKER_AUTH_CONFIG`. For example, if `registry.example.com:5000/namespace/image:tag` is specified in `.gitlab-ci.yml`, then the `DOCKER_AUTH_CONFIG` must also specify `registry.example.com:5000`. -Specifying only `registry.example.com` will not work. +Specifying only `registry.example.com` does not work. ### Configuring a Runner -If you have many pipelines that access the same registry, it'll -probably be better to setup registry access at the runner level. This +If you have many pipelines that access the same registry, it is +probably better to set up registry access at the runner level. This allows pipeline authors to have access to a private registry just by running a job on the appropriate runner. It also makes registry changes and credential rotations much simpler. Of course this means that any job on that runner can access the registry with the same privilege, even across projects. If you need to -control access to the registry, you'll need to be sure to control +control access to the registry, you need to be sure to control access to the runner. To add `DOCKER_AUTH_CONFIG` to a Runner: @@ -713,14 +713,14 @@ To configure credentials store, follow these steps: } ``` - - Or, if you are running self-managed Runners, add the above JSON to - `${GITLAB_RUNNER_HOME}/.docker/config.json`. GitLab Runner will read this configuration file - and will use the needed helper for this specific repository. + - Or, if you're running self-managed Runners, add the above JSON to + `${GITLAB_RUNNER_HOME}/.docker/config.json`. GitLab Runner reads this configuration file + and uses the needed helper for this specific repository. NOTE: **Note:** `credsStore` is used to access ALL the registries. -If you will want to use both images from private registry and public images from DockerHub, -pulling from DockerHub will fail, because Docker daemon will try to use the same credentials for **ALL** the registries. +If you want to use both images from private registry and public images from DockerHub, +pulling from DockerHub would fail, because Docker daemon tries to use the same credentials for **ALL** the registries. ### Using Credential Helpers @@ -762,9 +762,9 @@ To configure access for `aws_account_id.dkr.ecr.region.amazonaws.com`, follow th This configures Docker to use the credential helper for all Amazon ECR registries. - - Or, if you are running self-managed Runners, + - Or, if you're running self-managed Runners, add the above JSON to `${GITLAB_RUNNER_HOME}/.docker/config.json`. - GitLab Runner will read this configuration file and will use the needed helper for this + GitLab Runner reads this configuration file and uses the needed helper for this specific repository. 1. You can now use any private image from `aws_account_id.dkr.ecr.region.amazonaws.com` defined in @@ -774,7 +774,7 @@ To configure access for `aws_account_id.dkr.ecr.region.amazonaws.com`, follow th image: aws_account_id.dkr.ecr.region.amazonaws.com/private/image:latest ``` - In the example above, GitLab Runner will look at `aws_account_id.dkr.ecr.region.amazonaws.com` for the + In the example above, GitLab Runner looks at `aws_account_id.dkr.ecr.region.amazonaws.com` for the image `private/image:latest`. You can add configuration for as many registries as you want, adding more @@ -792,7 +792,7 @@ For all possible configuration variables check the documentation of each image provided in their corresponding Docker hub page. NOTE: **Note:** -All variables will be passed to all services containers. It's not +All variables are passed to all services containers. It's not designed to distinguish which variable should go where. ### PostgreSQL service example @@ -838,7 +838,7 @@ EOF ``` Here we use as an example the GitLab Runner repository which contains a -Makefile, so running `make` will execute the commands defined in the Makefile. +Makefile, so running `make` executes the commands defined in the Makefile. Your mileage may vary, so instead of `make` you could run the command which is specific to your project. @@ -849,9 +849,9 @@ docker run -d --name service-mysql mysql:latest docker run -d --name service-postgres postgres:latest ``` -This will create two service containers, named `service-mysql` and +This creates two service containers, named `service-mysql` and `service-postgres` which use the latest MySQL and PostgreSQL images -respectively. They will both run in the background (`-d`). +respectively. They both run in the background (`-d`). Finally, create a build container by executing the `build_script` file we created earlier: @@ -860,7 +860,7 @@ created earlier: docker run --name build -i --link=service-mysql:mysql --link=service-postgres:postgres ruby:2.6 /bin/bash < build_script ``` -The above command will create a container named `build` that is spawned from +The above command creates a container named `build` that's spawned from the `ruby:2.6` image and has two services linked to it. The `build_script` is piped using STDIN to the bash interpreter which in turn executes the `build_script` in the `build` container. @@ -872,6 +872,6 @@ with: docker rm -f -v build service-mysql service-postgres ``` -This will forcefully (`-f`) remove the `build` container, the two service +This forcefully (`-f`) removes the `build` container, the two service containers as well as all volumes (`-v`) that were created with the container creation. diff --git a/doc/ci/docker/using_kaniko.md b/doc/ci/docker/using_kaniko.md index 1580080ac6e..41c8f04f66e 100644 --- a/doc/ci/docker/using_kaniko.md +++ b/doc/ci/docker/using_kaniko.md @@ -63,6 +63,7 @@ build: name: gcr.io/kaniko-project/executor:debug entrypoint: [""] script: + - mkdir -p /kaniko/.docker - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --destination $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG only: diff --git a/doc/ci/environments/deployment_safety.md b/doc/ci/environments/deployment_safety.md index a4ff164a5ba..a0b7adb5279 100644 --- a/doc/ci/environments/deployment_safety.md +++ b/doc/ci/environments/deployment_safety.md @@ -1,3 +1,9 @@ +--- +stage: Release +group: Release Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Deployment safety Deployment jobs can be more sensitive than other jobs in a pipeline, diff --git a/doc/ci/examples/authenticating-with-hashicorp-vault/img/vault-read-secrets-production.png b/doc/ci/examples/authenticating-with-hashicorp-vault/img/vault-read-secrets-production.png Binary files differindex 65c8546deb2..16c15071dd7 100644 --- a/doc/ci/examples/authenticating-with-hashicorp-vault/img/vault-read-secrets-production.png +++ b/doc/ci/examples/authenticating-with-hashicorp-vault/img/vault-read-secrets-production.png diff --git a/doc/ci/examples/authenticating-with-hashicorp-vault/img/vault-read-secrets-staging.png b/doc/ci/examples/authenticating-with-hashicorp-vault/img/vault-read-secrets-staging.png Binary files differindex 2399d8f8879..7a92a3c2d50 100644 --- a/doc/ci/examples/authenticating-with-hashicorp-vault/img/vault-read-secrets-staging.png +++ b/doc/ci/examples/authenticating-with-hashicorp-vault/img/vault-read-secrets-staging.png diff --git a/doc/ci/examples/laravel_with_gitlab_and_envoy/index.md b/doc/ci/examples/laravel_with_gitlab_and_envoy/index.md index ee0938b4d4c..c27566d38cf 100644 --- a/doc/ci/examples/laravel_with_gitlab_and_envoy/index.md +++ b/doc/ci/examples/laravel_with_gitlab_and_envoy/index.md @@ -150,7 +150,7 @@ Now, let's clone our repository on the server just to make sure the `deployer` u git clone git@gitlab.example.com:<USERNAME>/laravel-sample.git ``` ->**Note:** +NOTE: **Note:** Answer **yes** if asked `Are you sure you want to continue connecting (yes/no)?`. It adds GitLab.com to the known hosts. @@ -174,7 +174,7 @@ server { } ``` ->**Note:** +NOTE: **Note:** You may replace the app's name in `/var/www/app/current/public` with the folder name of your application. ## Setting up Envoy @@ -398,7 +398,7 @@ But let's take a step forward to do it automatically with [Continuous Delivery]( We need to check every commit with a set of automated tests to become aware of issues at the earliest, and then, we can deploy to the target environment if we are happy with the result of the tests. [GitLab CI/CD](../../README.md) allows us to use [Docker](https://www.docker.com) engine to handle the process of testing and deploying our app. -In the case you're not familiar with Docker, refer to [How to Automate Docker Deployments](http://paislee.io/how-to-automate-docker-deployments/). +In case you're not familiar with Docker, refer to [Set up automated builds](https://docs.docker.com/get-started/). To be able to build, test, and deploy our app with GitLab CI/CD, we need to prepare our work environment. To do that, we'll use a Docker image which has the minimum requirements that a Laravel app needs to run. @@ -464,14 +464,14 @@ docker build -t registry.gitlab.com/<USERNAME>/laravel-sample . docker push registry.gitlab.com/<USERNAME>/laravel-sample ``` ->**Note:** +NOTE: **Note:** To run the above commands, we first need to have [Docker](https://docs.docker.com/engine/installation/) installed on our machine. Congratulations! You just pushed the first Docker image to the GitLab Registry, and if you refresh the page you should be able to see it:  ->**Note:** +NOTE: **Note:** You can also [use GitLab CI/CD](https://about.gitlab.com/blog/2016/05/23/gitlab-container-registry/#use-with-gitlab-ci) to build and push your Docker images, rather than doing that on your machine. We'll use this image further down in the `.gitlab-ci.yml` configuration file to handle the process of testing and deploying our app. @@ -551,7 +551,7 @@ services: ... ``` ->**Note:** +NOTE: **Note:** If you wish to test your app with different PHP versions and [database management systems](../../services/README.md), you can define different `image` and `services` keywords for each test job. #### Variables diff --git a/doc/ci/examples/test_phoenix_app_with_gitlab_ci_cd/index.md b/doc/ci/examples/test_phoenix_app_with_gitlab_ci_cd/index.md index c901f6c9c66..f2620461c09 100644 --- a/doc/ci/examples/test_phoenix_app_with_gitlab_ci_cd/index.md +++ b/doc/ci/examples/test_phoenix_app_with_gitlab_ci_cd/index.md @@ -123,7 +123,7 @@ Generated hello_gitlab_ci app The database for HelloGitlabCi.Repo has been created ``` -> **Note:** +NOTE: **Note:** Phoenix assumes that our PostgreSQL database will have a `postgres` user account with the correct permissions and a password of `postgres`. If it's not your case, check [Ecto's instructions](https://hexdocs.pm/ecto/Ecto.html#module-repositories). @@ -211,7 +211,8 @@ when running our Phoenix in our `localhost`. Without `.gitkeep`, Git will not upload this empty directory and we'll got an error when running our test on GitLab. - > **Note:** If we add a folder via the GitLab UI, GitLab itself will add the `.gitkeep` to that new dir. + NOTE: **Note:** + If we add a folder via the GitLab UI, GitLab itself will add the `.gitkeep` to that new dir. Now, let's run a local test and see if everything we did didn't break anything. diff --git a/doc/ci/img/ci_lint.png b/doc/ci/img/ci_lint.png Binary files differnew file mode 100644 index 00000000000..e62de011293 --- /dev/null +++ b/doc/ci/img/ci_lint.png diff --git a/doc/ci/img/ci_lint_dry_run.png b/doc/ci/img/ci_lint_dry_run.png Binary files differnew file mode 100644 index 00000000000..4092b66d534 --- /dev/null +++ b/doc/ci/img/ci_lint_dry_run.png diff --git a/doc/ci/img/metrics_reports_advanced_v13_0.png b/doc/ci/img/metrics_reports_advanced_v13_0.png Binary files differindex c092c6a84e4..e96fdcf620a 100644 --- a/doc/ci/img/metrics_reports_advanced_v13_0.png +++ b/doc/ci/img/metrics_reports_advanced_v13_0.png diff --git a/doc/ci/introduction/index.md b/doc/ci/introduction/index.md index db18624d4e9..c97f4e51d30 100644 --- a/doc/ci/introduction/index.md +++ b/doc/ci/introduction/index.md @@ -230,7 +230,7 @@ syntax and with its attributes. This document [introduces the concepts of GitLab CI/CD in the scope of GitLab Pages](../../user/project/pages/getting_started/pages_from_scratch.md), for deploying static websites. Although it's meant for users who want to write their own Pages script from scratch, it also serves as an introduction to the setup process for GitLab CI/CD. -It covers the very first general steps of writing a CI/CD configuration +It covers the first general steps of writing a CI/CD configuration file, so we recommend you read through it to understand GitLab's CI/CD logic, and learn how to write your own script (or tweak an existing one) for any application. diff --git a/doc/ci/jenkins/index.md b/doc/ci/jenkins/index.md index 1d029dcdd14..34600dd6540 100644 --- a/doc/ci/jenkins/index.md +++ b/doc/ci/jenkins/index.md @@ -1,359 +1,5 @@ --- -stage: Verify -group: Continuous Integration -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers -comments: false -type: index, howto +redirect_to: '../migration/jenkins.md' --- -# Migrating from Jenkins - -A lot of GitLab users have successfully migrated to GitLab CI/CD from Jenkins. To make this -easier if you're just getting started, we've collected several resources here that you might find useful -before diving in. Think of this page as a "GitLab CI/CD for Jenkins Users" guide. - -The following list of recommended steps was created after observing organizations -that were able to quickly complete this migration: - -1. Start by reading the GitLab CI/CD [Quick Start Guide](../quick_start/README.md) and [important product differences](#important-product-differences). -1. Learn the importance of [managing the organizational transition](#managing-the-organizational-transition). -1. [Add Runners](../runners/README.md) to your GitLab instance. -1. Educate and enable your developers to independently perform the following steps in their projects: - 1. Review the [Quick Start Guide](../quick_start/README.md) and [Pipeline Configuration Reference](../yaml/README.md). - 1. Use the [Jenkins Wrapper](#jenkinsfile-wrapper) to temporarily maintain fragile Jenkins jobs. - 1. Migrate the build and CI jobs and configure them to show results directly in your merge requests. They can use [Auto DevOps](../../topics/autodevops/index.md) as a starting point, and [customize](../../topics/autodevops/customize.md) or [decompose](../../topics/autodevops/customize.md#using-components-of-auto-devops) the configuration as needed. - 1. Add [Review Apps](../review_apps/index.md). - 1. Migrate the deployment jobs using [cloud deployment templates](../cloud_deployment/index.md), adding [environments](../environments/index.md), and [deploy boards](../..//user/project/deploy_boards.md). - 1. Work to unwrap any jobs still running with the use of the Jenkins wrapper. -1. Take stock of any common CI/CD job definitions then create and share [templates](#templates) for them. - -For an example of how to convert a Jenkins pipeline into a GitLab CI/CD pipeline, -or how to use Auto DevOps to test your code automatically, watch the -[Migrating from Jenkins to GitLab](https://www.youtube.com/watch?v=RlEVGOpYF5Y) video. - -Otherwise, read on for important information that will help you get the ball rolling. Welcome -to GitLab! - -If you have questions that are not answered here, the [GitLab community forum](https://forum.gitlab.com/) -can be a great resource. - -## Managing the organizational transition - -An important part of transitioning from Jenkins to GitLab is the cultural and organizational -changes that comes with the move, and successfully managing them. There are a few -things we have found that helps this: - -- Setting and communicating a clear vision of what your migration goals are helps - your users understand why the effort is worth it. The value will be clear when - the work is done, but people need to be aware while it's in progress too. -- Sponsorship and alignment from the relevant leadership team helps with the point above. -- Spending time educating your users on what's different, sharing this document with them, - and so on will help ensure you are successful. -- Finding ways to sequence or delay parts of the migration can help a lot, but you - don't want to leave things in a non-migrated (or partially-migrated) state for too - long. To gain all the benefits of GitLab, moving your existing Jenkins setup over - as-is, including any current problems, will not be enough. You need to take advantage - of the improvements that GitLab offers, and this requires (eventually) updating - your implementation as part of the transition. - -## JenkinsFile Wrapper - -We are building a [JenkinsFile Wrapper](https://gitlab.com/gitlab-org/jfr-container-builder/) which will allow -you to run a complete Jenkins instance inside of a GitLab job, including plugins. This can help ease the process -of transition, by letting you delay the migration of less urgent pipelines for a period of time. - -If you are interested in helping GitLab test the wrapper, join our [public testing issue](https://gitlab.com/gitlab-org/gitlab/-/issues/215675) for instructions and to provide your feedback. - -## Important product differences - -There are some high level differences between the products worth mentioning: - -- With GitLab you don't need a root `pipeline` keyword to wrap everything. -- The way pipelines are triggered and [trigger other pipelines](../yaml/README.md#trigger) - is different than Jenkins. GitLab pipelines can be triggered: - - - on push - - on [schedule](../pipelines/schedules.md) - - from the [GitLab UI](../pipelines/index.md#run-a-pipeline-manually) - - by [API call](../triggers/README.md) - - by [webhook](../triggers/README.md#triggering-a-pipeline-from-a-webhook) - - by [ChatOps](../chatops/README.md) - -- You can control which jobs run in which cases, depending on how they are triggered, - with the [`rules` syntax](../yaml/README.md#rules). -- GitLab [pipeline scheduling concepts](../pipelines/schedules.md) are also different than with Jenkins. -- You can reuse pipeline configurations using the [`include` keyword](../yaml/README.md#include) - and [templates](#templates). Your templates can be kept in a central repository (with different - permissions), and then any project can use them. This central project could also - contain scripts or other reusable code. -- You can also use the [`extends` keyword](../yaml/README.md#extends) to reuse configuration - within a single pipeline configuration. -- All jobs within a single stage always run in parallel, and all stages run in sequence. We are planning - to allow certain jobs to break this sequencing as needed with our [directed acyclic graph](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/47063) - feature. -- The [`parallel`](../yaml/README.md#parallel) keyword can automatically parallelize tasks, - like tests that support parallelization. -- Normally all jobs within a single stage run in parallel, and all stages run in sequence. - There are different [pipeline architectures](../pipelines/pipeline_architectures.md) - that allow you to change this behavior. -- The new [`rules` syntax](../yaml/README.md#rules) is the recommended method of - controlling when different jobs run. It is more powerful than the `only/except` syntax. -- One important difference is that jobs run independently of each other and have a - fresh environment in each job. Passing artifacts between jobs is controlled using the - [`artifacts`](../yaml/README.md#artifacts) and [`dependencies`](../yaml/README.md#dependencies) - keywords. When finished, the planned [Workspaces](https://gitlab.com/gitlab-org/gitlab/-/issues/29265) - feature will allow you to more easily persist a common workspace between serial jobs. -- The `.gitlab-ci.yml` file is checked in to the root of your repository, much like a Jenkinsfile, but - is in the YAML format (see [complete reference](../yaml/README.md)) instead of a Groovy DSL. It's most - analogous to the declarative Jenkinsfile format. -- Manual approvals or gates can be set up as [`when:manual` jobs](../yaml/README.md#whenmanual). These can - also leverage [`protected environments`](../yaml/README.md#protecting-manual-jobs-premium) - to control who is able to approve them. -- GitLab comes with a [container registry](../../user/packages/container_registry/index.md), and we recommend using - container images to set up your build environment. For example, set up one pipeline that builds your build environment - itself and publish that to the container registry. Then, have your pipelines use this instead of each building their - own environment, which will be slower and may be less consistent. We have extensive docs on [how to use the Container Registry](../../user/packages/container_registry/index.md). -- A central utilities repository can be a great place to put assorted scheduled jobs - or other manual jobs that function like utilities. Jenkins installations tend to - have a few of these. - -## Agents vs. Runners - -Both Jenkins agents and GitLab Runners are the hosts that run jobs. To convert the -Jenkins agent, simply uninstall it and then [install and register the runner](../runners/README.md). -Runners do not require much overhead, so you can size them similarly to the Jenkins -agents you were using. - -There are some important differences in the way Runners work in comparison to agents: - -- Runners can be set up as [shared across an instance, be added at the group level, or set up at the project level](../runners/README.md#types-of-runners). - They will self-select jobs from the scopes you've defined automatically. -- You can also [use tags](../runners/README.md#use-tags-to-limit-the-number-of-jobs-using-the-runner) for finer control, and - associate runners with specific jobs. For example, you can use a tag for jobs that - require dedicated, more powerful, or specific hardware. -- GitLab has [autoscaling for Runners](https://docs.gitlab.com/runner/configuration/autoscale.html) - which will let you configure them to be provisioned as needed, and scaled down when not. - This is similar to ephemeral agents in Jenkins. - -If you are using `gitlab.com`, you can take advantage of our [shared Runner fleet](../../user/gitlab_com/index.md#shared-runners) -to run jobs without provisioning your own Runners. We are investigating making them -[available for self-managed instances](https://gitlab.com/groups/gitlab-org/-/epics/835) -as well. - -## Groovy vs. YAML - -Jenkins Pipelines are based on [Groovy](https://groovy-lang.org/), so the pipeline specification is written as code. -GitLab works a bit differently, we use the more highly structured [YAML](https://yaml.org/) format, which -places scripting elements inside of `script:` blocks separate from the pipeline specification itself. - -This is a strength of GitLab, in that it helps keep the learning curve much simpler to get up and running -and avoids some of the problem of unconstrained complexity which can make your Jenkinsfile hard to understand -and manage. - -That said, we do of course still value DRY (don't repeat yourself) principles and want to ensure that -behaviors of your jobs can be codified once and applied as needed. You can use the `extends:` syntax to -[reuse configuration in your jobs](../yaml/README.md#extends), and `include:` can -be used to [reuse pipeline configurations](../yaml/README.md#include) in pipelines -in different projects: - -```yaml -.in-docker: - tags: - - docker - image: alpine - -rspec: - extends: - - .in-docker - script: - - rake rspec -``` - -## Artifact publishing - -Artifacts may work a bit differently than you've used them with Jenkins. In GitLab, any job can define -a set of artifacts to be saved by using the `artifacts:` keyword. This can be configured to point to a file -or set of files that can then be persisted from job to job. Read more on our detailed -[artifacts documentation](../pipelines/job_artifacts.md): - -```yaml -pdf: - script: xelatex mycv.tex - artifacts: - paths: - - ./mycv.pdf - - ./output/ - expire_in: 1 week -``` - -Additionally, we have package management features like a built-in container, NPM, and Maven registry that you -can leverage. You can see the complete list of packaging features (which includes links to documentation) -in the [Packaging section of our documentation](../../README.md#package). - -## Integrated features - -Where you may have used plugins to get things like code quality, unit tests, security scanning, and so on working in Jenkins, -GitLab takes advantage of our connected ecosystem to automatically pull these kinds of results into -your Merge Requests, pipeline details pages, and other locations. You may find that you actually don't -need to configure anything to have these appear. - -If they aren't working as expected, or if you'd like to see what's available, our [CI feature index](../README.md#feature-set) has the full list -of bundled features and links to the documentation for each. - -### Templates - -For advanced CI/CD teams, project templates can enable the reuse of pipeline configurations, -as well as encourage inner sourcing. - -In self-managed GitLab instances, you can build an [Instance Template Repository](../../user/admin_area/settings/instance_template_repository.md). -Development teams across the whole organization can select templates from a dropdown menu. -A group administrator is able to set a group to use as the source for the -[custom project templates](../../user/admin_area/custom_project_templates.md), which can -be used by all projects in the group. An instance administrator can set a group as -the source for [instance project templates](../../user/group/custom_project_templates.md), -which can be used by projects in that instance. - -## Converting a declarative Jenkinsfile - -A declarative Jenkinsfile contains "Sections" and "Directives" which are used to control the behavior of your -pipelines. There are equivalents for all of these in GitLab, which we've documented below. - -This section is based on the [Jenkinsfile syntax documentation](https://www.jenkins.io/doc/book/pipeline/syntax/) -and is meant to be a mapping of concepts there to concepts in GitLab. - -### Sections - -#### `agent` - -The agent section is used to define how a pipeline will be executed. For GitLab, we use the [GitLab Runner](../runners/README.md) -to provide this capability. You can configure your own runners in Kubernetes or on any host, or take advantage -of our shared runner fleet (note that the shared runner fleet is only available for GitLab.com users.) The link above will bring you to the documentation which will describe how to get -up and running quickly. We also support using [tags](../runners/README.md#use-tags-to-limit-the-number-of-jobs-using-the-runner) to direct different jobs -to different Runners (execution agents). - -The `agent` section also allows you to define which Docker images should be used for execution, for which we use -the [`image`](../yaml/README.md#image) keyword. The `image` can be set on a single job or at the top level, in which -case it will apply to all jobs in the pipeline: - -```yaml -my_job: - image: alpine - ... -``` - -#### `post` - -The `post` section defines the actions that should be performed at the end of the pipeline. GitLab also supports -this through the use of stages. You can define your stages as follows, and any jobs assigned to the `before_pipeline` -or `after_pipeline` stages will run as expected. You can call these stages anything you like: - -```yaml -stages: - - before_pipeline - - build - - test - - deploy - - after_pipeline -``` - -Setting a step to be performed before and after any job can be done via the -[`before_script` and `after_script` keywords](../yaml/README.md#before_script-and-after_script): - -```yaml -default: - before_script: - - echo "I run before any jobs starts in the entire pipeline, and can be responsible for setting up the environment." -``` - -#### `stages` - -GitLab CI/CD also lets you define stages, but is a little bit more free-form to configure. The GitLab [`stages` keyword](../yaml/README.md#stages) -is a top level setting that enumerates the list of stages, but you are not required to nest individual jobs underneath -the `stages` section. Any job defined in the `.gitlab-ci.yml` can be made a part of any stage through use of the -[`stage:` keyword](../yaml/README.md#stage). - -Note that, unless otherwise specified, every pipeline is instantiated with a `build`, `test`, and `deploy` stage -which are run in that order. Jobs that have no `stage` defined are placed by default in the `test` stage. -Of course, each job that refers to a stage must refer to a stage that exists in the pipeline configuration. - -```yaml -stages: - - build - - test - - deploy - -my_job: - stage: build - ... -``` - -#### `steps` - -The `steps` section is equivalent to the [`script` section](../yaml/README.md#script) of an individual job. This is -a simple YAML array with each line representing an individual command to be run: - -```yaml -my_job: - script: - - echo "hello! the current time is:" - - time - ... -``` - -### Directives - -#### `environment` - -In GitLab, we use the [`variables` keyword](../yaml/README.md#variables) to define different variables at runtime. -These can also be set up through the GitLab UI, under CI/CD settings. See also our [general documentation on variables](../variables/README.md), -including the section on [protected variables](../variables/README.md#protect-a-custom-variable) which can be used -to limit access to certain variables to certain environments or runners: - -```yaml -variables: - POSTGRES_USER: user - POSTGRES_PASSWORD: testing_password -``` - -#### `options` - -Here, options for different things exist associated with the object in question itself. For example, options related -to jobs are defined in relation to the job itself. If you're looking for a certain option, you should be able to find -where it's located by searching our [complete configuration reference](../yaml/README.md) page. - -#### `parameters` - -GitLab does not require you to define which variables you want to be available when starting a manual job. A user -can provide any variables they like. - -#### `triggers` / `cron` - -Because GitLab is integrated tightly with Git, SCM polling options for triggers are not needed. We support an easy to use -[syntax for scheduling pipelines](../pipelines/schedules.md). - -#### `tools` - -GitLab does not support a separate `tools` directive. Our best-practice recommendation is to use pre-built -container images, which can be cached, and can be built to already contain the tools you need for your pipelines. Pipelines can -be set up to automatically build these images as needed and deploy them to the [container registry](../../user/packages/container_registry/index.md). - -If you're not using container images with Docker/Kubernetes, for example on Mac or FreeBSD, then the `shell` executor does require you to -set up your environment either in advance or as part of the jobs. You could create a `before_script` -action that handles this for you. - -#### `input` - -Similar to the `parameters` keyword, this is not needed because a manual job can always be provided runtime -variable entry. - -#### `when` - -GitLab does support a [`when` keyword](../yaml/README.md#when) which is used to indicate when a job should be -run in case of (or despite) failure, but most of the logic for controlling pipelines can be found in -our very powerful [`only/except` rules system](../yaml/README.md#onlyexcept-basic) -(see also our [advanced syntax](../yaml/README.md#onlyexcept-basic)): - -```yaml -my_job: - only: [branches] -``` +This document was moved to [another location](../migration/jenkins.md). diff --git a/doc/ci/junit_test_reports.md b/doc/ci/junit_test_reports.md index aedda3c8341..8bc55a6e4f3 100644 --- a/doc/ci/junit_test_reports.md +++ b/doc/ci/junit_test_reports.md @@ -66,10 +66,6 @@ execution time and the error output. ## How to set it up -NOTE: **Note:** -For a list of supported languages on JUnit tests, check the -[Wikipedia article](https://en.wikipedia.org/wiki/JUnit#Ports). - To enable the JUnit reports in merge requests, you need to add [`artifacts:reports:junit`](pipelines/job_artifacts.md#artifactsreportsjunit) in `.gitlab-ci.yml`, and specify the path(s) of the generated test reports. @@ -80,9 +76,11 @@ collects the JUnit test report from each job. After each job is executed, the XML reports are stored in GitLab as artifacts and their results are shown in the merge request widget. +To make the JUnit output files browsable, include them with the +[`artifacts:paths`](yaml/README.md#artifactspaths) keyword as well, as shown in the [Ruby example](#ruby-example). + NOTE: **Note:** -If you also want the ability to browse JUnit output files, include the -[`artifacts:paths`](yaml/README.md#artifactspaths) keyword. An example of this is shown in the Ruby example below. +You cannot have multiple tests with the same name and class in your JUnit report. ### Ruby example @@ -140,6 +138,9 @@ java: junit: build/test-results/test/**/TEST-*.xml ``` +NOTE: **Note:** +Support for `**` was added in [GitLab Runner 13.0](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/2620). + #### Maven For parsing [Surefire](https://maven.apache.org/surefire/maven-surefire-plugin/) @@ -236,9 +237,8 @@ Test: ## Viewing JUnit test reports on GitLab -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/24792) in GitLab 12.5. -> - It's deployed behind a feature flag, disabled by default. -> - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enabling-the-junit-test-reports-feature-core-only). **(CORE ONLY)** +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/24792) in GitLab 12.5 behind a feature flag (`junit_pipeline_view`), disabled by default. +> - The feature flag was removed and the feature was [made generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/216478) in GitLab 13.3. If JUnit XML files are generated and uploaded as part of a pipeline, these reports can be viewed inside the pipelines details page. The **Tests** tab on this page will @@ -247,27 +247,10 @@ display a list of test suites and cases reported from the XML file.  You can view all the known test suites and click on each of these to see further -details, including the cases that makeup the suite. Cases are ordered by status, -with failed showing at the top, skipped next and successful cases last. +details, including the cases that make up the suite. You can also retrieve the reports via the [GitLab API](../api/pipelines.md#get-a-pipelines-test-report). -### Enabling the JUnit test reports feature **(CORE ONLY)** - -This feature comes with the `:junit_pipeline_view` feature flag disabled by default. This -feature is disabled due to some performance issues with very large data sets. -When [the performance is improved](https://gitlab.com/groups/gitlab-org/-/epics/2854), the feature will be enabled by default. - -To enable this feature, ask a GitLab administrator with [Rails console access](../administration/feature_flags.md#how-to-enable-and-disable-features-behind-flags) to run the -following command: - -```ruby -Feature.enable(:junit_pipeline_view) - -# Enable the feature for a specific project, GitLab 13.0 and above only. -Feature.enable(:junit_pipeline_view, Project.find(<your-project-id-here>)) -``` - ## Viewing JUnit screenshots on GitLab > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/202114) in GitLab 13.0. diff --git a/doc/ci/large_repositories/index.md b/doc/ci/large_repositories/index.md index b4652050654..00e912d47dc 100644 --- a/doc/ci/large_repositories/index.md +++ b/doc/ci/large_repositories/index.md @@ -124,6 +124,10 @@ are dependent on Git version. [`GIT_FETCH_EXTRA_FLAGS`](../yaml/README.md#git-fetch-extra-flags) allows you to modify `git fetch` behavior by passing extra flags. +For example, if your project contains a large number of tags that your CI jobs don't rely on, +you could add [`--no-tags`](https://git-scm.com/docs/git-fetch#Documentation/git-fetch.txt---no-tags) +to the extra flags to make your fetches faster and more compact. + See the [`GIT_FETCH_EXTRA_FLAGS` documentation](../yaml/README.md#git-fetch-extra-flags) for more information. @@ -253,3 +257,11 @@ concurrent = 4 This makes the cloning configuration to be part of given Runner and does not require us to update each `.gitlab-ci.yml`. + +## Pre-clone step + +For very active repositories with a large number of references and files, you can also +optimize your CI jobs by seeding repository data with GitLab Runner's [`pre_clone_script`](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-runners-section). + +See [our development documentation](../../development/pipelines.md#pre-clone-step) for +an overview of how we implemented this approach on GitLab.com for the main GitLab repository. diff --git a/doc/ci/lint.md b/doc/ci/lint.md new file mode 100644 index 00000000000..716a4218d97 --- /dev/null +++ b/doc/ci/lint.md @@ -0,0 +1,41 @@ +# CI Lint + +If you want to test the validity of your GitLab CI/CD configuration before committing +the changes, you can use the CI Lint tool. This tool checks for syntax and logical +errors by default, and can simulate pipeline creation to try to find more complicated +issues as well. + +To access the CI Lint tool, navigate to **CI/CD > Pipelines** or **CI/CD > Jobs** +in your project and click **CI lint**. + +## Validate basic logic and syntax + +By default, the CI lint checks the syntax of your CI YAML configuration and also runs +some basic logical validations. + +To use the CI lint, paste a complete CI configuration (`.gitlab-ci.yml` for example) +into the text box and click **Validate**: + + + +## Pipeline simulation + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/229794) in GitLab 13.3. + +Not all pipeline configuration issues can be found by the [basic CI lint validation](#validate-basic-logic-and-syntax). +You can simulate the creation of a pipeline for deeper validation that can discover +more complicated issues. + +To validate the configuration by running a pipeline simulation: + +1. Paste the GitLab CI configuration to verify into the text box. +1. Click the **Simulate pipeline creation for the default branch** checkbox. +1. Click **Validate**. + + + +### Pipeline simulation limitations + +Simulations run as `git push` events against the default branch. You must have +[permissions](../user/permissions.md#project-members-permissions) to create pipelines +on this branch to validate with a simulation. diff --git a/doc/ci/merge_request_pipelines/index.md b/doc/ci/merge_request_pipelines/index.md index 2a6008e6307..73b971c2d41 100644 --- a/doc/ci/merge_request_pipelines/index.md +++ b/doc/ci/merge_request_pipelines/index.md @@ -166,31 +166,33 @@ Read the [documentation on Pipelines for Merged Results](pipelines_for_merged_re Read the [documentation on Merge Trains](pipelines_for_merged_results/merge_trains/index.md). -## Important notes about merge requests from forked projects +## Run pipelines in the parent project for merge requests from a forked project **(STARTER)** -Note that the current behavior is subject to change. In the usual contribution -flow, external contributors follow the following steps: +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/217451) in GitLab 13.3. -1. Fork a parent project. -1. Create a merge request from the forked project that targets the `master` branch - in the parent project. -1. A pipeline runs on the merge request. -1. A maintainer from the parent project checks the pipeline result, and merge - into a target branch if the latest pipeline has passed. +By default, external contributors working from forks can't create pipelines in the +parent project. When a pipeline for merge requests is triggered by a merge request +coming from a fork: -Currently, those pipelines are created in a **forked** project, not in the -parent project. This means you cannot completely trust the pipeline result, -because, technically, external contributors can disguise their pipeline results -by tweaking their GitLab Runner in the forked project. +- It's created and runs in the fork (source) project, not the parent (target) project. +- It uses the fork project's CI/CD configuration and resources. -There are multiple reasons why GitLab doesn't allow those pipelines to be -created in the parent project, but one of the biggest reasons is security concern. -External users could steal secret variables from the parent project by modifying -`.gitlab-ci.yml`, which could be some sort of credentials. This should not happen. +Sometimes parent project members want the pipeline to run in the parent +project. This could be to ensure that the post-merge pipeline passes in the parent project. +For example, a fork project could try to use a corrupted Runner that doesn't execute +test scripts properly, but reports a passed pipeline. Reviewers in the parent project +could mistakenly trust the merge request because it passed a faked pipeline. -We're discussing a secure solution of running pipelines for merge requests -that are submitted from forked projects, -see [the issue about the permission extension](https://gitlab.com/gitlab-org/gitlab/-/issues/11934). +Parent project members with at least [Developer permissions](../../user/permissions.md) +can create pipelines in the parent project for merge requests +from a forked project. In the merge request, go to the **Pipelines** and click +**Run Pipeline** button. + +CAUTION: **Caution:** +Fork merge requests could contain malicious code that tries to steal secrets in the +parent project when the pipeline runs, even before merge. Reviewers must carefully +check the changes in the merge request before triggering the pipeline. GitLab shows +a warning that must be accepted before the pipeline can be triggered. ## Additional predefined variables @@ -206,19 +208,19 @@ The variable names begin with the `CI_MERGE_REQUEST_` prefix. ### Two pipelines created when pushing to a merge request If you are experiencing duplicated pipelines when using `rules`, take a look at -the [important differences between `rules` and `only`/`except`](../yaml/README.md#differences-between-rules-and-onlyexcept), +the [important differences between `rules` and `only`/`except`](../yaml/README.md#prevent-duplicate-pipelines), which will help you get your starting configuration correct. If you are seeing two pipelines when using `only/except`, please see the caveats related to using `only/except` above (or, consider moving to `rules`). +It is not possible to run a job for branch pipelines first, then only for merge request +pipelines after the merge request is created (skipping the duplicate branch pipeline). See +the [related issue](https://gitlab.com/gitlab-org/gitlab/-/issues/201845) for more details. + ### Two pipelines created when pushing an invalid CI configuration file Pushing to a branch with an invalid CI configuration file can trigger the creation of two types of failed pipelines. One pipeline is a failed merge request pipeline, and the other is a failed branch pipeline, but both are caused by the same invalid configuration. - -In rare cases, duplicate pipelines are created. - -See [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/201845) for details. diff --git a/doc/ci/merge_request_pipelines/pipelines_for_merged_results/index.md b/doc/ci/merge_request_pipelines/pipelines_for_merged_results/index.md index 84fbefb080f..685c93b3be4 100644 --- a/doc/ci/merge_request_pipelines/pipelines_for_merged_results/index.md +++ b/doc/ci/merge_request_pipelines/pipelines_for_merged_results/index.md @@ -45,8 +45,6 @@ To enable pipelines for merge results: - You must have maintainer [permissions](../../../user/permissions.md). - You must be using [GitLab Runner](https://gitlab.com/gitlab-org/gitlab-runner) 11.9 or later. -- You must not be forking or using cross-repo workflows. To follow progress, - see [#11934](https://gitlab.com/gitlab-org/gitlab/-/issues/11934). - You must not be using [fast forward merges](../../../user/project/merge_requests/fast_forward_merge.md) yet. To follow progress, see [#58226](https://gitlab.com/gitlab-org/gitlab/-/issues/26996). diff --git a/doc/ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md b/doc/ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md index e12c9d109cc..d91d88c8e12 100644 --- a/doc/ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md +++ b/doc/ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md @@ -11,6 +11,8 @@ last_update: 2019-07-03 > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9186) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.0. > - [Squash and merge](../../../../user/project/merge_requests/squash_and_merge.md) support [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/13001) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.6. +For more information about why you might want to use Merge Trains, read [How merge trains keep your master green](https://about.gitlab.com/blog/2020/01/30/all-aboard-merge-trains/). + When [pipelines for merged results](../index.md#pipelines-for-merged-results-premium) are enabled, the pipeline jobs run as if the changes from your source branch have already been merged into the target branch. diff --git a/doc/ci/migration/circleci.md b/doc/ci/migration/circleci.md index 625b15ca4fb..78705815c24 100644 --- a/doc/ci/migration/circleci.md +++ b/doc/ci/migration/circleci.md @@ -1,4 +1,7 @@ --- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers comments: false type: index, howto --- diff --git a/doc/ci/migration/jenkins.md b/doc/ci/migration/jenkins.md new file mode 100644 index 00000000000..1d029dcdd14 --- /dev/null +++ b/doc/ci/migration/jenkins.md @@ -0,0 +1,359 @@ +--- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +comments: false +type: index, howto +--- + +# Migrating from Jenkins + +A lot of GitLab users have successfully migrated to GitLab CI/CD from Jenkins. To make this +easier if you're just getting started, we've collected several resources here that you might find useful +before diving in. Think of this page as a "GitLab CI/CD for Jenkins Users" guide. + +The following list of recommended steps was created after observing organizations +that were able to quickly complete this migration: + +1. Start by reading the GitLab CI/CD [Quick Start Guide](../quick_start/README.md) and [important product differences](#important-product-differences). +1. Learn the importance of [managing the organizational transition](#managing-the-organizational-transition). +1. [Add Runners](../runners/README.md) to your GitLab instance. +1. Educate and enable your developers to independently perform the following steps in their projects: + 1. Review the [Quick Start Guide](../quick_start/README.md) and [Pipeline Configuration Reference](../yaml/README.md). + 1. Use the [Jenkins Wrapper](#jenkinsfile-wrapper) to temporarily maintain fragile Jenkins jobs. + 1. Migrate the build and CI jobs and configure them to show results directly in your merge requests. They can use [Auto DevOps](../../topics/autodevops/index.md) as a starting point, and [customize](../../topics/autodevops/customize.md) or [decompose](../../topics/autodevops/customize.md#using-components-of-auto-devops) the configuration as needed. + 1. Add [Review Apps](../review_apps/index.md). + 1. Migrate the deployment jobs using [cloud deployment templates](../cloud_deployment/index.md), adding [environments](../environments/index.md), and [deploy boards](../..//user/project/deploy_boards.md). + 1. Work to unwrap any jobs still running with the use of the Jenkins wrapper. +1. Take stock of any common CI/CD job definitions then create and share [templates](#templates) for them. + +For an example of how to convert a Jenkins pipeline into a GitLab CI/CD pipeline, +or how to use Auto DevOps to test your code automatically, watch the +[Migrating from Jenkins to GitLab](https://www.youtube.com/watch?v=RlEVGOpYF5Y) video. + +Otherwise, read on for important information that will help you get the ball rolling. Welcome +to GitLab! + +If you have questions that are not answered here, the [GitLab community forum](https://forum.gitlab.com/) +can be a great resource. + +## Managing the organizational transition + +An important part of transitioning from Jenkins to GitLab is the cultural and organizational +changes that comes with the move, and successfully managing them. There are a few +things we have found that helps this: + +- Setting and communicating a clear vision of what your migration goals are helps + your users understand why the effort is worth it. The value will be clear when + the work is done, but people need to be aware while it's in progress too. +- Sponsorship and alignment from the relevant leadership team helps with the point above. +- Spending time educating your users on what's different, sharing this document with them, + and so on will help ensure you are successful. +- Finding ways to sequence or delay parts of the migration can help a lot, but you + don't want to leave things in a non-migrated (or partially-migrated) state for too + long. To gain all the benefits of GitLab, moving your existing Jenkins setup over + as-is, including any current problems, will not be enough. You need to take advantage + of the improvements that GitLab offers, and this requires (eventually) updating + your implementation as part of the transition. + +## JenkinsFile Wrapper + +We are building a [JenkinsFile Wrapper](https://gitlab.com/gitlab-org/jfr-container-builder/) which will allow +you to run a complete Jenkins instance inside of a GitLab job, including plugins. This can help ease the process +of transition, by letting you delay the migration of less urgent pipelines for a period of time. + +If you are interested in helping GitLab test the wrapper, join our [public testing issue](https://gitlab.com/gitlab-org/gitlab/-/issues/215675) for instructions and to provide your feedback. + +## Important product differences + +There are some high level differences between the products worth mentioning: + +- With GitLab you don't need a root `pipeline` keyword to wrap everything. +- The way pipelines are triggered and [trigger other pipelines](../yaml/README.md#trigger) + is different than Jenkins. GitLab pipelines can be triggered: + + - on push + - on [schedule](../pipelines/schedules.md) + - from the [GitLab UI](../pipelines/index.md#run-a-pipeline-manually) + - by [API call](../triggers/README.md) + - by [webhook](../triggers/README.md#triggering-a-pipeline-from-a-webhook) + - by [ChatOps](../chatops/README.md) + +- You can control which jobs run in which cases, depending on how they are triggered, + with the [`rules` syntax](../yaml/README.md#rules). +- GitLab [pipeline scheduling concepts](../pipelines/schedules.md) are also different than with Jenkins. +- You can reuse pipeline configurations using the [`include` keyword](../yaml/README.md#include) + and [templates](#templates). Your templates can be kept in a central repository (with different + permissions), and then any project can use them. This central project could also + contain scripts or other reusable code. +- You can also use the [`extends` keyword](../yaml/README.md#extends) to reuse configuration + within a single pipeline configuration. +- All jobs within a single stage always run in parallel, and all stages run in sequence. We are planning + to allow certain jobs to break this sequencing as needed with our [directed acyclic graph](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/47063) + feature. +- The [`parallel`](../yaml/README.md#parallel) keyword can automatically parallelize tasks, + like tests that support parallelization. +- Normally all jobs within a single stage run in parallel, and all stages run in sequence. + There are different [pipeline architectures](../pipelines/pipeline_architectures.md) + that allow you to change this behavior. +- The new [`rules` syntax](../yaml/README.md#rules) is the recommended method of + controlling when different jobs run. It is more powerful than the `only/except` syntax. +- One important difference is that jobs run independently of each other and have a + fresh environment in each job. Passing artifacts between jobs is controlled using the + [`artifacts`](../yaml/README.md#artifacts) and [`dependencies`](../yaml/README.md#dependencies) + keywords. When finished, the planned [Workspaces](https://gitlab.com/gitlab-org/gitlab/-/issues/29265) + feature will allow you to more easily persist a common workspace between serial jobs. +- The `.gitlab-ci.yml` file is checked in to the root of your repository, much like a Jenkinsfile, but + is in the YAML format (see [complete reference](../yaml/README.md)) instead of a Groovy DSL. It's most + analogous to the declarative Jenkinsfile format. +- Manual approvals or gates can be set up as [`when:manual` jobs](../yaml/README.md#whenmanual). These can + also leverage [`protected environments`](../yaml/README.md#protecting-manual-jobs-premium) + to control who is able to approve them. +- GitLab comes with a [container registry](../../user/packages/container_registry/index.md), and we recommend using + container images to set up your build environment. For example, set up one pipeline that builds your build environment + itself and publish that to the container registry. Then, have your pipelines use this instead of each building their + own environment, which will be slower and may be less consistent. We have extensive docs on [how to use the Container Registry](../../user/packages/container_registry/index.md). +- A central utilities repository can be a great place to put assorted scheduled jobs + or other manual jobs that function like utilities. Jenkins installations tend to + have a few of these. + +## Agents vs. Runners + +Both Jenkins agents and GitLab Runners are the hosts that run jobs. To convert the +Jenkins agent, simply uninstall it and then [install and register the runner](../runners/README.md). +Runners do not require much overhead, so you can size them similarly to the Jenkins +agents you were using. + +There are some important differences in the way Runners work in comparison to agents: + +- Runners can be set up as [shared across an instance, be added at the group level, or set up at the project level](../runners/README.md#types-of-runners). + They will self-select jobs from the scopes you've defined automatically. +- You can also [use tags](../runners/README.md#use-tags-to-limit-the-number-of-jobs-using-the-runner) for finer control, and + associate runners with specific jobs. For example, you can use a tag for jobs that + require dedicated, more powerful, or specific hardware. +- GitLab has [autoscaling for Runners](https://docs.gitlab.com/runner/configuration/autoscale.html) + which will let you configure them to be provisioned as needed, and scaled down when not. + This is similar to ephemeral agents in Jenkins. + +If you are using `gitlab.com`, you can take advantage of our [shared Runner fleet](../../user/gitlab_com/index.md#shared-runners) +to run jobs without provisioning your own Runners. We are investigating making them +[available for self-managed instances](https://gitlab.com/groups/gitlab-org/-/epics/835) +as well. + +## Groovy vs. YAML + +Jenkins Pipelines are based on [Groovy](https://groovy-lang.org/), so the pipeline specification is written as code. +GitLab works a bit differently, we use the more highly structured [YAML](https://yaml.org/) format, which +places scripting elements inside of `script:` blocks separate from the pipeline specification itself. + +This is a strength of GitLab, in that it helps keep the learning curve much simpler to get up and running +and avoids some of the problem of unconstrained complexity which can make your Jenkinsfile hard to understand +and manage. + +That said, we do of course still value DRY (don't repeat yourself) principles and want to ensure that +behaviors of your jobs can be codified once and applied as needed. You can use the `extends:` syntax to +[reuse configuration in your jobs](../yaml/README.md#extends), and `include:` can +be used to [reuse pipeline configurations](../yaml/README.md#include) in pipelines +in different projects: + +```yaml +.in-docker: + tags: + - docker + image: alpine + +rspec: + extends: + - .in-docker + script: + - rake rspec +``` + +## Artifact publishing + +Artifacts may work a bit differently than you've used them with Jenkins. In GitLab, any job can define +a set of artifacts to be saved by using the `artifacts:` keyword. This can be configured to point to a file +or set of files that can then be persisted from job to job. Read more on our detailed +[artifacts documentation](../pipelines/job_artifacts.md): + +```yaml +pdf: + script: xelatex mycv.tex + artifacts: + paths: + - ./mycv.pdf + - ./output/ + expire_in: 1 week +``` + +Additionally, we have package management features like a built-in container, NPM, and Maven registry that you +can leverage. You can see the complete list of packaging features (which includes links to documentation) +in the [Packaging section of our documentation](../../README.md#package). + +## Integrated features + +Where you may have used plugins to get things like code quality, unit tests, security scanning, and so on working in Jenkins, +GitLab takes advantage of our connected ecosystem to automatically pull these kinds of results into +your Merge Requests, pipeline details pages, and other locations. You may find that you actually don't +need to configure anything to have these appear. + +If they aren't working as expected, or if you'd like to see what's available, our [CI feature index](../README.md#feature-set) has the full list +of bundled features and links to the documentation for each. + +### Templates + +For advanced CI/CD teams, project templates can enable the reuse of pipeline configurations, +as well as encourage inner sourcing. + +In self-managed GitLab instances, you can build an [Instance Template Repository](../../user/admin_area/settings/instance_template_repository.md). +Development teams across the whole organization can select templates from a dropdown menu. +A group administrator is able to set a group to use as the source for the +[custom project templates](../../user/admin_area/custom_project_templates.md), which can +be used by all projects in the group. An instance administrator can set a group as +the source for [instance project templates](../../user/group/custom_project_templates.md), +which can be used by projects in that instance. + +## Converting a declarative Jenkinsfile + +A declarative Jenkinsfile contains "Sections" and "Directives" which are used to control the behavior of your +pipelines. There are equivalents for all of these in GitLab, which we've documented below. + +This section is based on the [Jenkinsfile syntax documentation](https://www.jenkins.io/doc/book/pipeline/syntax/) +and is meant to be a mapping of concepts there to concepts in GitLab. + +### Sections + +#### `agent` + +The agent section is used to define how a pipeline will be executed. For GitLab, we use the [GitLab Runner](../runners/README.md) +to provide this capability. You can configure your own runners in Kubernetes or on any host, or take advantage +of our shared runner fleet (note that the shared runner fleet is only available for GitLab.com users.) The link above will bring you to the documentation which will describe how to get +up and running quickly. We also support using [tags](../runners/README.md#use-tags-to-limit-the-number-of-jobs-using-the-runner) to direct different jobs +to different Runners (execution agents). + +The `agent` section also allows you to define which Docker images should be used for execution, for which we use +the [`image`](../yaml/README.md#image) keyword. The `image` can be set on a single job or at the top level, in which +case it will apply to all jobs in the pipeline: + +```yaml +my_job: + image: alpine + ... +``` + +#### `post` + +The `post` section defines the actions that should be performed at the end of the pipeline. GitLab also supports +this through the use of stages. You can define your stages as follows, and any jobs assigned to the `before_pipeline` +or `after_pipeline` stages will run as expected. You can call these stages anything you like: + +```yaml +stages: + - before_pipeline + - build + - test + - deploy + - after_pipeline +``` + +Setting a step to be performed before and after any job can be done via the +[`before_script` and `after_script` keywords](../yaml/README.md#before_script-and-after_script): + +```yaml +default: + before_script: + - echo "I run before any jobs starts in the entire pipeline, and can be responsible for setting up the environment." +``` + +#### `stages` + +GitLab CI/CD also lets you define stages, but is a little bit more free-form to configure. The GitLab [`stages` keyword](../yaml/README.md#stages) +is a top level setting that enumerates the list of stages, but you are not required to nest individual jobs underneath +the `stages` section. Any job defined in the `.gitlab-ci.yml` can be made a part of any stage through use of the +[`stage:` keyword](../yaml/README.md#stage). + +Note that, unless otherwise specified, every pipeline is instantiated with a `build`, `test`, and `deploy` stage +which are run in that order. Jobs that have no `stage` defined are placed by default in the `test` stage. +Of course, each job that refers to a stage must refer to a stage that exists in the pipeline configuration. + +```yaml +stages: + - build + - test + - deploy + +my_job: + stage: build + ... +``` + +#### `steps` + +The `steps` section is equivalent to the [`script` section](../yaml/README.md#script) of an individual job. This is +a simple YAML array with each line representing an individual command to be run: + +```yaml +my_job: + script: + - echo "hello! the current time is:" + - time + ... +``` + +### Directives + +#### `environment` + +In GitLab, we use the [`variables` keyword](../yaml/README.md#variables) to define different variables at runtime. +These can also be set up through the GitLab UI, under CI/CD settings. See also our [general documentation on variables](../variables/README.md), +including the section on [protected variables](../variables/README.md#protect-a-custom-variable) which can be used +to limit access to certain variables to certain environments or runners: + +```yaml +variables: + POSTGRES_USER: user + POSTGRES_PASSWORD: testing_password +``` + +#### `options` + +Here, options for different things exist associated with the object in question itself. For example, options related +to jobs are defined in relation to the job itself. If you're looking for a certain option, you should be able to find +where it's located by searching our [complete configuration reference](../yaml/README.md) page. + +#### `parameters` + +GitLab does not require you to define which variables you want to be available when starting a manual job. A user +can provide any variables they like. + +#### `triggers` / `cron` + +Because GitLab is integrated tightly with Git, SCM polling options for triggers are not needed. We support an easy to use +[syntax for scheduling pipelines](../pipelines/schedules.md). + +#### `tools` + +GitLab does not support a separate `tools` directive. Our best-practice recommendation is to use pre-built +container images, which can be cached, and can be built to already contain the tools you need for your pipelines. Pipelines can +be set up to automatically build these images as needed and deploy them to the [container registry](../../user/packages/container_registry/index.md). + +If you're not using container images with Docker/Kubernetes, for example on Mac or FreeBSD, then the `shell` executor does require you to +set up your environment either in advance or as part of the jobs. You could create a `before_script` +action that handles this for you. + +#### `input` + +Similar to the `parameters` keyword, this is not needed because a manual job can always be provided runtime +variable entry. + +#### `when` + +GitLab does support a [`when` keyword](../yaml/README.md#when) which is used to indicate when a job should be +run in case of (or despite) failure, but most of the logic for controlling pipelines can be found in +our very powerful [`only/except` rules system](../yaml/README.md#onlyexcept-basic) +(see also our [advanced syntax](../yaml/README.md#onlyexcept-basic)): + +```yaml +my_job: + only: [branches] +``` diff --git a/doc/ci/multi_project_pipelines.md b/doc/ci/multi_project_pipelines.md index 40d495a69f1..3df2db8539e 100644 --- a/doc/ci/multi_project_pipelines.md +++ b/doc/ci/multi_project_pipelines.md @@ -62,6 +62,14 @@ together, allowing you to visualize their relationships on pipeline graphs. These relationships are displayed in the pipeline graph by showing inbound and outbound connections for upstream and downstream pipeline dependencies. +When using: + +- Variables or [`rules`](yaml/README.md#rulesif) to control job behavior, the value of + the [`$CI_PIPELINE_SOURCE` predefined variable](variables/predefined_variables.md) is + `pipeline` for multi-project pipeline triggered through the API with `CI_JOB_TOKEN`. +- [`only/except`](yaml/README.md#onlyexcept-basic) to control job behavior, use the + `pipelines` keyword. + ## Creating multi-project pipelines from `.gitlab-ci.yml` > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/8997) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.8. @@ -100,6 +108,15 @@ downstream project (`my/deployment` in this case). If a downstream project can not be found, or a user does not have access rights to create pipeline there, the `staging` job is going to be marked as _failed_. +When using: + +- Variables or [`rules`](yaml/README.md#rulesif) to control job behavior, the value of + the [`$CI_PIPELINE_SOURCE` predefined variable](variables/predefined_variables.md) is + `pipeline` for multi-project pipelines triggered with a bridge job (using the + [`trigger:`](yaml/README.md#trigger) keyword). +- [`only/except`](yaml/README.md#onlyexcept-basic) to control job behavior, use the + `pipelines` keyword. + CAUTION: **Caution:** In the example, `staging` will be marked as succeeded as soon as a downstream pipeline gets created. If you want to display the downstream pipeline's status instead, see @@ -138,6 +155,12 @@ Use: GitLab will use a commit that is currently on the HEAD of the branch when creating a downstream pipeline. +NOTE: **Note:** +Pipelines triggered on a protected branch in a downstream project use the [permissions](../user/permissions.md) +of the user that ran the trigger job in the upstream project. If the user does not +have permission to run CI/CD pipelines against the protected branch, the pipeline fails. See +[pipeline security for protected branches](pipelines/index.md#pipeline-security-on-protected-branches). + ### Passing variables to a downstream pipeline Sometimes you might want to pass variables to a downstream pipeline. @@ -254,5 +277,5 @@ tag in a different project: Any pipelines that complete successfully for new tags in the subscribed project will now trigger a pipeline on the current project's default branch. The maximum -number of upstream pipeline subscriptions is 2, for both the upstream and -downstream projects. +number of upstream pipeline subscriptions is 2 by default, for both the upstream and +downstream projects. This [application limit](../administration/instance_limits.md#number-of-cicd-subscriptions-to-a-project) can be changed on self-managed instances by a GitLab administrator. diff --git a/doc/ci/parent_child_pipelines.md b/doc/ci/parent_child_pipelines.md index c936bd8f5fe..1cfa698bfa5 100644 --- a/doc/ci/parent_child_pipelines.md +++ b/doc/ci/parent_child_pipelines.md @@ -43,8 +43,8 @@ Child pipelines work well with other GitLab CI/CD features: - Since the parent pipeline in `.gitlab-ci.yml` and the child pipeline run as normal pipelines, they can have their own behaviors and sequencing in relation to triggers. -All of this will work with the [`include:`](yaml/README.md#include) feature so you can compose -the child pipeline configuration. +See the [`trigger:`](yaml/README.md#trigger) keyword documentation for full details on how to +include the child pipeline configuration. <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> For an overview, see [Parent-Child Pipelines feature demo](https://youtu.be/n8KpBSqZNbk). @@ -145,6 +145,8 @@ build a matrix of targets and architectures. <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> For an overview, see [Create child pipelines using dynamically generated configurations](https://youtu.be/nMdfus2JWHM). +We also have an [example project using Dynamic Child Pipelines with Jsonnet](https://gitlab.com/gitlab-org/project-templates/jsonnet) which shows how to use a data templating language to generate your `.gitlab-ci.yml` at runtime. You could use a similar process for other templating languages like [Dhall](https://dhall-lang.org/) or [`ytt`](https://get-ytt.io/). + In GitLab 12.9, the child pipeline could fail to be created in certain cases, causing the parent pipeline to fail. This is [resolved in GitLab 12.10](https://gitlab.com/gitlab-org/gitlab/-/issues/209070). diff --git a/doc/ci/pipelines/index.md b/doc/ci/pipelines/index.md index 18b3fe10bec..8419b474d54 100644 --- a/doc/ci/pipelines/index.md +++ b/doc/ci/pipelines/index.md @@ -87,13 +87,13 @@ to its **Pipelines** tab.  -Clicking a pipeline will bring you to the **Pipeline Details** page and show +Click a pipeline to open the **Pipeline Details** page and show the jobs that were run for that pipeline. From here you can cancel a running pipeline, retry jobs on a failed pipeline, or [delete a pipeline](#delete-a-pipeline). [Starting in GitLab 12.3](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/50499), a link to the latest pipeline for the last commit of a given branch is available at `/project/pipelines/[branch]/latest`. -Also, `/project/pipelines/latest` will redirect you to the latest pipeline for the last commit +Also, `/project/pipelines/latest` redirects you to the latest pipeline for the last commit on the project's default branch. [Starting in GitLab 13.0](https://gitlab.com/gitlab-org/gitlab/-/issues/215367), @@ -120,14 +120,14 @@ To execute a pipeline manually: 1. Enter any [environment variables](../variables/README.md) required for the pipeline run. 1. Click the **Create pipeline** button. -The pipeline will execute the jobs as configured. +The pipeline now executes the jobs as configured. ### Run a pipeline by using a URL query string > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/24146) in GitLab 12.5. You can use a query string to pre-populate the **Run Pipeline** page. For example, the query string -`.../pipelines/new?ref=my_branch&var[foo]=bar&file_var[file_foo]=file_bar` will pre-populate the +`.../pipelines/new?ref=my_branch&var[foo]=bar&file_var[file_foo]=file_bar` pre-populates the **Run Pipeline** page with: - **Run for** field: `my_branch`. @@ -174,7 +174,7 @@ stage has a job with a manual action. > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/27188) in GitLab 11.11. Multiple manual actions in a single stage can be started at the same time using the "Play all manual" button. -Once the user clicks this button, each individual manual action will be triggered and refreshed +Once you click this button, each individual manual action is triggered and refreshed to an updated status. This functionality is only available: @@ -193,7 +193,7 @@ page, then using the **Delete** button.  CAUTION: **Warning:** -Deleting a pipeline will expire all pipeline caches, and delete all related objects, +Deleting a pipeline expires all pipeline caches, and deletes all related objects, such as builds, logs, artifacts, and triggers. **This action cannot be undone.** ### Pipeline quotas @@ -252,6 +252,7 @@ on that specific branch: - Run manual pipelines (using the [Web UI](#run-a-pipeline-manually) or [pipelines API](#pipelines-api)). - Run scheduled pipelines. - Run pipelines using triggers. +- Run on-demand DAST scan. - Trigger manual actions on existing pipelines. - Retry or cancel existing jobs (using the Web UI or pipelines API). @@ -263,13 +264,13 @@ sensitive information like deployment credentials and tokens. branches, preventing untrusted code from executing on the protected runner and preserving deployment keys and other credentials from being unintentionally accessed. In order to ensure that jobs intended to be executed on protected -runners will not use regular runners, they must be tagged accordingly. +runners do not use regular runners, they must be tagged accordingly. ## View jobs in a pipeline When you access a pipeline, you can see the related jobs for that pipeline. -Clicking an individual job will show you its job log, and allow you to: +Clicking an individual job shows you its job log, and allows you to: - Cancel the job. - Retry the job. @@ -325,10 +326,10 @@ If you have many similar jobs, your [pipeline graph](#visualize-pipelines) becom to read. You can automatically group similar jobs together. If the job names are formatted in a certain way, -they will be collapsed into a single group in regular pipeline graphs (not the mini graphs). +they are collapsed into a single group in regular pipeline graphs (not the mini graphs). -You'll know when a pipeline has grouped jobs if you don't see the retry or -cancel button inside them. Hovering over them will show the number of grouped +You can recognize when a pipeline has grouped jobs if you don't see the retry or +cancel button inside them. Hovering over them shows the number of grouped jobs. Click to expand them.  @@ -342,7 +343,7 @@ separate each job name with a number and one of the following: You can use these symbols interchangeably. -For example, these three jobs will be in a group named `build ruby`: +In the example below, these three jobs are in a group named `build ruby`: ```yaml build ruby 1/3: @@ -365,7 +366,7 @@ In the pipeline, the result is a group named `build ruby` with three jobs:  -The jobs will be ordered by comparing the numbers from left to right. You +The jobs are be ordered by comparing the numbers from left to right. You usually want the first number to be the index and the second number to be the total. [This regular expression](https://gitlab.com/gitlab-org/gitlab/blob/2f3dc314f42dbd79813e6251792853bc231e69dd/app/models/commit_status.rb#L99) @@ -383,7 +384,7 @@ the pipeline view, *not* the play (**{play}**) button. This is useful when you want to alter the execution of a job that uses [custom environment variables](../variables/README.md#custom-environment-variables). -Adding a variable name (key) and value here will override the value defined in +Add a variable name (key) and value here to override the value defined in [the UI or `.gitlab-ci.yml`](../variables/README.md#custom-environment-variables), for a single run of the manual job. @@ -410,7 +411,7 @@ For example, if you start rolling out new code and: > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/14664) in GitLab 12.0. -Job logs are divided into sections that can be collapsed or expanded. Each section will display +Job logs are divided into sections that can be collapsed or expanded. Each section displays the duration. In the following example: @@ -422,8 +423,11 @@ In the following example: #### Custom collapsible sections -You can create collapsible sections in job logs by manually outputting special codes -that GitLab will use to determine what sections to collapse: +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/14664) in GitLab 12.0. + +You can create [collapsible sections in job logs](../pipelines/index.md#expand-and-collapse-job-log-sections) +by manually outputting special codes +that GitLab uses to determine what sections to collapse: - Section start marker: `section_start:UNIX_TIMESTAMP:SECTION_NAME\r\e[0K` + `TEXT_OF_SECTION_HEADER` - Section end marker: `section_end:UNIX_TIMESTAMP:SECTION_NAME\r\e[0K` diff --git a/doc/ci/pipelines/job_artifacts.md b/doc/ci/pipelines/job_artifacts.md index c4457d17dc2..be6886fe6b2 100644 --- a/doc/ci/pipelines/job_artifacts.md +++ b/doc/ci/pipelines/job_artifacts.md @@ -11,13 +11,13 @@ type: reference, howto > - Introduced in GitLab 8.2 and GitLab Runner 0.7.0. > - Starting with GitLab 8.4 and GitLab Runner 1.0, the artifacts archive format changed to `ZIP`, and it's now possible to browse its contents, with the added ability of downloading the files separately. > - In GitLab 8.17, builds were renamed to jobs. -> - The artifacts browser will be available only for new artifacts that are sent to GitLab using GitLab Runner version 1.0 and up. It won't be possible to browse old artifacts already uploaded to GitLab. +> - The artifacts browser is available only for new artifacts that are sent to GitLab using GitLab Runner version 1.0 and up. You cannot browse old artifacts already uploaded to GitLab. Job artifacts are a list of files and directories created by a job once it finishes. This feature is [enabled by default](../../administration/job_artifacts.md) in all GitLab installations. -Job artifacts created by GitLab Runner are uploaded to GitLab and are downloadable as a single archive using the GitLab UI or the [GitLab API](../../api/jobs.md#get-job-artifacts). +Job artifacts created by GitLab Runner are uploaded to GitLab and are downloadable as a single archive using the GitLab UI or the [GitLab API](../../api/job_artifacts.md#get-job-artifacts). <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> For an overview, watch the video [GitLab CI Pipeline, Artifacts, and Environments](https://www.youtube.com/watch?v=PCKDICEe10s). @@ -42,9 +42,9 @@ the latex source file `mycv.tex`. We then define the `artifacts` paths which in turn are defined with the `paths` keyword. All paths to files and directories are relative to the repository that was cloned during the build. -The artifacts will be uploaded when the job succeeds by default, but can be set to upload -when the job fails, or always, if the [`artifacts:when`](../yaml/README.md#artifactswhen) -parameter is used. These uploaded artifacts will be kept in GitLab for 1 week as defined +By default, the artifacts upload when the job succeeds. You can also set artifacts to upload +when the job fails, or always, by using [`artifacts:when`](../yaml/README.md#artifactswhen) +parameter. GitLab keeps these uploaded artifacts for 1 week, as defined by the `expire_in` definition. You can keep the artifacts from expiring via the [web interface](#browsing-artifacts). If the expiry time is not defined, it defaults to the [instance wide setting](../../user/admin_area/settings/continuous_integration.md#default-artifacts-expiration-core-only). @@ -94,13 +94,12 @@ rspec: junit: rspec.xml ``` -The collected JUnit reports will be uploaded to GitLab as an artifact and will -be automatically shown in merge requests. +The collected JUnit reports upload to GitLab as an artifact and display in merge requests. NOTE: **Note:** -In case the JUnit tool you use exports to multiple XML files, you can specify -multiple test report paths within a single job and they will be automatically -concatenated into a single file. Use a filename pattern (`junit: rspec-*.xml`), +If the JUnit tool you use exports to multiple XML files, specify +multiple test report paths within a single job to +concatenate them into a single file. Use a filename pattern (`junit: rspec-*.xml`), an array of filenames (`junit: [rspec-1.xml, rspec-2.xml, rspec-3.xml]`), or a combination thereof (`junit: [rspec.xml, test-results/TEST-*.xml]`). @@ -130,8 +129,8 @@ There are a couple of exceptions to the [original dotenv rules](https://github.c > - Requires [GitLab Runner](https://docs.gitlab.com/runner/) 11.5 and above. The `cobertura` report collects [Cobertura coverage XML files](../../user/project/merge_requests/test_coverage_visualization.md). -The collected Cobertura coverage reports will be uploaded to GitLab as an artifact -and will be automatically shown in merge requests. +The collected Cobertura coverage reports upload to GitLab as an artifact +and display in merge requests. Cobertura was originally developed for Java, but there are many third party ports for other languages like JavaScript, Python, Ruby, and so on. @@ -142,7 +141,7 @@ third party ports for other languages like JavaScript, Python, Ruby, and so on. > - Requires [GitLab Runner](https://docs.gitlab.com/runner/) 11.5 and above. The `terraform` report obtains a Terraform `tfplan.json` file. [JQ processing required to remove credentials](../../user/infrastructure/index.md#output-terraform-plan-information-into-a-merge-request). The collected Terraform -plan report will be uploaded to GitLab as an artifact and will be automatically shown +plan report uploads to GitLab as an artifact and displays in merge requests. For more information, see [Output `terraform plan` information into a merge request](../../user/infrastructure/index.md#output-terraform-plan-information-into-a-merge-request). @@ -155,8 +154,7 @@ in merge requests. For more information, see The `codequality` report collects [CodeQuality issues](../../user/project/merge_requests/code_quality.md) as artifacts. -The collected Code Quality report will be uploaded to GitLab as an artifact and will -be summarized in merge requests. +The collected Code Quality report uploads to GitLab as an artifact and is summarized in merge requests. #### `artifacts:reports:sast` **(ULTIMATE)** @@ -166,8 +164,8 @@ be summarized in merge requests. The `sast` report collects [SAST vulnerabilities](../../user/application_security/sast/index.md) as artifacts. -The collected SAST report will be uploaded to GitLab as an artifact and will be summarized -in the merge requests and pipeline view. It's also used to provide data for security +The collected SAST report uploads to GitLab as an artifact and is summarized +in merge requests and the pipeline view. It's also used to provide data for security dashboards. #### `artifacts:reports:secret_detection` **(ULTIMATE)** @@ -190,8 +188,7 @@ dashboards. The `dependency_scanning` report collects [Dependency Scanning vulnerabilities](../../user/application_security/dependency_scanning/index.md) as artifacts. -The collected Dependency Scanning report will be uploaded to GitLab as an artifact and will -be summarized in the merge requests and pipeline view. It's also used to provide data for security +The collected Dependency Scanning report uploads to GitLab as an artifact and is summarized in merge requests and the pipeline view. It's also used to provide data for security dashboards. #### `artifacts:reports:container_scanning` **(ULTIMATE)** @@ -202,8 +199,8 @@ dashboards. The `container_scanning` report collects [Container Scanning vulnerabilities](../../user/application_security/container_scanning/index.md) as artifacts. -The collected Container Scanning report will be uploaded to GitLab as an artifact and will -be summarized in the merge requests and pipeline view. It's also used to provide data for security +The collected Container Scanning report uploads to GitLab as an artifact and +is summarized in merge requests and the pipeline view. It's also used to provide data for security dashboards. #### `artifacts:reports:dast` **(ULTIMATE)** @@ -214,8 +211,7 @@ dashboards. The `dast` report collects [DAST vulnerabilities](../../user/application_security/dast/index.md) as artifacts. -The collected DAST report will be uploaded to GitLab as an artifact and will -be summarized in the merge requests and pipeline view. It's also used to provide data for security +The collected DAST report uploads to GitLab as an artifact and is summarized in merge requests and the pipeline view. It's also used to provide data for security dashboards. #### `artifacts:reports:license_management` **(ULTIMATE)** @@ -231,8 +227,7 @@ introduced in GitLab 12.8. The `license_management` report collects [Licenses](../../user/compliance/license_compliance/index.md) as artifacts. -The collected License Compliance report will be uploaded to GitLab as an artifact and will -be summarized in the merge requests and pipeline view. It's also used to provide data for security +The collected License Compliance report uploads to GitLab as an artifact and is summarized in merge requests and the pipeline view. It's also used to provide data for security dashboards. #### `artifacts:reports:license_scanning` **(ULTIMATE)** @@ -243,8 +238,7 @@ dashboards. The `license_scanning` report collects [Licenses](../../user/compliance/license_compliance/index.md) as artifacts. -The License Compliance report will be uploaded to GitLab as an artifact and will -be automatically shown in merge requests, pipeline view and provide data for security +The License Compliance report uploads to GitLab as an artifact and displays automatically in merge requests and the pipeline view, and provide data for security dashboards. #### `artifacts:reports:performance` **(PREMIUM)** @@ -255,8 +249,7 @@ dashboards. The `performance` report collects [Browser Performance Testing metrics](../../user/project/merge_requests/browser_performance_testing.md) as artifacts. -The collected Browser Performance report will be uploaded to GitLab as an artifact and will -be automatically shown in merge requests. +The collected Browser Performance report uploads to GitLab as an artifact and displays in merge requests. #### `artifacts:reports:load_performance` **(PREMIUM)** @@ -276,8 +269,7 @@ shown in merge requests automatically. The `metrics` report collects [Metrics](../metrics_reports.md) as artifacts. -The collected Metrics report will be uploaded to GitLab as an artifact and will -be automatically shown in merge requests. +The collected Metrics report uploads to GitLab as an artifact and displays in merge requests. #### `artifacts:reports:requirements` **(ULTIMATE)** @@ -286,15 +278,15 @@ be automatically shown in merge requests. The `requirements` report collects `requirements.json` files as artifacts. -The collected Requirements report will be uploaded to GitLab as an artifact and -existing [requirements](../../user/project/requirements/index.md) will be +The collected Requirements report uploads to GitLab as an artifact and +existing [requirements](../../user/project/requirements/index.md) are marked as Satisfied. ## Browsing artifacts > - From GitLab 9.2, PDFs, images, videos, and other formats can be previewed directly in the job artifacts browser without the need to download them. > - Introduced in [GitLab 10.1](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/14399), HTML files in a public project can be previewed directly in a new tab without the need to download them when [GitLab Pages](../../administration/pages/index.md) is enabled. The same applies for textual formats (currently supported extensions: `.txt`, `.json`, and `.log`). -> - Introduced in [GitLab 12.4](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/16675), artifacts in private projects can be previewed when [GitLab Pages access control](../../administration/pages/index.md#access-control) is enabled. +> - Introduced in [GitLab 12.4](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/16675), artifacts in internal and private projects can be previewed when [GitLab Pages access control](../../administration/pages/index.md#access-control) is enabled. After a job finishes, if you visit the job's specific page, there are three buttons. You can download the artifacts archive or browse its contents, whereas @@ -311,6 +303,8 @@ Below you can see what browsing looks like. In this case we have browsed inside the archive and at this point there is one directory, a couple files, and one HTML file that you can view directly online when [GitLab Pages](../../administration/pages/index.md) is enabled (opens in a new tab). +Select artifacts in internal and private projects can only be previewed when +[GitLab Pages access control](../../administration/pages/index.md#access-control) is enabled.  @@ -346,10 +340,8 @@ so you can use it for scripting purposes. NOTE: **Note:** The latest artifacts are created by jobs in the **most recent** successful pipeline -for the specific ref. If you run two types of pipelines for the same ref, the latest -artifact will be determined by timing. For example, if a branch pipeline created -by merging a merge request runs at the same time as a scheduled pipeline, the -latest artifact will be from the pipeline that completed most recently. +for the specific ref. If you run two types of pipelines for the same ref, timing determines the latest +artifact. For example, if a merge request creates a branch pipeline at the same time as a scheduled pipeline, the pipeline that completed most recently creates the latest artifact. Artifacts for other pipelines can be accessed with direct access to them. @@ -423,7 +415,7 @@ information in the UI. DANGER: **Danger:** This is a destructive action that leads to data loss. Use with caution. -You can erase a single job via the UI, which will also remove the job's +You can erase a single job via the UI, which also removes the job's artifacts and trace, if you are: - The owner of the job. @@ -437,7 +429,7 @@ To erase a job: ## Retrieve artifacts of private projects when using GitLab CI -In order to retrieve a job artifact of a different project, you might need to use a private token in order to [authenticate and download](../../api/jobs.md#get-job-artifacts) the artifacts. +In order to retrieve a job artifact of a different project, you might need to use a private token in order to [authenticate and download](../../api/job_artifacts.md#get-job-artifacts) the artifacts. <!-- ## Troubleshooting diff --git a/doc/ci/pipelines/settings.md b/doc/ci/pipelines/settings.md index 39acef14443..40093167213 100644 --- a/doc/ci/pipelines/settings.md +++ b/doc/ci/pipelines/settings.md @@ -125,7 +125,8 @@ can use <https://rubular.com> to test your regex. The regex returns the **last** match found in the output. If the pipeline succeeds, the coverage is shown in the merge request widget and -in the jobs table. +in the jobs table. If multiple jobs in the pipeline have coverage reports, they will +be averaged.  @@ -207,7 +208,7 @@ If you want all pending non-HEAD pipelines on branches to auto-cancel each time a new pipeline is created, such as after a Git push or manually from the UI, you can enable this in the project settings: -1. Go to **{settings}** **Settings > CI / CD**. +1. Go to **Settings > CI / CD**. 1. Expand **General Pipelines**. 1. Check the **Auto-cancel redundant, pending pipelines** checkbox. 1. Click **Save changes**. @@ -226,7 +227,7 @@ newer one, which may not be what you want. To avoid this scenario: -1. Go to **{settings}** **Settings > CI / CD**. +1. Go to **Settings > CI / CD**. 1. Expand **General pipelines**. 1. Check the **Skip outdated deployment jobs** checkbox. 1. Click **Save changes**. diff --git a/doc/ci/quick_start/README.md b/doc/ci/quick_start/README.md index 25469570603..050df243af4 100644 --- a/doc/ci/quick_start/README.md +++ b/doc/ci/quick_start/README.md @@ -7,24 +7,6 @@ type: reference # Getting started with GitLab CI/CD -NOTE: **Note:** -Starting from version 8.0, GitLab [Continuous Integration](https://about.gitlab.com/stages-devops-lifecycle/continuous-integration/) (CI) -is fully integrated into GitLab itself and is [enabled](../enable_or_disable_ci.md) by default on all -projects. - -NOTE: **Note:** -Please keep in mind that only project Maintainers and Admin users have -the permissions to access a project's settings. - -NOTE: **Note:** -Coming over to GitLab from Jenkins? Check out our [reference](../jenkins/index.md) -for converting your pre-existing pipelines over to our format. - -NOTE: **Note:** -There are a few different [basic pipeline architectures](../pipelines/pipeline_architectures.md) -that you can consider for use in your project. You may want to familiarize -yourself with these prior to getting started. - GitLab offers a [continuous integration](https://about.gitlab.com/stages-devops-lifecycle/continuous-integration/) service. For each commit or push to trigger your CI [pipeline](../pipelines/index.md), you must: @@ -49,7 +31,11 @@ something. It's also common to use pipelines to automatically deploy tested code to staging and production environments. ---- +If you're already familiar with general CI/CD concepts, you can review which +[pipeline architectures](../pipelines/pipeline_architectures.md) can be used +in your projects. If you're coming over to GitLab from Jenkins, you can check out +our [reference](../migration/jenkins.md) for converting your pre-existing pipelines +over to our format. This guide assumes that you have: @@ -82,18 +68,22 @@ blog about it](https://about.gitlab.com/blog/2015/05/06/why-were-replacing-gitla ### Creating a simple `.gitlab-ci.yml` file ->**Note:** -`.gitlab-ci.yml` is a [YAML](https://en.wikipedia.org/wiki/YAML) file -so you have to pay extra attention to indentation. Always use spaces, not tabs. +NOTE: **Note:** +A GitLab team member has made an [unofficial visual pipeline editor](https://unofficial.gitlab.tools/visual-pipelines/). +There is a [plan to make it an official part of GitLab](https://gitlab.com/groups/gitlab-org/-/epics/4069) +in the future, but it's available for anyone who wants to try it at the above link. You need to create a file named `.gitlab-ci.yml` in the root directory of your -repository. Below is an example for a Ruby on Rails project. +repository. This is a [YAML](https://en.wikipedia.org/wiki/YAML) file +so you have to pay extra attention to indentation. Always use spaces, not tabs. + +Below is an example for a Ruby on Rails project: ```yaml image: "ruby:2.5" before_script: - - apt-get update -qq && apt-get install -y -qq sqlite3 libsqlite3-dev nodejs + - sudo apt-get update -qq && sudo apt-get install -y -qq sqlite3 libsqlite3-dev nodejs - ruby -v - which ruby - gem install bundler --no-document @@ -124,9 +114,7 @@ Jobs are used to create jobs, which are then picked by What is important is that each job is run independently from each other. If you want to check whether the `.gitlab-ci.yml` of your project is valid, there is a -Lint tool under the page `/-/ci/lint` of your project namespace. You can also find -a "CI Lint" button to go to this page under **CI/CD ➔ Pipelines** and -**Pipelines ➔ Jobs** in your project. +[CI Lint tool](../lint.md) available in every project. For more information and a complete `.gitlab-ci.yml` syntax, please read [the reference documentation on `.gitlab-ci.yml`](../yaml/README.md). diff --git a/doc/ci/runners/README.md b/doc/ci/runners/README.md index 21c99f928d8..6d248156004 100644 --- a/doc/ci/runners/README.md +++ b/doc/ci/runners/README.md @@ -7,7 +7,7 @@ type: reference # Configuring GitLab Runners <!-- This topic contains several commented-out sections that were accidentally added in 13.2.--> -<!-- The commented-out sections will be added back in 13.3.--> +<!-- The commented-out sections are added back in 13.3.--> In GitLab CI/CD, Runners run the code defined in [`.gitlab-ci.yml`](../yaml/README.md). A GitLab Runner is a lightweight, highly-scalable agent that picks up a CI job through @@ -118,7 +118,7 @@ You can also enable shared Runners for individual projects. To enable shared Runners: -1. Go to the project's **{settings}** **Settings > CI/CD** and expand the **Runners** section. +1. Go to the project's **Settings > CI/CD** and expand the **Runners** section. 1. Click **Allow shared Runners**. #### Disable shared Runners @@ -128,12 +128,12 @@ You must have Owner permissions for the project<!-- or group-->. To disable shared Runners for a project: -1. Go to the project's **{settings}** **Settings > CI/CD** and expand the **Runners** section. +1. Go to the project's **Settings > CI/CD** and expand the **Runners** section. 1. In the **Shared Runners** area, click **Disable shared Runners**. <!--To disable shared Runners for a group: -1. Go to the group's **{settings}** **Settings > CI/CD** and expand the **Runners** section. +1. Go to the group's **Settings > CI/CD** and expand the **Runners** section. 1. In the **Shared Runners** area, click **Disable shared Runners globally**. 1. Optionally, to allow shared Runners to be enabled for individual projects or subgroups, click **Allow projects/subgroups to override the global setting**. @@ -155,20 +155,20 @@ To create a group Runner: 1. [Install Runner](https://docs.gitlab.com/runner/install/). 1. Go to the group you want to make the Runner work for. -1. Go to **{settings}** **Settings > CI/CD** and expand the **Runners** section. +1. Go to **Settings > CI/CD** and expand the **Runners** section. 1. Note the URL and token. 1. [Register the Runner](https://docs.gitlab.com/runner/register/). -<!-- #### View and manage group Runners +#### View and manage group Runners -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/37366/) in GitLab 13.3. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/37366/) in GitLab 13.2. You can view and manage all Runners for a group, its subgroups, and projects. You can do this for your self-managed GitLab instance or for GitLab.com. You must have [Owner permissions](../../user/permissions.md#group-members-permissions) for the group. 1. Go to the group where you want to view the Runners. -1. Go to **{settings}** **Settings > CI/CD** and expand the **Runners** section. +1. Go to **Settings > CI/CD** and expand the **Runners** section. 1. The following fields are displayed. | Attribute | Description | @@ -183,7 +183,7 @@ You must have [Owner permissions](../../user/permissions.md#group-members-permis | Tags | Tags associated with the Runner | | Last contact | Timestamp indicating when the GitLab instance last contacted the Runner | -From this page, you can edit, pause, and remove Runners from the group, its subgroups, and projects. --> +From this page, you can edit, pause, and remove Runners from the group, its subgroups, and projects. #### Pause or remove a group Runner @@ -191,11 +191,11 @@ You can pause or remove a group Runner for your self-managed GitLab instance or You must have [Owner permissions](../../user/permissions.md#group-members-permissions) for the group. 1. Go to the group you want to remove or pause the Runner for. -1. Go to **{settings}** **Settings > CI/CD** and expand the **Runners** section. +1. Go to **Settings > CI/CD** and expand the **Runners** section. 1. Click **Pause** or **Remove Runner**. -<!-- - If you pause a group Runner that is used by multiple projects, the Runner pauses for all projects. --> -<!-- - From the group view, you cannot remove a Runner that is assigned to more than one project. --> -<!-- You must remove it from each project first. --> + - If you pause a group Runner that is used by multiple projects, the Runner pauses for all projects. + - From the group view, you cannot remove a Runner that is assigned to more than one project. + You must remove it from each project first. 1. On the confirmation dialog, click **OK**. ### Specific Runners @@ -223,7 +223,7 @@ You must have [Owner permissions](../../user/permissions.md#project-members-perm To create a specific Runner: 1. [Install Runner](https://docs.gitlab.com/runner/install/). -1. Go to the project's **{settings}** **Settings > CI/CD** and expand the **Runners** section. +1. Go to the project's **Settings > CI/CD** and expand the **Runners** section. 1. Note the URL and token. 1. [Register the Runner](https://docs.gitlab.com/runner/register/). @@ -237,7 +237,7 @@ enable a specific Runner to apply to additional projects. To enable or disable a specific Runner for a project: -1. Go to the project's **{settings}** **Settings > CI/CD** and expand the **Runners** section. +1. Go to the project's **Settings > CI/CD** and expand the **Runners** section. 1. Click **Enable for this project** or **Disable for this project**. #### Prevent a specific Runner from being enabled for other projects @@ -248,7 +248,7 @@ but can also be changed later. To lock or unlock a Runner: -1. Go to the project's **{settings}** **Settings > CI/CD** and expand the **Runners** section. +1. Go to the project's **Settings > CI/CD** and expand the **Runners** section. 1. Find the Runner you want to lock or unlock. Make sure it's enabled. 1. Click the pencil button. 1. Check the **Lock to current projects** option. @@ -266,7 +266,7 @@ if smaller than the [project defined timeout](../pipelines/settings.md#timeout), This feature can be used to prevent your shared Runner from being overwhelmed by a project that has jobs with a long timeout (for example, one week). -When not configured, Runners will not override the project timeout. +When not configured, Runners do not override the project timeout. How this feature works: @@ -317,7 +317,7 @@ and ignores other jobs. To protect or unprotect a Runner: -1. Go to the project's **{settings}** **Settings > CI/CD** and expand the **Runners** section. +1. Go to the project's **Settings > CI/CD** and expand the **Runners** section. 1. Find the Runner you want to protect or unprotect. Make sure it's enabled. 1. Click the pencil button. 1. Check the **Protected** option. @@ -329,8 +329,7 @@ To protect or unprotect a Runner: Whenever a project is forked, it copies the settings of the jobs that relate to it. This means that if you have shared Runners set up for a project and -someone forks that project, the shared Runners will also serve jobs of this -project. +someone forks that project, the shared Runners serve jobs of this project. ### Attack vectors in Runners @@ -346,14 +345,14 @@ may then be used to obtain the values of secret variables or to clone project co To reset the token: -1. Go to the project's **{settings}** **Settings > CI/CD**. +1. Go to the project's **Settings > CI/CD**. 1. Expand the **General pipelines settings** section. 1. Find the **Runner token** form field and click the **Reveal value** button. 1. Delete the value and save the form. 1. After the page is refreshed, expand the **Runners settings** section and check the registration token - it should be changed. -From now on the old token is no longer valid and will not register +From now on the old token is no longer valid and does not register any new Runners to the project. If you are using any tools to provision and register new Runners, the tokens used in those tools should be updated to reflect the value of the new token. @@ -376,7 +375,7 @@ different places. To view the IP address of a shared Runner you must have admin access to the GitLab instance. To determine this: -1. Visit **{admin}** **Admin Area > Overview > Runners**. +1. Visit **Admin Area > Overview > Runners**. 1. Look for the Runner in the table and you should see a column for **IP Address**.  @@ -386,7 +385,7 @@ the GitLab instance. To determine this: To can find the IP address of a Runner for a specific project, you must have Owner [permissions](../../user/permissions.md#project-members-permissions) for the project. -1. Go to the project's **{settings}** **Settings > CI/CD** and expand the **Runners** section. +1. Go to the project's **Settings > CI/CD** and expand the **Runners** section. 1. On the details page you should see a row for **IP Address**.  @@ -409,7 +408,7 @@ To change this, you must have Owner [permissions](../../user/permissions.md#proj To make a Runner pick untagged jobs: -1. Go to the project's **{settings}** **Settings > CI/CD** and expand the **Runners** section. +1. Go to the project's **Settings > CI/CD** and expand the **Runners** section. 1. Find the Runner you want to pick untagged jobs and make sure it's enabled. 1. Click the pencil button. 1. Check the **Run untagged jobs** option. diff --git a/doc/ci/services/redis.md b/doc/ci/services/redis.md index 193f78f240b..77668d9104b 100644 --- a/doc/ci/services/redis.md +++ b/doc/ci/services/redis.md @@ -34,7 +34,7 @@ And that's it. Redis will now be available to be used within your testing framework. You can also use any other Docker image available on [Docker Hub](https://hub.docker.com/_/redis). -For example, to use Redis 2.8 the service becomes `redis:2.8`. +For example, to use Redis 6.0 the service becomes `redis:6.0`. ## Use Redis with the Shell executor diff --git a/doc/ci/ssh_keys/README.md b/doc/ci/ssh_keys/README.md index 4ad4758b281..b1847ffbc60 100644 --- a/doc/ci/ssh_keys/README.md +++ b/doc/ci/ssh_keys/README.md @@ -66,7 +66,7 @@ to access it. This is where an SSH key pair comes in handy. ## Install ssh-agent if not already installed, it is required by Docker. ## (change apt-get to yum if you use an RPM-based image) ## - - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' + - 'command -v ssh-agent >/dev/null || ( apt-get update -y && apt-get install openssh-client -y )' ## ## Run ssh-agent (inside the build environment) diff --git a/doc/ci/triggers/README.md b/doc/ci/triggers/README.md index 47f11a6228c..2b006b8779b 100644 --- a/doc/ci/triggers/README.md +++ b/doc/ci/triggers/README.md @@ -7,12 +7,6 @@ type: tutorial # Triggering pipelines through the API -> **Notes**: -> -> - [Introduced](https://about.gitlab.com/releases/2015/08/22/gitlab-7-14-released/) in GitLab 7.14. -> - GitLab 8.12 has a completely redesigned job permissions system. Read all -> about the [new model and its implications](../../user/project/new_ci_build_permissions_model.md#pipeline-triggers). - Triggers can be used to force a pipeline rerun of a specific `ref` (branch or tag) with an API call. @@ -97,7 +91,7 @@ This allows you to use that for multi-project pipelines and download artifacts from any project to which you have access as this follows the same principles with the [permission model](../../user/permissions.md#job-permissions). -Read more about the [jobs API](../../api/jobs.md#download-the-artifacts-archive). +Read more about the [jobs API](../../api/job_artifacts.md#download-the-artifacts-archive). ## Adding a new trigger @@ -120,11 +114,6 @@ The action is irreversible. ## Triggering a pipeline -> **Notes**: -> -> - Valid refs are only the branches and tags. If you pass a commit SHA as a ref, -> it will not trigger a job. - To trigger a job you need to send a `POST` request to GitLab's API endpoint: ```plaintext @@ -132,8 +121,8 @@ POST /projects/:id/trigger/pipeline ``` The required parameters are the [trigger's `token`](#authentication-tokens) -and the Git `ref` on which the trigger will be performed. Valid refs are the -branch and the tag. The `:id` of a project can be found by +and the Git `ref` on which the trigger will be performed. Valid refs are +branches or tags. The `:id` of a project can be found by [querying the API](../../api/projects.md) or by visiting the **CI/CD** settings page which provides self-explanatory examples. @@ -142,16 +131,12 @@ UI under the **Jobs** page and the jobs are marked as triggered 'by API'.  ---- - You can see which trigger caused the rebuild by visiting the single job page. A part of the trigger's token is exposed in the UI as you can see from the image below.  ---- - By using cURL you can trigger a pipeline rerun with minimal effort, for example: ```shell @@ -191,14 +176,6 @@ This means that whenever a new tag is pushed on project A, the job will run and ## Triggering a pipeline from a webhook -> **Notes**: -> -> - Introduced in GitLab 8.14. -> - `ref` should be passed as part of the URL in order to take precedence over -> `ref` from the webhook body that designates the branch ref that fired the -> trigger in the source repository. -> - `ref` should be URL-encoded if it contains slashes. - To trigger a job from a webhook of another project you need to add the following webhook URL for Push and Tag events (change the project ID, ref and token): @@ -206,6 +183,10 @@ webhook URL for Push and Tag events (change the project ID, ref and token): https://gitlab.example.com/api/v4/projects/9/ref/master/trigger/pipeline?token=TOKEN ``` +`ref` should be passed as part of the URL in order to take precedence over +`ref` from the webhook body that designates the branch ref that fired the +trigger in the source repository. `ref` should be URL-encoded if it contains slashes. + ## Making use of trigger variables You can pass any number of arbitrary variables in the trigger API call and they @@ -271,7 +252,7 @@ of all types of variables. ## Using cron to trigger nightly pipelines ->**Note:** +NOTE: **Note:** The following behavior can also be achieved through GitLab's UI with [pipeline schedules](../pipelines/schedules.md). diff --git a/doc/ci/troubleshooting.md b/doc/ci/troubleshooting.md index a019f8232a9..96d94a6c165 100644 --- a/doc/ci/troubleshooting.md +++ b/doc/ci/troubleshooting.md @@ -7,6 +7,24 @@ type: reference # Troubleshooting CI/CD +## Pipeline warnings + +Pipeline configuration warnings are shown when you: + +- [View pipeline details](pipelines/index.md#view-pipelines). +- [Validate configuration with the CI Lint tool](yaml/README.md#validate-the-gitlab-ciyml). +- [Manually run a pipeline](pipelines/index.md#run-a-pipeline-manually). + +### "Job may allow multiple pipelines to run for a single action" + +When you use [`rules`](yaml/README.md#rules) with a `when:` clause without +an `if:` clause, multiple pipelines may run. Usually +this occurs when you push a commit to a branch that has an open merge request associated with it. + +To [prevent duplicate pipelines](yaml/README.md#prevent-duplicate-pipelines), use +[`workflow: rules`](yaml/README.md#workflowrules) or rewrite your rules +to control which pipelines can run. + ## Merge request pipeline widget The merge request pipeline widget shows information about the pipeline status in a Merge Request. It's displayed above the [merge request ability to merge widget](#merge-request-ability-to-merge-widget). @@ -15,16 +33,15 @@ There are several messages that can be displayed depending on the status of the ### "Checking pipeline status" -This message is shown when the merge request has no pipeline associated with the latest commit yet and [Pipelines must succeed](../user/project/merge_requests/merge_when_pipeline_succeeds.md#only-allow-merge-requests-to-be-merged-if-the-pipeline-succeeds) is turned on. This might be because: +This message is shown when the merge request has no pipeline associated with the latest commit yet. This might be because: - GitLab hasn't finished creating the pipeline yet. - You are using an external CI service and GitLab hasn't heard back from the service yet. - You are not using CI/CD pipelines in your project. +- The latest pipeline was deleted (this is a [known issue](https://gitlab.com/gitlab-org/gitlab/-/issues/214323)). After the pipeline is created, the message will update with the pipeline status. -Note: Currently if you delete the latest pipeline of a Merge Request, this message will be shown instead of a meaningful error message. This is a known issue and should be resolved soon. - ## Merge request ability to merge widget The merge request status widget shows the **Merge** button and whether or not a merge request is ready to merge. If the merge request can't be merged, the reason for this is displayed. diff --git a/doc/ci/variables/README.md b/doc/ci/variables/README.md index 4f9a1d8dd27..61bc466692e 100644 --- a/doc/ci/variables/README.md +++ b/doc/ci/variables/README.md @@ -74,8 +74,8 @@ pages: - echo $CI_PAGES_DOMAIN ``` -For GitLab.com users, the output will be `gitlab.io`. For your -private instance, the output will be whatever your sysadmin has +For GitLab.com users, the output is `gitlab.io`. For your +private instance, the output is whatever your sysadmin has defined. ## Custom environment variables @@ -120,8 +120,8 @@ From within the UI, you can add or update custom environment variables: - **Value**: No limitations. - **Type**: `File` or `Variable`. - **Environment scope**: `All`, or specific environments. - - **Protect variable** (Optional): If selected, the variable will only be available in pipelines that run on protected branches or tags. - - **Mask variable** (Optional): If selected, the variable's **Value** will be masked in job logs. The variable fails to save if the value does not meet the [masking requirements](#masked-variable-requirements). + - **Protect variable** (Optional): If selected, the variable is only available in pipelines that run on protected branches or tags. + - **Mask variable** (Optional): If selected, the variable's **Value** is masked in job logs. The variable fails to save if the value does not meet the [masking requirements](#masked-variable-requirements). After a variable is created, you can update any of the details by clicking the **{pencil}** **Edit** button. @@ -137,7 +137,7 @@ test_variable: - cat $GREETING # the temp file itself contains the variable value ``` -The output will be: +The output is:  @@ -187,7 +187,7 @@ kubectl config set-cluster e2e --server="$KUBE_URL" --certificate-authority="$KU > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/13784) in GitLab 11.10 -Variables can be masked so that the value of the variable will be hidden in job logs. +Variables can be masked so that the value of the variable is hidden in job logs. To mask a variable: @@ -308,7 +308,7 @@ job_name: You can also list all environment variables with the `export` command in Bash or `dir env:` command in PowerShell. -Be aware that this will also expose the values of all the variables +Be aware that this also exposes the values of all the variables you set, in the job log: ```yaml @@ -376,8 +376,8 @@ These variables are saved in the repository, and they are meant to store non-sensitive project configuration, like `RAILS_ENV` or `DATABASE_URL`. -For example, if you set the variable below globally (not inside a job), it will -be used in all executed commands and scripts: +For example, if you set the variable below globally (not inside a job), it is +used in all executed commands and scripts: ```yaml variables: @@ -419,9 +419,9 @@ Group-level variables can be added by: 1. Navigating to your group's **Settings > CI/CD** page. 1. Inputting variable types, keys, and values in the **Variables** section. - Any variables of [subgroups](../../user/group/subgroups/index.md) will be inherited recursively. + Any variables of [subgroups](../../user/group/subgroups/index.md) are inherited recursively. -Once you set them, they will be available for all subsequent pipelines. Any group-level user defined variables can be viewed in projects by: +Once you set them, they are available for all subsequent pipelines. Any group-level user defined variables can be viewed in projects by: 1. Navigating to the project's **Settings > CI/CD** page. 1. Expanding the **Variables** section. @@ -444,11 +444,11 @@ To add an instance-level variable: 1. Navigate to your admin area's **Settings > CI/CD** and expand the **Variables** section. 1. Click the **Add variable** button, and fill in the details: - - **Key**: Must be one line, using only letters, numbers, or `_` (underscore), with no spaces. - - **Value**: 700 characters allowed. - - **Type**: `File` or `Variable`. - - **Protect variable** (Optional): If selected, the variable will only be available in pipelines that run on protected branches or tags. - - **Mask variable** (Optional): If selected, the variable's **Value** will not be shown in job logs. The variable will not be saved if the value does not meet the [masking requirements](#masked-variable-requirements). + - **Key**: Must be one line, using only letters, numbers, or `_` (underscore), with no spaces. + - **Value**: [Since GitLab 13.3](https://gitlab.com/gitlab-org/gitlab/-/issues/220028), 10,000 characters allowed. This is also bounded by the limits of the selected Runner operating system. In GitLab 13.0 to 13.2, 700 characters allowed. + - **Type**: `File` or `Variable`. + - **Protect variable** (Optional): If selected, the variable is only available in pipelines that run on protected branches or tags. + - **Mask variable** (Optional): If selected, the variable's **Value** is not shown in job logs. The variable is not saved if the value does not meet the [masking requirements](#masked-variable-requirements). After a variable is created, you can update any of the details by clicking the **{pencil}** **Edit** button. @@ -540,14 +540,14 @@ For example, if you define: - `API_TOKEN=secure` as a project variable. - `API_TOKEN=yaml` in your `.gitlab-ci.yml`. -`API_TOKEN` will take the value `secure` as the project +`API_TOKEN` takes the value `secure` as the project variables take precedence over those defined in `.gitlab-ci.yml`. ## Unsupported variables Variable names are limited by the underlying shell used to execute scripts (see [available shells](https://docs.gitlab.com/runner/shells/index.html). Each shell has its own unique set of reserved variable names. -You will also want to keep in mind the [scope of environment variables](where_variables_can_be_used.md) to ensure a variable is defined in the scope +You also want to keep in mind the [scope of environment variables](where_variables_can_be_used.md) to ensure a variable is defined in the scope in which you wish to use it. ## Where variables can be used @@ -585,8 +585,8 @@ pass CI variables to the running application by prefixing the key of the variable with `K8S_SECRET_`. These [prefixed -variables](../../topics/autodevops/customize.md#application-secret-variables) will -then be available as environment variables on the running application +variables](../../topics/autodevops/customize.md#application-secret-variables) are +then available as environment variables on the running application container. CAUTION: **Caution:** @@ -649,94 +649,132 @@ This follows the usual rules for [`only` / `except` policies](../yaml/README.md# ### Syntax of environment variable expressions -Below you can find supported syntax reference: +Below you can find supported syntax reference. + +#### Equality matching using a string + +Examples: + +- `$VARIABLE == "some value"` +- `$VARIABLE != "some value"` (introduced in GitLab 11.11) + +You can use equality operator `==` or `!=` to compare a variable content to a +string. We support both, double quotes and single quotes to define a string +value, so both `$VARIABLE == "some value"` and `$VARIABLE == 'some value'` +are supported. `"some value" == $VARIABLE` is correct too. + +#### Checking for an undefined value + +Examples: + +- `$VARIABLE == null` +- `$VARIABLE != null` (introduced in GitLab 11.11) + +It sometimes happens that you want to check whether a variable is defined +or not. To do that, you can compare a variable to `null` keyword, like +`$VARIABLE == null`. This expression evaluates to true if +variable is not defined when `==` is used, or to false if `!=` is used. + +#### Checking for an empty variable -1. Equality matching using a string +Examples: - Examples: +- `$VARIABLE == ""` +- `$VARIABLE != ""` (introduced in GitLab 11.11) - - `$VARIABLE == "some value"` - - `$VARIABLE != "some value"` (introduced in GitLab 11.11) +If you want to check whether a variable is defined, but is empty, you can +simply compare it against an empty string, like `$VAR == ''` or non-empty +string `$VARIABLE != ""`. - You can use equality operator `==` or `!=` to compare a variable content to a - string. We support both, double quotes and single quotes to define a string - value, so both `$VARIABLE == "some value"` and `$VARIABLE == 'some value'` - are supported. `"some value" == $VARIABLE` is correct too. +#### Comparing two variables -1. Checking for an undefined value +Examples: - Examples: +- `$VARIABLE_1 == $VARIABLE_2` +- `$VARIABLE_1 != $VARIABLE_2` (introduced in GitLab 11.11) - - `$VARIABLE == null` - - `$VARIABLE != null` (introduced in GitLab 11.11) +It is possible to compare two variables. This compares values +of these variables. - It sometimes happens that you want to check whether a variable is defined - or not. To do that, you can compare a variable to `null` keyword, like - `$VARIABLE == null`. This expression evaluates to true if - variable is not defined when `==` is used, or to false if `!=` is used. +#### Variable presence check -1. Checking for an empty variable +Example: `$STAGING` - Examples: +If you only want to create a job when there is some variable present, +which means that it is defined and non-empty, you can simply use +variable name as an expression, like `$STAGING`. If `$STAGING` variable +is defined, and is non empty, expression evaluates to `true`. +`$STAGING` value needs to be a string, with length higher than zero. +Variable that contains only whitespace characters is not an empty variable. - - `$VARIABLE == ""` - - `$VARIABLE != ""` (introduced in GitLab 11.11) +#### Regex pattern matching - If you want to check whether a variable is defined, but is empty, you can - simply compare it against an empty string, like `$VAR == ''` or non-empty - string `$VARIABLE != ""`. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/43601) in GitLab 11.0 -1. Comparing two variables +Examples: - Examples: +- `=~`: True if pattern is matched. Ex: `$VARIABLE =~ /^content.*/` +- `!~`: True if pattern is not matched. Ex: `$VARIABLE_1 !~ /^content.*/` ([Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/61900) in GitLab 11.11) - - `$VARIABLE_1 == $VARIABLE_2` - - `$VARIABLE_1 != $VARIABLE_2` (introduced in GitLab 11.11) +Variable pattern matching with regular expressions uses the +[RE2 regular expression syntax](https://github.com/google/re2/wiki/Syntax). +Expressions evaluate as `true` if: - It is possible to compare two variables. This is going to compare values - of these variables. +- Matches are found when using `=~`. +- Matches are *not* found when using `!~`. -1. Variable presence check +Pattern matching is case-sensitive by default. Use `i` flag modifier, like +`/pattern/i` to make a pattern case-insensitive. - Example: `$STAGING` +#### Conjunction / Disjunction - If you only want to create a job when there is some variable present, - which means that it is defined and non-empty, you can simply use - variable name as an expression, like `$STAGING`. If `$STAGING` variable - is defined, and is non empty, expression will evaluate to truth. - `$STAGING` value needs to be a string, with length higher than zero. - Variable that contains only whitespace characters is not an empty variable. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/62867) in GitLab 12.0 -1. Pattern matching (introduced in GitLab 11.0) +Examples: - Examples: +- `$VARIABLE1 =~ /^content.*/ && $VARIABLE2 == "something"` +- `$VARIABLE1 =~ /^content.*/ && $VARIABLE2 =~ /thing$/ && $VARIABLE3` +- `$VARIABLE1 =~ /^content.*/ || $VARIABLE2 =~ /thing$/ && $VARIABLE3` - - `=~`: True if pattern is matched. Ex: `$VARIABLE =~ /^content.*/` - - `!~`: True if pattern is not matched. Ex: `$VARIABLE_1 !~ /^content.*/` ([Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/61900) in GitLab 11.11) +It is possible to join multiple conditions using `&&` or `||`. Any of the otherwise +supported syntax may be used in a conjunctive or disjunctive statement. +Precedence of operators follows the +[Ruby 2.5 standard](https://ruby-doc.org/core-2.5.0/doc/syntax/precedence_rdoc.html), +so `&&` is evaluated before `||`. - Variable pattern matching with regular expressions uses the - [RE2 regular expression syntax](https://github.com/google/re2/wiki/Syntax). - Expressions evaluate as `true` if: +#### Parentheses - - Matches are found when using `=~`. - - Matches are *not* found when using `!~`. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/230938) in GitLab 13.3 - Pattern matching is case-sensitive by default. Use `i` flag modifier, like - `/pattern/i` to make a pattern case-insensitive. +It is possible to use parentheses to group conditions. Parentheses have the highest +precedence of all operators. Expressions enclosed in parentheses are evaluated first, +and the result is used for the rest of the expression. -1. Conjunction / Disjunction ([introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/27925) in GitLab 12.0) +Many nested parentheses can be used to create complex conditions, and the inner-most +expressions in parentheses are evaluated first. For an expression to be valid an equal +number of `(` and `)` need to be used. - Examples: +Examples: - - `$VARIABLE1 =~ /^content.*/ && $VARIABLE2 == "something"` - - `$VARIABLE1 =~ /^content.*/ && $VARIABLE2 =~ /thing$/ && $VARIABLE3` - - `$VARIABLE1 =~ /^content.*/ || $VARIABLE2 =~ /thing$/ && $VARIABLE3` +- `($VARIABLE1 =~ /^content.*/ || $VARIABLE2) && ($VARIABLE3 =~ /thing$/ || $VARIABLE4)` +- `($VARIABLE1 =~ /^content.*/ || $VARIABLE2 =~ /thing$/) && $VARIABLE3` +- `$CI_COMMIT_BRANCH == "my-branch" || (($VARIABLE1 == "thing" || $VARIABLE2 == "thing") && $VARIABLE3)` - It is possible to join multiple conditions using `&&` or `||`. Any of the otherwise - supported syntax may be used in a conjunctive or disjunctive statement. - Precedence of operators follows the - [Ruby 2.5 standard](https://ruby-doc.org/core-2.5.0/doc/syntax/precedence_rdoc.html), - so `&&` is evaluated before `||`. +The feature is currently deployed behind a feature flag that is **enabled by default**. +[GitLab administrators with access to the GitLab Rails console](../../administration/feature_flags.md) +can opt to disable it for your instance. + +To enable it: + +```ruby +Feature.enable(:ci_if_parenthesis_enabled) +``` + +To disable it: + +```ruby +Feature.disable(:ci_if_parenthesis_enabled) +``` ### Storing regular expressions in variables diff --git a/doc/ci/variables/predefined_variables.md b/doc/ci/variables/predefined_variables.md index 98a2e7ae22f..c79ea4b0d05 100644 --- a/doc/ci/variables/predefined_variables.md +++ b/doc/ci/variables/predefined_variables.md @@ -16,9 +16,8 @@ version of Runner required. NOTE: **Note:** Starting with GitLab 9.0, we have deprecated some variables. Read the -[9.0 Renaming](deprecated_variables.md#gitlab-90-renamed-variables) section to find out their replacements. **You are -strongly advised to use the new variables as we will remove the old ones in -future GitLab releases.** +[9.0 Renaming](deprecated_variables.md#gitlab-90-renamed-variables) section to find out their replacements. +**To avoid problems with deprecated and removed variables in future releases, you are strongly advised to use the new variables.** You can add a command to your `.gitlab-ci.yml` file to [output the values of all variables available for a job](README.md#list-all-environment-variables). @@ -49,6 +48,7 @@ Kubernetes-specific environment variables are detailed in the | `CI_CONFIG_PATH` | 9.4 | 0.5 | The path to CI configuration file. Defaults to `.gitlab-ci.yml` | | `CI_DEBUG_TRACE` | all | 1.7 | Whether [debug logging (tracing)](README.md#debug-logging) is enabled | | `CI_DEFAULT_BRANCH` | 12.4 | all | The name of the default branch for the project. | +| `CI_DEPLOY_FREEZE` | 13.2 | all | Included with the value `true` if the pipeline runs during a [deploy freeze window](../../user/project/releases/index.md#prevent-unintentional-releases-by-setting-a-deploy-freeze). | | `CI_DEPLOY_PASSWORD` | 10.8 | all | Authentication password of the [GitLab Deploy Token](../../user/project/deploy_tokens/index.md#gitlab-deploy-token), only present if the Project has one related. | | `CI_DEPLOY_USER` | 10.8 | all | Authentication username of the [GitLab Deploy Token](../../user/project/deploy_tokens/index.md#gitlab-deploy-token), only present if the Project has one related. | | `CI_DISPOSABLE_ENVIRONMENT` | all | 10.1 | Marks that the job is executed in a disposable environment (something that is created only for this job and disposed of/destroyed after the execution - all executors except `shell` and `ssh`). If the environment is disposable, it is set to true, otherwise it is not defined at all. | @@ -56,6 +56,8 @@ Kubernetes-specific environment variables are detailed in the | `CI_ENVIRONMENT_SLUG` | 8.15 | all | A simplified version of the environment name, suitable for inclusion in DNS, URLs, Kubernetes labels, etc. Only present if [`environment:name`](../yaml/README.md#environmentname) is set. | | `CI_ENVIRONMENT_URL` | 9.3 | all | The URL of the environment for this job. Only present if [`environment:url`](../yaml/README.md#environmenturl) is set. | | `CI_EXTERNAL_PULL_REQUEST_IID` | 12.3 | all | Pull Request ID from GitHub if the [pipelines are for external pull requests](../ci_cd_for_external_repos/index.md#pipelines-for-external-pull-requests). Available only if `only: [external_pull_requests]` or [`rules`](../yaml/README.md#rules) syntax is used and the pull request is open. | +| `CI_EXTERNAL_PULL_REQUEST_SOURCE_REPOSITORY` | 13.3 | all | The source repository name of the pull request if [the pipelines are for external pull requests](../ci_cd_for_external_repos/index.md#pipelines-for-external-pull-requests). Available only if `only: [external_pull_requests]` or [`rules`](../yaml/README.md#rules) syntax is used and the pull request is open. | +| `CI_EXTERNAL_PULL_REQUEST_TARGET_REPOSITORY` | 13.3 | all | The target repository name of the pull request if [the pipelines are for external pull requests](../ci_cd_for_external_repos/index.md#pipelines-for-external-pull-requests). Available only if `only: [external_pull_requests]` or [`rules`](../yaml/README.md#rules) syntax is used and the pull request is open. | | `CI_EXTERNAL_PULL_REQUEST_SOURCE_BRANCH_NAME` | 12.3 | all | The source branch name of the pull request if [the pipelines are for external pull requests](../ci_cd_for_external_repos/index.md#pipelines-for-external-pull-requests). Available only if `only: [external_pull_requests]` or [`rules`](../yaml/README.md#rules) syntax is used and the pull request is open. | | `CI_EXTERNAL_PULL_REQUEST_SOURCE_BRANCH_SHA` | 12.3 | all | The HEAD SHA of the source branch of the pull request if [the pipelines are for external pull requests](../ci_cd_for_external_repos/index.md#pipelines-for-external-pull-requests). Available only if `only: [external_pull_requests]` or [`rules`](../yaml/README.md#rules) syntax is used and the pull request is open. | | `CI_EXTERNAL_PULL_REQUEST_TARGET_BRANCH_NAME` | 12.3 | all | The target branch name of the pull request if [the pipelines are for external pull requests](../ci_cd_for_external_repos/index.md#pipelines-for-external-pull-requests). Available only if `only: [external_pull_requests]` or [`rules`](../yaml/README.md#rules) syntax is used and the pull request is open. | @@ -71,8 +73,8 @@ Kubernetes-specific environment variables are detailed in the | `CI_JOB_URL` | 11.1 | 0.5 | Job details URL | | `CI_KUBERNETES_ACTIVE` | 13.0 | all | Included with the value `true` only if the pipeline has a Kubernetes cluster available for deployments. Not included if no cluster is available. Can be used as an alternative to [`only:kubernetes`/`except:kubernetes`](../yaml/README.md#onlykubernetesexceptkubernetes) with [`rules:if`](../yaml/README.md#rulesif) | | `CI_MERGE_REQUEST_ASSIGNEES` | 11.9 | all | Comma-separated list of username(s) of assignee(s) for the merge request if [the pipelines are for merge requests](../merge_request_pipelines/index.md). Available only if `only: [merge_requests]` or [`rules`](../yaml/README.md#rules) syntax is used and the merge request is created. | -| `CI_MERGE_REQUEST_ID` | 11.6 | all | The project-level ID of the merge request. Only available if [the pipelines are for merge requests](../merge_request_pipelines/index.md) and the merge request is created. | -| `CI_MERGE_REQUEST_IID` | 11.6 | all | The instance-level IID of the merge request. Only available If [the pipelines are for merge requests](../merge_request_pipelines/index.md) and the merge request is created. | +| `CI_MERGE_REQUEST_ID` | 11.6 | all | The instance-level ID of the merge request. Only available if [the pipelines are for merge requests](../merge_request_pipelines/index.md) and the merge request is created. | +| `CI_MERGE_REQUEST_IID` | 11.6 | all | The project-level IID (internal ID) of the merge request. Only available If [the pipelines are for merge requests](../merge_request_pipelines/index.md) and the merge request is created. | | `CI_MERGE_REQUEST_LABELS` | 11.9 | all | Comma-separated label names of the merge request if [the pipelines are for merge requests](../merge_request_pipelines/index.md). Available only if `only: [merge_requests]` or [`rules`](../yaml/README.md#rules) syntax is used and the merge request is created. | | `CI_MERGE_REQUEST_MILESTONE` | 11.9 | all | The milestone title of the merge request if [the pipelines are for merge requests](../merge_request_pipelines/index.md). Available only if `only: [merge_requests]` or [`rules`](../yaml/README.md#rules) syntax is used and the merge request is created. | | `CI_MERGE_REQUEST_PROJECT_ID` | 11.6 | all | The ID of the project of the merge request if [the pipelines are for merge requests](../merge_request_pipelines/index.md). Available only if `only: [merge_requests]` or [`rules`](../yaml/README.md#rules) syntax is used and the merge request is created. | @@ -92,9 +94,9 @@ Kubernetes-specific environment variables are detailed in the | `CI_NODE_TOTAL` | 11.5 | all | Total number of instances of this job running in parallel. If the job is not parallelized, this variable is set to `1`. | | `CI_PAGES_DOMAIN` | 11.8 | all | The configured domain that hosts GitLab Pages. | | `CI_PAGES_URL` | 11.8 | all | URL to GitLab Pages-built pages. Always belongs to a subdomain of `CI_PAGES_DOMAIN`. | -| `CI_PIPELINE_ID` | 8.10 | all | The unique ID of the current pipeline that GitLab CI/CD uses internally | -| `CI_PIPELINE_IID` | 11.0 | all | The unique ID of the current pipeline scoped to project | -| `CI_PIPELINE_SOURCE` | 10.0 | all | Indicates how the pipeline was triggered. Possible options are: `push`, `web`, `schedule`, `api`, `external`, `chat`, `webide`, `merge_request_event`, `external_pull_request_event`, `parent_pipeline`, [`trigger`, or `pipeline`](../triggers/README.md#authentication-tokens) (renamed to `cross_project_pipeline` since 13.0). For pipelines created before GitLab 9.5, this will show as `unknown`. | +| `CI_PIPELINE_ID` | 8.10 | all | The instance-level ID of the current pipeline. | +| `CI_PIPELINE_IID` | 11.0 | all | The project-level IID (internal ID) of the current pipeline. | +| `CI_PIPELINE_SOURCE` | 10.0 | all | Indicates how the pipeline was triggered. Possible options are: `push`, `web`, `schedule`, `api`, `external`, `chat`, `webide`, `merge_request_event`, `external_pull_request_event`, `parent_pipeline`, [`trigger`, or `pipeline`](../triggers/README.md#authentication-tokens) (renamed to `cross_project_pipeline` since 13.0). For pipelines created before GitLab 9.5, this is displayed as `unknown`. | | `CI_PIPELINE_TRIGGERED` | all | all | The flag to indicate that job was [triggered](../triggers/README.md) | | `CI_PIPELINE_URL` | 11.1 | 0.5 | Pipeline details URL | | `CI_PROJECT_DIR` | all | all | The full path where the repository is cloned and where the job is run. If the GitLab Runner `builds_dir` parameter is set, this variable is set relative to the value of `builds_dir`. For more information, see [Advanced configuration](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-runners-section) for GitLab Runner. | @@ -108,7 +110,7 @@ Kubernetes-specific environment variables are detailed in the | `CI_PROJECT_TITLE` | 12.4 | all | The human-readable project name as displayed in the GitLab web interface. | | `CI_PROJECT_URL` | 8.10 | 0.5 | The HTTP(S) address to access project | | `CI_PROJECT_VISIBILITY` | 10.3 | all | The project visibility (internal, private, public) | -| `CI_REGISTRY` | 8.10 | 0.5 | If the Container Registry is enabled it returns the address of GitLab's Container Registry. This variable will include a `:port` value if one has been specified in the registry configuration. | +| `CI_REGISTRY` | 8.10 | 0.5 | If the Container Registry is enabled it returns the address of GitLab's Container Registry. This variable includes a `:port` value if one has been specified in the registry configuration. | | `CI_REGISTRY_IMAGE` | 8.10 | 0.5 | If the Container Registry is enabled for the project it returns the address of the registry tied to the specific project | | `CI_REGISTRY_PASSWORD` | 9.0 | all | The password to use to push containers to the GitLab Container Registry, for the current project. | | `CI_REGISTRY_USER` | 9.0 | all | The username to use to push containers to the GitLab Container Registry, for the current project. | diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md index e1d1d27efed..694754a33d1 100644 --- a/doc/ci/yaml/README.md +++ b/doc/ci/yaml/README.md @@ -34,8 +34,8 @@ We have complete examples of configuring pipelines: > from 30 days to under 8 hours with GitLab. NOTE: **Note:** -If you have a [mirrored repository where GitLab pulls from](../../user/project/repository/repository_mirroring.md#pulling-from-a-remote-repository-starter), -you may need to enable pipeline triggering in your project's +If you have a [mirrored repository that GitLab pulls from](../../user/project/repository/repository_mirroring.md#pulling-from-a-remote-repository-starter), +you may need to enable pipeline triggering. Go to your project's **Settings > Repository > Pull from a remote repository > Trigger pipelines for mirror updates**. ## Introduction @@ -63,8 +63,8 @@ jobs, where each of the jobs executes a different command. Of course a command can execute code directly (`./configure;make;make install`) or run a script (`test.sh`) in the repository. -Jobs are picked up by [Runners](../runners/README.md) and executed within the -environment of the Runner. What is important, is that each job is run +Jobs are picked up by [runners](../runners/README.md) and executed within the +environment of the runner. What is important is that each job is run independently from each other. ### Validate the `.gitlab-ci.yml` @@ -103,37 +103,34 @@ The following table lists available parameters for jobs: | Keyword | Description | |:---------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| [`script`](#script) | Shell script which is executed by Runner. | -| [`image`](#image) | Use Docker images. Also available: `image:name` and `image:entrypoint`. | -| [`services`](#services) | Use Docker services images. Also available: `services:name`, `services:alias`, `services:entrypoint`, and `services:command`. | -| [`before_script`](#before_script-and-after_script) | Override a set of commands that are executed before job. | +| [`script`](#script) | Shell script that is executed by a runner. | | [`after_script`](#before_script-and-after_script) | Override a set of commands that are executed after job. | -| [`stage`](#stage) | Defines a job stage (default: `test`). | -| [`only`](#onlyexcept-basic) | Limit when jobs are created. Also available: [`only:refs`, `only:kubernetes`, `only:variables`, and `only:changes`](#onlyexcept-advanced). | -| [`except`](#onlyexcept-basic) | Limit when jobs are not created. Also available: [`except:refs`, `except:kubernetes`, `except:variables`, and `except:changes`](#onlyexcept-advanced). | -| [`rules`](#rules) | List of conditions to evaluate and determine selected attributes of a job, and whether or not it's created. May not be used alongside `only`/`except`. | -| [`tags`](#tags) | List of tags which are used to select Runner. | -| [`allow_failure`](#allow_failure) | Allow job to fail. Failed job does not contribute to commit status. | -| [`when`](#when) | When to run job. Also available: `when:manual` and `when:delayed`. | -| [`environment`](#environment) | Name of an environment to which the job deploys. Also available: `environment:name`, `environment:url`, `environment:on_stop`, `environment:auto_stop_in` and `environment:action`. | +| [`allow_failure`](#allow_failure) | Allow job to fail. Failed job does not contribute to commit status. | +| [`artifacts`](#artifacts) | List of files and directories to attach to a job on success. Also available: `artifacts:paths`, `artifacts:exclude`, `artifacts:expose_as`, `artifacts:name`, `artifacts:untracked`, `artifacts:when`, `artifacts:expire_in`, `artifacts:reports`. | +| [`before_script`](#before_script-and-after_script) | Override a set of commands that are executed before job. | | [`cache`](#cache) | List of files that should be cached between subsequent runs. Also available: `cache:paths`, `cache:key`, `cache:untracked`, and `cache:policy`. | -| [`artifacts`](#artifacts) | List of files and directories to attach to a job on success. Also available: `artifacts:paths`, `artifacts:exclude`, `artifacts:expose_as`, `artifacts:name`, `artifacts:untracked`, `artifacts:when`, `artifacts:expire_in`, `artifacts:reports`, `artifacts:reports:codequality`, `artifacts:reports:junit`, `artifacts:reports:cobertura`, and `artifacts:reports:terraform`.<br><br>In GitLab [Enterprise Edition](https://about.gitlab.com/pricing/), these are available: `artifacts:reports:sast`, `artifacts:reports:dependency_scanning`, `artifacts:reports:container_scanning`, `artifacts:reports:dast`, `artifacts:reports:license_scanning`, `artifacts:reports:license_management` (removed in GitLab 13.0), `artifacts:reports:performance`, `artifacts:reports:load_performance`, and `artifacts:reports:metrics`. | -| [`dependencies`](#dependencies) | Restrict which artifacts are passed to a specific job by providing a list of jobs to fetch artifacts from. | | [`coverage`](#coverage) | Code coverage settings for a given job. | +| [`dependencies`](#dependencies) | Restrict which artifacts are passed to a specific job by providing a list of jobs to fetch artifacts from. | +| [`environment`](#environment) | Name of an environment to which the job deploys. Also available: `environment:name`, `environment:url`, `environment:on_stop`, `environment:auto_stop_in` and `environment:action`. | +| [`except`](#onlyexcept-basic) | Limit when jobs are not created. Also available: [`except:refs`, `except:kubernetes`, `except:variables`, and `except:changes`](#onlyexcept-advanced). | +| [`extends`](#extends) | Configuration entries that this job inherits from. | +| [`image`](#image) | Use Docker images. Also available: `image:name` and `image:entrypoint`. | +| [`include`](#include) | Allows this job to include external YAML files. Also available: `include:local`, `include:file`, `include:template`, and `include:remote`. | +| [`interruptible`](#interruptible) | Defines if a job can be canceled when made redundant by a newer run. | +| [`only`](#onlyexcept-basic) | Limit when jobs are created. Also available: [`only:refs`, `only:kubernetes`, `only:variables`, and `only:changes`](#onlyexcept-advanced). | +| [`pages`](#pages) | Upload the result of a job to use with GitLab Pages. | +| [`parallel`](#parallel) | How many instances of a job should be run in parallel. | +| [`release`](#release) | Instructs the runner to generate a [Release](../../user/project/releases/index.md) object. | +| [`resource_group`](#resource_group) | Limit job concurrency. | | [`retry`](#retry) | When and how many times a job can be auto-retried in case of a failure. | +| [`rules`](#rules) | List of conditions to evaluate and determine selected attributes of a job, and whether or not it's created. May not be used alongside `only`/`except`. | +| [`services`](#services) | Use Docker services images. Also available: `services:name`, `services:alias`, `services:entrypoint`, and `services:command`. | +| [`stage`](#stage) | Defines a job stage (default: `test`). | +| [`tags`](#tags) | List of tags that are used to select a runner. | | [`timeout`](#timeout) | Define a custom job-level timeout that takes precedence over the project-wide setting. | -| [`parallel`](#parallel) | How many instances of a job should be run in parallel. | | [`trigger`](#trigger) | Defines a downstream pipeline trigger. | -| [`include`](#include) | Allows this job to include external YAML files. Also available: `include:local`, `include:file`, `include:template`, and `include:remote`. | -| [`extends`](#extends) | Configuration entries that this job is going to inherit from. | -| [`pages`](#pages) | Upload the result of a job to use with GitLab Pages. | | [`variables`](#variables) | Define job variables on a job level. | -| [`interruptible`](#interruptible) | Defines if a job can be canceled when made redundant by a newer run. | -| [`resource_group`](#resource_group) | Limit job concurrency. | -| [`release`](#release) | Instructs the Runner to generate a [Release](../../user/project/releases/index.md) object. | - -NOTE: **Note:** -Parameters `types` and `type` are [deprecated](#deprecated-parameters). +| [`when`](#when) | When to run job. Also available: `when:manual` and `when:delayed`. | ## Global parameters @@ -293,31 +290,66 @@ There are also two edge cases worth mentioning: > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/29654) in GitLab 12.5 -The top-level `workflow:` key applies to the entirety of a pipeline, and will -determine whether or not a pipeline is created. It currently accepts a single +The top-level `workflow:` key applies to the entirety of a pipeline, and +determines whether or not a pipeline is created. It accepts a single `rules:` key that operates similarly to [`rules:` defined within jobs](#rules), enabling dynamic configuration of the pipeline. If you are new to GitLab CI/CD and `workflow: rules`, you may find the [`workflow:rules` templates](#workflowrules-templates) useful. -To define your own `workflow: rules`, the configuration options currently available are: +To define your own `workflow: rules`, the available configuration options are: - [`if`](#rulesif): Define a rule. - [`when`](#when): May be set to `always` or `never` only. If not provided, the default value is `always`. -The list of `if` rules is evaluated until a single one is matched. If none -match, the last `when` will be used: +If a pipeline attempts to run but matches no rule, it's dropped and doesn't run. + +Use the example rules below exactly as written to allow pipelines that match the rule +to run. Add `when: never` to prevent pipelines that match the rule from running. See +the [common `if` clauses for `rules`](#common-if-clauses-for-rules) for more examples. + +| Example rules | Details | +|------------------------------------------------------|-----------------------------------------------------------| +| `if: '$CI_PIPELINE_SOURCE == "merge_request_event"'` | Control when merge request pipelines run. | +| `if: '$CI_PIPELINE_SOURCE == "push"'` | Control when both branch pipelines and tag pipelines run. | +| `if: $CI_COMMIT_TAG` | Control when tag pipelines run. | +| `if: $CI_COMMIT_BRANCH` | Control when branch pipelines run. | + +For example, in the following configuration, pipelines run for all `push` events (changes to +branches and new tags). Only push events with `-wip` in the commit message are excluded. Scheduled +pipelines and merge request pipelines don't run, as there's no rule allowing them. ```yaml workflow: rules: - if: $CI_COMMIT_REF_NAME =~ /-wip$/ when: never - - if: $CI_COMMIT_TAG + - if: '$CI_PIPELINE_SOURCE == "push"' +``` + +This example has strict rules, and no other pipelines can run. + +Alternatively, you can have loose rules by using only `when: never` rules, followed +by a final `when: always` rule. This allows all types of pipelines, except for any +that match the `when: never` rules: + +```yaml +workflow: + rules: + - if: '$CI_PIPELINE_SOURCE == "schedule"' + when: never + - if: '$CI_PIPELINE_SOURCE == "push"' when: never - when: always ``` +This example never allows pipelines for schedules or `push` (branches and tags) pipelines, +but does allow pipelines in **all** other cases, *including* merge request pipelines. + +As with `rules` defined in jobs, be careful not to use a configuration that allows +merge request pipelines and branch pipelines to run at the same time, or you could +have [duplicate pipelines](#prevent-duplicate-pipelines). + #### `workflow:rules` templates > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/217732) in GitLab 13.0. @@ -367,7 +399,7 @@ It's also possible to have template files stored in a central repository and pro configuration files. This helps avoid duplicated configuration, for example, global default variables for all projects. `include` requires the external YAML file to have the extensions `.yml` or `.yaml`, -otherwise the external file won't be included. +otherwise the external file is not included. `include` supports the following inclusion methods: @@ -376,14 +408,14 @@ otherwise the external file won't be included. | [`local`](#includelocal) | Include a file from the local project repository. | | [`file`](#includefile) | Include a file from a different project repository. | | [`remote`](#includeremote) | Include a file from a remote URL. Must be publicly accessible. | -| [`template`](#includetemplate) | Include templates which are provided by GitLab. | +| [`template`](#includetemplate) | Include templates that are provided by GitLab. | The `include` methods do not support [variable expansion](../variables/where_variables_can_be_used.md#variables-usage). NOTE: **Note:** `.gitlab-ci.yml` configuration included by all methods is evaluated at pipeline creation. The configuration is a snapshot in time and persisted in the database. Any changes to -referenced `.gitlab-ci.yml` configuration won't be reflected in GitLab until the next pipeline is created. +referenced `.gitlab-ci.yml` configuration is not reflected in GitLab until the next pipeline is created. The files defined by `include` are: @@ -393,7 +425,7 @@ The files defined by `include` are: TIP: **Tip:** Use merging to customize and override included CI/CD configurations with local -definitions. Local definitions in `.gitlab-ci.yml` will override included definitions. +definitions. Local definitions in `.gitlab-ci.yml` override included definitions. NOTE: **Note:** Using [YAML anchors](#anchors) across different YAML files sourced by `include` is not @@ -405,11 +437,11 @@ of using YAML anchors, you can use the [`extends` keyword](#extends). `include:local` includes a file from the same repository as `.gitlab-ci.yml`. It's referenced using full paths relative to the root directory (`/`). -You can only use files that are currently tracked by Git on the same branch +You can only use files that are tracked by Git on the same branch your configuration file is on. In other words, when using a `include:local`, make sure that both `.gitlab-ci.yml` and the local file are on the same branch. -All [nested includes](#nested-includes) will be executed in the scope of the same project, +All [nested includes](#nested-includes) are executed in the scope of the same project, so it's possible to use local, project, remote, or template includes. NOTE: **Note:** @@ -423,7 +455,7 @@ include: ``` TIP: **Tip:** -Local includes can be used as a replacement for symbolic links which are not followed. +Local includes can be used as a replacement for symbolic links that are not followed. This can be defined as a short local include: @@ -462,7 +494,7 @@ include: file: '/templates/.gitlab-ci-template.yml' ``` -All [nested includes](#nested-includes) will be executed in the scope of the target project, +All [nested includes](#nested-includes) are executed in the scope of the target project, so it's possible to use local (relative to target project), project, remote or template includes. @@ -478,7 +510,7 @@ include: - remote: 'https://gitlab.com/awesome-project/raw/master/.gitlab-ci-template.yml' ``` -All [nested includes](#nested-includes) will be executed without context as public user, so only another remote +All [nested includes](#nested-includes) are executed without context as public user, so only another remote or public project, or template, is allowed. #### `include:template` @@ -504,7 +536,7 @@ include: - template: Auto-DevOps.gitlab-ci.yml ``` -All [nested includes](#nested-includes) will be executed only with the permission of the user, +All [nested includes](#nested-includes) are executed only with the permission of the user, so it's possible to use project, remote or template includes. #### Nested includes @@ -584,7 +616,7 @@ For more information, see [Available settings for `services`](../docker/using_do ### `script` `script` is the only required keyword that a job needs. It's a shell script -which is executed by the Runner. For example: +that is executed by the runner. For example: ```yaml job: @@ -603,14 +635,14 @@ job: ``` NOTE: **Note:** -Sometimes, `script` commands will need to be wrapped in single or double quotes. -For example, commands that contain a colon (`:`) need to be wrapped in quotes so +Sometimes, `script` commands must be wrapped in single or double quotes. +For example, commands that contain a colon (`:`) must be wrapped in quotes so that the YAML parser knows to interpret the whole thing as a string rather than a "key: value" pair. Be careful when using special characters: `:`, `{`, `}`, `[`, `]`, `,`, `&`, `*`, `#`, `?`, `|`, `-`, `<`, `>`, `=`, `!`, `%`, `@`, `` ` ``. -If any of the script commands return an exit code different from zero, the job -will fail and further commands won't be executed. This behavior can be avoided by +If any of the script commands return an exit code other than zero, the job +fails and further commands are not executed. You can avoid this behavior by storing the exit code in a variable: ```yaml @@ -631,7 +663,7 @@ This must be an array. Scripts specified in `before_script` are concatenated with any scripts specified in the main [`script`](#script), and executed together in a single shell. -`after_script` is used to define the command that will be run after each +`after_script` is used to define the command that runs after each job, including failed ones. This must be an array. Scripts specified in `after_script` are executed in a new shell, separate from any @@ -640,12 +672,12 @@ Scripts specified in `after_script` are executed in a new shell, separate from a - Have a current working directory set back to the default. - Have no access to changes done by scripts defined in `before_script` or `script`, including: - Command aliases and variables exported in `script` scripts. - - Changes outside of the working tree (depending on the Runner executor), like + - Changes outside of the working tree (depending on the runner executor), like software installed by a `before_script` or `script` script. - Have a separate timeout, which is hard coded to 5 minutes. See [related issue](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/2716) for details. - Don't affect the job's exit code. If the `script` section succeeds and the - `after_script` times out or fails, the job will exit with code `0` (`Job Succeeded`). + `after_script` times out or fails, the job exits with code `0` (`Job Succeeded`). It's possible to overwrite a globally defined `before_script` or `after_script` if you set it per-job: @@ -704,14 +736,14 @@ job: - Write-Host "This text is not colored" ``` -#### Multiline commands +#### Multi-line commands You can split long commands into multi-line commands to improve readability -using [`|` (literal) and `>` (folded) YAML multiline block scalar indicators](https://yaml-multiline.info/). +using [`|` (literal) and `>` (folded) YAML multi-line block scalar indicators](https://yaml-multiline.info/). CAUTION: **Warning:** If multiple commands are combined into one command string, only the last command's -failure or success will be reported, +failure or success is reported, [incorrectly ignoring failures from earlier commands due to a bug](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/25394). If the success of the job depends on the success or failure of these commands, you can run the commands as separate `script:` items, or add `exit 1` commands @@ -774,7 +806,7 @@ First command line is split over two lines. Second command line. ``` -When the `>` or `|` block scalar indicators are omitted, GitLab will form the command +When you omit the `>` or `|` block scalar indicators, GitLab forms the command by concatenating non-empty lines, so make sure the lines can run when concatenated. Shell [here documents](https://en.wikipedia.org/wiki/Here_document) work with the @@ -799,9 +831,13 @@ $ tr a-z A-Z << END_TEXT # collapsed multi-line command FOUR FIVE SIX ``` +#### Custom collapsible sections + +See [custom collapsible sections](../pipelines/index.md#custom-collapsible-sections). + ### `stage` -`stage` is defined per-job and relies on [`stages`](#stages) which is defined +`stage` is defined per-job and relies on [`stages`](#stages), which is defined globally. It allows to group jobs into different stages, and jobs of the same `stage` are executed in parallel (subject to [certain conditions](#using-your-own-runners)). For example: @@ -836,16 +872,16 @@ job 5: script: make something useful at the end of pipeline ``` -#### Using your own Runners +#### Using your own runners -When using your own Runners, GitLab Runner runs only one job at a time by default (see the -`concurrent` flag in [Runner global settings](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-global-section) -for more information). +When you use your own runners, GitLab Runner runs only one job at a time by default. See the +`concurrent` flag in [runner global settings](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-global-section) +for more information. -Jobs will run on your own Runners in parallel only if: +Jobs run on your own runners in parallel only if: -- Run on different Runners. -- The Runner's `concurrent` setting has been changed. +- Run on different runners. +- The runner's `concurrent` setting has been changed. #### `.pre` and `.post` @@ -890,14 +926,14 @@ For example, the following are equivalent configuration: ``` NOTE: **Note:** -A pipeline won't be created if it only contains jobs in `.pre` or `.post` stages. +A pipeline is not created if all jobs are in `.pre` or `.post` stages. ### `extends` > Introduced in GitLab 11.3. -`extends` defines entry names that a job that uses `extends` is going to -inherit from. +`extends` defines entry names that a job that uses `extends` +inherits from. It's an alternative to using [YAML anchors](#anchors) and is a little more flexible and readable: @@ -919,12 +955,12 @@ rspec: ``` In the example above, the `rspec` job inherits from the `.tests` template job. -GitLab will perform a reverse deep merge based on the keys. GitLab will: +GitLab performs a reverse deep merge based on the keys. GitLab: -- Merge the `rspec` contents into `.tests` recursively. -- Not merge the values of the keys. +- Merges the `rspec` contents into `.tests` recursively. +- Doesn't merge the values of the keys. -This results in the following `rspec` job: +The result is this `rspec` job: ```yaml rspec: @@ -945,8 +981,8 @@ If you do want to include the `rake test`, see [`before_script` and `after_scrip `.tests` in this example is a [hidden job](#hide-jobs), but it's possible to inherit from regular jobs as well. -`extends` supports multi-level inheritance, however it's not recommended to -use more than three levels. The maximum nesting level that is supported is 10. +`extends` supports multi-level inheritance. You should avoid using more than 3 levels, +but you can use as many as ten. The following example has two levels of inheritance: ```yaml @@ -980,7 +1016,7 @@ In GitLab 12.0 and later, it's also possible to use multiple parents for `extends` is able to merge hashes but not arrays. The algorithm used for merge is "closest scope wins", so -keys from the last member will always override anything defined on other +keys from the last member always override anything defined on other levels. For example: ```yaml @@ -1062,7 +1098,7 @@ useTemplate: extends: .template ``` -This will run a job called `useTemplate` that runs `echo Hello!` as defined in +This example runs a job called `useTemplate` that runs `echo Hello!` as defined in the `.template` job, and uses the `alpine` Docker image as defined in the local job. ### `rules` @@ -1077,8 +1113,8 @@ If included, the job also has [certain attributes](#rules-attributes) added to it. CAUTION: **Caution:** -`rules` can't be used in combination with [`only/except`](#onlyexcept-basic) because it is a replacement for -that functionality. If you attempt to do this, the linter returns a +`rules` replaces [`only/except`](#onlyexcept-basic) and can't be used in conjunction with it. +If you attempt to use both keywords in the same job, the linter returns a `key may not be used with rules` error. #### Rules attributes @@ -1182,19 +1218,22 @@ job: - In **all other cases**, the job is added to the pipeline, with `when: on_success`. CAUTION: **Caution:** -If you use `when: on_success`, `always`, or `delayed` as the final rule, two +If you use a `when:` clause as the final rule (not including `when: never`), two simultaneous pipelines may start. Both push pipelines and merge request pipelines can be triggered by the same event (a push to the source branch for an open merge request). -See the [important differences between `rules` and `only`/`except`](#differences-between-rules-and-onlyexcept) +See how to [prevent duplicate pipelines](#prevent-duplicate-pipelines) for more details. -#### Differences between `rules` and `only`/`except` +#### Prevent duplicate pipelines -Jobs defined with `only/except` do not trigger merge request pipelines by default. -You must explicitly add `only: merge_requests`. +Jobs defined with `rules` can trigger multiple pipelines with the same action. You +don't have to explicitly configure rules for each type of pipeline to trigger them +accidentally. Rules that are too loose (allowing too many types of pipelines) could +cause a second pipeline to run unexpectedly. -Jobs defined with `rules` can trigger all types of pipelines. -You do not have to explicitly configure each type. +Some configurations that have the potential to cause duplicate pipelines cause a +[pipeline warning](../troubleshooting.md#pipeline-warnings) to be displayed. +[Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/219431) in GitLab 13.3. For example: @@ -1210,19 +1249,79 @@ job: This job does not run when `$CUSTOM_VARIABLE` is false, but it *does* run in **all** other pipelines, including **both** push (branch) and merge request pipelines. With this configuration, every push to an open merge request's source branch -causes duplicated pipelines. Explicitly allowing both push and merge request pipelines -in the same job could have the same effect. +causes duplicated pipelines. + +There are multiple ways to avoid this: + +- Use [`workflow: rules`](#workflowrules) to specify which types of pipelines + can run. To eliminate duplicate pipelines, allow only merge request pipelines + or push (branch) pipelines. + +- Rewrite the rules to run the job only in very specific cases, + and avoid using a final `when:` rule: -We recommend using [`workflow: rules`](#workflowrules) to limit which types of pipelines -are permitted. Allowing only merge request pipelines, or only branch pipelines, -eliminates duplicated pipelines. Alternatively, you can rewrite the rules to be -stricter, or avoid using a final `when` (`always`, `on_success` or `delayed`). + ```yaml + job: + script: "echo This does NOT create double pipelines!" + rules: + - if: '$CUSTOM_VARIABLE == "true" && $CI_PIPELINE_SOURCE == "merge_request_event"' + ``` -Also, we don't recommend mixing `only/except` jobs with `rules` jobs in the same pipeline. -It may not cause YAML errors, but debugging the exact execution behavior can be complex -due to the different default behaviors of `only/except` and `rules`. +You can prevent duplicate pipelines by changing the job rules to avoid either push (branch) +pipelines or merge request pipelines. However, if you use a `- when: always` rule without +`workflow: rules`, GitLab still displays a [pipeline warning](../troubleshooting.md#pipeline-warnings). -##### `rules:if` +For example, the following does not trigger double pipelines, but is not recommended +without `workflow: rules`: + +```yaml +job: + script: "echo This does NOT create double pipelines!" + rules: + - if: '$CI_PIPELINE_SOURCE == "push"' + when: never + - when: always +``` + +Do not include both push and merge request pipelines in the same job: + +```yaml +job: + script: "echo This creates double pipelines!" + rules: + - if: '$CI_PIPELINE_SOURCE == "push"' + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' +``` + +Also, do not mix `only/except` jobs with `rules` jobs in the same pipeline. +It may not cause YAML errors, but the different default behaviors of `only/except` +and `rules` can cause issues that are difficult to troubleshoot: + +```yaml +job-with-no-rules: + script: "echo This job runs in branch pipelines." + +job-with-rules: + script: "echo This job runs in merge request pipelines." + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' +``` + +For every change pushed to the branch, duplicate pipelines run. One +branch pipeline runs a single job (`job-with-no-rules`), and one merge request pipeline +runs the other job (`job-with-rules`). Jobs with no rules default +to [`except: merge_requests`](#onlyexcept-basic), so `job-with-no-rules` +runs in all cases except merge requests. + +It is not possible to define rules based on whether or not a branch has an open +merge request associated with it. You can't configure a job to be included in: + +- Only branch pipelines when the branch doesn't have a merge request associated with it. +- Only merge request pipelines when the branch has a merge request associated with it. + +See the [related issue](https://gitlab.com/gitlab-org/gitlab/-/issues/201845) for more details. + +#### `rules:if` `rules:if` clauses determine whether or not jobs are added to a pipeline by evaluating a simple `if` statement. If the `if` statement is true, the job is either included @@ -1233,7 +1332,8 @@ or excluded from a pipeline. In plain English, `if` rules can be interpreted as `rules:if` differs slightly from `only:variables` by accepting only a single expression string per rule, rather than an array of them. Any set of expressions to be -evaluated can be conjoined into a single expression by using `&&` or `||`, and use +evaluated can be [conjoined into a single expression](../variables/README.md#conjunction--disjunction) +by using `&&` or `||`, and use the [variable matching syntax](../variables/README.md#syntax-of-environment-variable-expressions). `if:` clauses are evaluated based on the values of [predefined environment variables](../variables/predefined_variables.md) @@ -1264,23 +1364,25 @@ Some details regarding the logic that determines the `when` for the job: - You can define `when` once per rule, or once at the job-level, which applies to all rules. You can't mix `when` at the job-level with `when` in rules. +##### Common `if` clauses for `rules` + For behavior similar to the [`only`/`except` keywords](#onlyexcept-basic), you can -check the value of the `$CI_PIPELINE_SOURCE` variable. - -| Value | Description | -|-------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `push` | For pipelines triggered by a `git push` event, including for branches and tags. | -| `web` | For pipelines created by using **Run pipeline** button in the GitLab UI, from the project's **CI/CD > Pipelines** section. | -| `trigger` | For pipelines created by using a trigger token. | -| `schedule` | For [scheduled pipelines](../pipelines/schedules.md). | -| `api` | For pipelines triggered by the [pipelines API](../../api/pipelines.md#create-a-new-pipeline). | -| `external` | When using CI services other than GitLab. | -| `pipelines` | For multi-project pipelines created by [using the API with `CI_JOB_TOKEN`](../triggers/README.md#when-used-with-multi-project-pipelines). | -| `chat` | For pipelines created by using a [GitLab ChatOps](../chatops/README.md) command. | -| `webide` | For pipelines created by using the [WebIDE](../../user/project/web_ide/index.md). | +check the value of the `$CI_PIPELINE_SOURCE` variable: + +| Value | Description | +|-------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `api` | For pipelines triggered by the [pipelines API](../../api/pipelines.md#create-a-new-pipeline). | +| `chat` | For pipelines created by using a [GitLab ChatOps](../chatops/README.md) command. | +| `external` | When using CI services other than GitLab. | +| `external_pull_request_event` | When an external pull request on GitHub is created or updated. See [Pipelines for external pull requests](../ci_cd_for_external_repos/index.md#pipelines-for-external-pull-requests). | | `merge_request_event` | For pipelines created when a merge request is created or updated. Required to enable [merge request pipelines](../merge_request_pipelines/index.md), [merged results pipelines](../merge_request_pipelines/pipelines_for_merged_results/index.md), and [merge trains](../merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md). | -| `external_pull_request_event` | When an external pull request on GitHub is created or updated. See [Pipelines for external pull requests](../ci_cd_for_external_repos/index.md#pipelines-for-external-pull-requests). | -| `parent_pipeline` | For pipelines triggered by a [parent/child pipeline](../parent_child_pipelines.md) with `rules`, use this in the child pipeline configuration so that it can be triggered by the parent pipeline. | +| `parent_pipeline` | For pipelines triggered by a [parent/child pipeline](../parent_child_pipelines.md) with `rules`, use this in the child pipeline configuration so that it can be triggered by the parent pipeline. | +| `pipeline` | For [multi-project pipelines](../multi_project_pipelines.md) created by [using the API with `CI_JOB_TOKEN`](../multi_project_pipelines.md#triggering-multi-project-pipelines-through-api), or the [`trigger`](#trigger) keyword. | +| `push` | For pipelines triggered by a `git push` event, including for branches and tags. | +| `schedule` | For [scheduled pipelines](../pipelines/schedules.md). | +| `trigger` | For pipelines created by using a [trigger token](../triggers/README.md#trigger-token). | +| `web` | For pipelines created by using **Run pipeline** button in the GitLab UI, from the project's **CI/CD > Pipelines** section. | +| `webide` | For pipelines created by using the [WebIDE](../../user/project/web_ide/index.md). | For example: @@ -1325,14 +1427,14 @@ Other commonly used variables for `if` clauses: - `if: '$CUSTOM_VARIABLE == "value1"'`: If the custom variable `CUSTOM_VARIABLE` is exactly `value1`. -##### `rules:changes` +#### `rules:changes` To determine if jobs should be added to a pipeline, `rules: changes` clauses check the files changed by Git push events. `rules: changes` works exactly the same way as [`only: changes` and `except: changes`](#onlychangesexceptchanges), accepting an array of paths. Similarly, it always returns true if there is no -Git push event. It should only be used for branch pipelines or merge request pipelines. +Git push event, for example, when a new tag is created. It should only be used for branch pipelines or merge request pipelines. For example: @@ -1357,11 +1459,14 @@ In this example: to continue running even if the job is not triggered (`allow_failure: true`). - If `Dockerfile` has not changed, do not add job to any pipeline (same as `when: never`). -##### `rules:exists` +To implement a rule similar to [`except: changes`](#onlychangesexceptchanges), +use `when: never`. + +#### `rules:exists` > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/24021) in GitLab 12.4. -`exists` accepts an array of paths and will match if any of these paths exist +`exists` accepts an array of paths and matches if any of these paths exist as files in the repository. For example: @@ -1389,9 +1494,9 @@ job: NOTE: **Note:** For performance reasons, using `exists` with patterns is limited to 10000 -checks. After the 10000th check, rules with patterned globs will always match. +checks. After the 10000th check, rules with patterned globs always match. -##### `rules:allow_failure` +#### `rules:allow_failure` > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/30235) in GitLab 12.8. @@ -1412,18 +1517,18 @@ job: allow_failure: true ``` -In this example, if the first rule matches, then the job will have `when: manual` and `allow_failure: true`. +In this example, if the first rule matches, then the job has `when: manual` and `allow_failure: true`. #### Complex rule clauses -To conjoin `if`, `changes`, and `exists` clauses with an AND, use them in the +To conjoin `if`, `changes`, and `exists` clauses with an `AND`, use them in the same rule. In the following example: -- We run the job manually if `Dockerfile` or any file in `docker/scripts/` - has changed AND `$VAR == "string value"`. -- Otherwise, the job won't be included in the pipeline. +- If the dockerfile or any file in `/docker/scripts` has changed, and var=blah, + then the job runs manually +- Otherwise, the job isn't included in the pipeline. ```yaml docker build: @@ -1453,9 +1558,9 @@ the most out of your pipelines. `only` and `except` are two parameters that set a job policy to limit when jobs are created: -1. `only` defines the names of branches and tags for which the job will run. -1. `except` defines the names of branches and tags for which the job will - **not** run. +1. `only` defines the names of branches and tags the job runs for. +1. `except` defines the names of branches and tags the job does + **not** run for. There are a few rules that apply to the usage of job policy: @@ -1467,20 +1572,20 @@ There are a few rules that apply to the usage of job policy: In addition, `only` and `except` allow the use of special keywords: -| **Value** | **Description** | -|--------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `branches` | When the Git reference for a pipeline is a branch. | -| `tags` | When the Git reference for a pipeline is a tag. | -| `api` | For pipelines triggered by the [pipelines API](../../api/pipelines.md#create-a-new-pipeline). | -| `external` | When using CI services other than GitLab. | -| `pipelines` | For multi-project pipelines created by using the API with `CI_JOB_TOKEN`. | -| `pushes` | For pipelines triggered by a `git push` event, including for branches and tags. | -| `schedules` | For [scheduled pipelines](../pipelines/schedules.md). | -| `triggers` | For pipelines created by using a trigger token. | -| `web` | For pipelines created by using **Run pipeline** button in the GitLab UI, from the project's **CI/CD > Pipelines** section. | -| `merge_requests` | For pipelines created when a merge request is created or updated. Enables [merge request pipelines](../merge_request_pipelines/index.md), [merged results pipelines](../merge_request_pipelines/pipelines_for_merged_results/index.md), and [merge trains](../merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md). | -| `external_pull_requests` | When an external pull request on GitHub is created or updated (See [Pipelines for external pull requests](../ci_cd_for_external_repos/index.md#pipelines-for-external-pull-requests)). | -| `chat` | For pipelines created by using a [GitLab ChatOps](../chatops/README.md) command. | +| **Value** | **Description** | +|--------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `api` | For pipelines triggered by the [pipelines API](../../api/pipelines.md#create-a-new-pipeline). | +| `branches` | When the Git reference for a pipeline is a branch. | +| `chat` | For pipelines created by using a [GitLab ChatOps](../chatops/README.md) command. | +| `external` | When using CI services other than GitLab. | +| `external_pull_requests` | When an external pull request on GitHub is created or updated (See [Pipelines for external pull requests](../ci_cd_for_external_repos/index.md#pipelines-for-external-pull-requests)). | +| `merge_requests` | For pipelines created when a merge request is created or updated. Enables [merge request pipelines](../merge_request_pipelines/index.md), [merged results pipelines](../merge_request_pipelines/pipelines_for_merged_results/index.md), and [merge trains](../merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md). | +| `pipelines` | For [multi-project pipelines](../multi_project_pipelines.md) created by [using the API with `CI_JOB_TOKEN`](../multi_project_pipelines.md#triggering-multi-project-pipelines-through-api), or the [`trigger`](#trigger) keyword. | +| `pushes` | For pipelines triggered by a `git push` event, including for branches and tags. | +| `schedules` | For [scheduled pipelines](../pipelines/schedules.md). | +| `tags` | When the Git reference for a pipeline is a tag. | +| `triggers` | For pipelines created by using a [trigger token](../triggers/README.md#trigger-token). | +| `web` | For pipelines created by using **Run pipeline** button in the GitLab UI, from the project's **CI/CD > Pipelines** section. | In the example below, `job` will run only for refs that start with `issue-`, whereas all branches will be skipped: @@ -1863,8 +1968,8 @@ properly corrected any failures from previous pipelines. Without [pipelines for merge requests](../merge_request_pipelines/index.md), pipelines run on branches or tags that don't have an explicit association with a merge request. -In this case, a previous SHA is used to calculate the diff, which equivalent to `git diff HEAD~`. -This could result in some unexpected behavior, including: +In this case, a previous SHA is used to calculate the diff, which is equivalent to `git diff HEAD~`. +This can result in some unexpected behavior, including: - When pushing a new branch or a new tag to GitLab, the policy always evaluates to true. - When pushing a new commit, the changed files are calculated using the previous commit @@ -1943,9 +2048,7 @@ This example creates four paths of execution: - The maximum number of jobs that a single job can need in the `needs:` array is limited: - For GitLab.com, the limit is ten. For more information, see our [infrastructure issue](https://gitlab.com/gitlab-com/gl-infra/infrastructure/-/issues/7541). - - For self-managed instances, the limit is: - - 10, if the `ci_dag_limit_needs` feature flag is enabled (default). - - 50, if the `ci_dag_limit_needs` feature flag is disabled. + - For self-managed instances, the limit is: 50. This limit [can be changed](#changing-the-needs-job-limit-core-only). - If `needs:` refers to a job that is marked as `parallel:`. the current job will depend on all parallel jobs created. - `needs:` is similar to `dependencies:` in that it needs to use jobs from prior stages, @@ -1954,23 +2057,20 @@ This example creates four paths of execution: - Related to the above, stages must be explicitly defined for all jobs that have the keyword `needs:` or are referred to by one. -##### Changing the `needs:` job limit +##### Changing the `needs:` job limit **(CORE ONLY)** -The maximum number of jobs that can be defined within `needs:` defaults to 10, but -can be changed to 50 via a feature flag. To change the limit to 50, -[start a Rails console session](../../administration/troubleshooting/debug.md#starting-a-rails-console-session) -and run: +The maximum number of jobs that can be defined within `needs:` defaults to 50. -```ruby -Feature::disable(:ci_dag_limit_needs) -``` - -To set it back to 10, run the opposite command: +A GitLab administrator with [access to the GitLab Rails console](../../administration/feature_flags.md) +can choose a custom limit. For example, to set the limit to 100: ```ruby -Feature::enable(:ci_dag_limit_needs) +Plan.default.actual_limits.update!(ci_needs_size_limit: 100) ``` +NOTE: **Note:** +To disable the ability to use DAG, set the limit to `0`. + #### Artifact downloads with `needs` > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/14311) in GitLab v12.6. @@ -2023,7 +2123,7 @@ rspec: `needs` can be used to download artifacts from up to five jobs in pipelines on [other refs in the same project](#artifact-downloads-between-pipelines-in-the-same-project), -or pipelines in different projects: +or pipelines in different projects, groups and namespaces: ```yaml build_job: @@ -2031,14 +2131,18 @@ build_job: script: - ls -lhR needs: - - project: group/project-name + - project: namespace/group/project-name job: build-1 ref: master artifacts: true ``` `build_job` will download the artifacts from the latest successful `build-1` job -on the `master` branch in the `group/project-name` project. +on the `master` branch in the `group/project-name` project. If the project is in the +same group or namespace, you can omit them from the `project:` key. For example, +`project: group/project-name` or `project: project-name`. + +The user running the pipeline must have at least `reporter` access to the group or project, or the group/project must have public visibility. ##### Artifact downloads between pipelines in the same project @@ -2059,18 +2163,38 @@ build_job: artifacts: true ``` +Environment variables support for `project:`, `job:`, and `ref` was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/202093) +in GitLab 13.3. This is under development, but it is ready for production use. It is deployed +behind the `ci_expand_names_for_cross_pipeline_artifacts` feature flag, which is **disabled by default**. +[GitLab administrators with access to the GitLab Rails console](../../administration/feature_flags.md) +can enable it for your instance. + +For example: + +```yaml +build_job: + stage: build + script: + - ls -lhR + needs: + - project: $CI_PROJECT_PATH + job: $DEPENDENCY_JOB_NAME + ref: $CI_COMMIT_BRANCH + artifacts: true +``` + NOTE: **Note:** Downloading artifacts from jobs that are run in [`parallel:`](#parallel) is not supported. ### `tags` -`tags` is used to select specific Runners from the list of all Runners that are +`tags` is used to select specific runners from the list of all runners that are allowed to run this project. -During the registration of a Runner, you can specify the Runner's tags, for +During the registration of a runner, you can specify the runner's tags, for example `ruby`, `postgres`, `development`. -`tags` allow you to run jobs with Runners that have the specified tags +`tags` allow you to run jobs with runners that have the specified tags assigned to them: ```yaml @@ -2080,11 +2204,11 @@ job: - postgres ``` -The specification above, will make sure that `job` is built by a Runner that +The specification above, will make sure that `job` is built by a runner that has both `ruby` AND `postgres` tags defined. Tags are also a great way to run different jobs on different platforms, for -example, given an OS X Runner with tag `osx` and Windows Runner with tag +example, given an OS X runner with tag `osx` and Windows runner with tag `windows`, the following jobs run on respective platforms: ```yaml @@ -2160,6 +2284,9 @@ failure. [manual actions](#whenmanual) below. 1. `delayed` - execute job after a certain period (added in GitLab 11.14). Read about [delayed actions](#whendelayed) below. +1. `never`: + - With [`rules`](#rules), don't execute job. + - With [`workflow:rules`](#workflowrules), don't run pipeline. For example: @@ -2323,8 +2450,8 @@ timed rollout 10%: You can stop the active timer of a delayed job by clicking the **{time-out}** (**Unschedule**) button. This job will never be executed in the future unless you execute the job manually. -You can start a delayed job immediately by clicking the **Play** button. -GitLab Runner will pick your job soon and start the job. +To start a delayed job immediately, click the **Play** button. +Soon GitLab Runner picks up and starts the job. ### `environment` @@ -2394,11 +2521,10 @@ deploy to production: > including predefined, secure variables and `.gitlab-ci.yml` [`variables`](#variables). > You however can't use variables defined under `script`. -This is an optional value that when set, it exposes buttons in various places -in GitLab which when clicked take you to the defined URL. +This optional value exposes buttons that take you to the defined URL -In the example below, if the job finishes successfully, it will create buttons -in the merge requests and in the environments/deployments pages which will point +In this example, if the job finishes successfully, it creates buttons +in the merge requests and in the environments/deployments pages that point to `https://prod.example.com`. ```yaml @@ -2427,8 +2553,13 @@ Read the `environment:action` section for an example. > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/22191) in GitLab 8.13. -The `action` keyword is to be used in conjunction with `on_stop` and is defined -in the job that is called to close the environment. +The `action` keyword can be used to specify jobs that prepare, start, or stop environments. + +| **Value** | **Description** | +|-----------|---------------------------------------------------------------------------------------------------------------------------------------------------------------| +| start | Default value. Indicates that job starts the environment. Deployment will be created after job starts. | +| prepare | Indicates that job is only preparing the environment. Does not affect deployments. [Read more about environments](../environments/index.md#prepare-an-environment) | +| stop | Indicates that job stops deployment. See the example below. | Take for instance: @@ -2556,13 +2687,13 @@ deploy as review app: The `deploy as review app` job will be marked as deployment to dynamically create the `review/$CI_COMMIT_REF_NAME` environment, where `$CI_COMMIT_REF_NAME` -is an [environment variable](../variables/README.md) set by the Runner. The +is an [environment variable](../variables/README.md) set by the runner. The `$CI_ENVIRONMENT_SLUG` variable is based on the environment name, but suitable for inclusion in URLs. In this case, if the `deploy as review app` job was run in a branch named `pow`, this environment would be accessible with an URL like `https://review-pow.example.com/`. -This of course implies that the underlying server which hosts the application +This implies that the underlying server that hosts the application is properly configured. The common use case is to create dynamic environments for branches and use them @@ -2581,7 +2712,7 @@ TIP: **Learn more:** Read how caching works and find out some good practices in the [caching dependencies documentation](../caching/index.md). -`cache` is used to specify a list of files and directories which should be +`cache` is used to specify a list of files and directories that should be cached between jobs. You can only use paths that are within the local working copy. @@ -2590,7 +2721,7 @@ globally and all jobs will use that definition. #### `cache:paths` -Use the `paths` directive to choose which files or directories will be cached. Paths +Use the `paths` directive to choose which files or directories to cache. Paths are relative to the project directory (`$CI_PROJECT_DIR`) and can't directly link outside it. Wildcards can be used that follow the [glob](https://en.wikipedia.org/wiki/Glob_(programming)) patterns and: @@ -2645,8 +2776,8 @@ or any other way that fits your workflow. This way, you can fine tune caching, allowing you to cache data between different jobs or even different branches. The `cache:key` variable can use any of the -[predefined variables](../variables/README.md), and the default key, if not -set, is just literal `default` which means everything is shared between +[predefined variables](../variables/README.md). The default key, if not +set, is just literal `default`, which means everything is shared between pipelines and jobs by default, starting from GitLab 9.0. NOTE: **Note:** @@ -2818,7 +2949,7 @@ skip the download step. > - Not all executors are [supported](https://docs.gitlab.com/runner/executors/#compatibility-chart). > - Job artifacts are only collected for successful jobs by default. -`artifacts` is used to specify a list of files and directories which should be +`artifacts` is used to specify a list of files and directories that are attached to the job when it [succeeds, fails, or always](#artifactswhen). The artifacts will be sent to GitLab after the job finishes and will @@ -2961,11 +3092,11 @@ Note the following: > Introduced in GitLab 8.6 and GitLab Runner v1.1.0. -The `name` directive allows you to define the name of the created artifacts -archive. That way, you can have a unique name for every archive which could be -useful when you'd like to download the archive from GitLab. The `artifacts:name` +Use the `name` directive to define the name of the created artifacts +archive. You can specify a unique name for every archive, which can be +useful when you want to download the archive from GitLab. The `artifacts:name` variable can make use of any of the [predefined variables](../variables/README.md). -The default name is `artifacts`, which becomes `artifacts.zip` when downloaded. +The default name is `artifacts`, which becomes `artifacts.zip` when you download it. NOTE: **Note:** If your branch-name contains forward slashes @@ -3102,8 +3233,11 @@ stored on GitLab. If the expiry time is not defined, it defaults to the [instance wide setting](../../user/admin_area/settings/continuous_integration.md#default-artifacts-expiration-core-only) (30 days by default). -You can use the **Keep** button on the job page to override expiration and -keep artifacts forever. +To override the expiration date and protect artifacts from being automatically deleted: + +- Use the **Keep** button on the job page. +- Set the value of `expire_in` to `never`. [Available](https://gitlab.com/gitlab-org/gitlab/-/issues/22761) + in GitLab 13.3 and later. After their expiry, artifacts are deleted hourly by default (via a cron job), and are not accessible anymore. @@ -3118,6 +3252,7 @@ provided. Examples of valid values: - `6 mos 1 day` - `47 yrs 6 mos and 4d` - `3 weeks and 2 days` +- `never` To expire artifacts 1 week after being uploaded: @@ -3143,20 +3278,20 @@ These are the available report types: | Parameter | Description | |--------------------------------------------------------------------------------------------------------------------------------------|-------------| -| [`artifacts:reports:junit`](../pipelines/job_artifacts.md#artifactsreportsjunit) | The `junit` report collects JUnit XML files. | -| [`artifacts:reports:dotenv`](../pipelines/job_artifacts.md#artifactsreportsdotenv) | The `dotenv` report collects a set of environment variables. | | [`artifacts:reports:cobertura`](../pipelines/job_artifacts.md#artifactsreportscobertura) | The `cobertura` report collects Cobertura coverage XML files. | -| [`artifacts:reports:terraform`](../pipelines/job_artifacts.md#artifactsreportsterraform) | The `terraform` report collects Terraform `tfplan.json` files. | | [`artifacts:reports:codequality`](../pipelines/job_artifacts.md#artifactsreportscodequality) | The `codequality` report collects CodeQuality issues. | -| [`artifacts:reports:sast`](../pipelines/job_artifacts.md#artifactsreportssast-ultimate) **(ULTIMATE)** | The `sast` report collects Static Application Security Testing vulnerabilities. | -| [`artifacts:reports:dependency_scanning`](../pipelines/job_artifacts.md#artifactsreportsdependency_scanning-ultimate) **(ULTIMATE)** | The `dependency_scanning` report collects Dependency Scanning vulnerabilities. | | [`artifacts:reports:container_scanning`](../pipelines/job_artifacts.md#artifactsreportscontainer_scanning-ultimate) **(ULTIMATE)** | The `container_scanning` report collects Container Scanning vulnerabilities. | | [`artifacts:reports:dast`](../pipelines/job_artifacts.md#artifactsreportsdast-ultimate) **(ULTIMATE)** | The `dast` report collects Dynamic Application Security Testing vulnerabilities. | +| [`artifacts:reports:dependency_scanning`](../pipelines/job_artifacts.md#artifactsreportsdependency_scanning-ultimate) **(ULTIMATE)** | The `dependency_scanning` report collects Dependency Scanning vulnerabilities. | +| [`artifacts:reports:dotenv`](../pipelines/job_artifacts.md#artifactsreportsdotenv) | The `dotenv` report collects a set of environment variables. | +| [`artifacts:reports:junit`](../pipelines/job_artifacts.md#artifactsreportsjunit) | The `junit` report collects JUnit XML files. | | [`artifacts:reports:license_management`](../pipelines/job_artifacts.md#artifactsreportslicense_management-ultimate) **(ULTIMATE)** | The `license_management` report collects Licenses (*removed from GitLab 13.0*). | | [`artifacts:reports:license_scanning`](../pipelines/job_artifacts.md#artifactsreportslicense_scanning-ultimate) **(ULTIMATE)** | The `license_scanning` report collects Licenses. | -| [`artifacts:reports:performance`](../pipelines/job_artifacts.md#artifactsreportsperformance-premium) **(PREMIUM)** | The `performance` report collects Browser Performance metrics. | | [`artifacts:reports:load_performance`](../pipelines/job_artifacts.md#artifactsreportsload_performance-premium) **(PREMIUM)** | The `load_performance` report collects load performance metrics. | | [`artifacts:reports:metrics`](../pipelines/job_artifacts.md#artifactsreportsmetrics-premium) **(PREMIUM)** | The `metrics` report collects Metrics. | +| [`artifacts:reports:performance`](../pipelines/job_artifacts.md#artifactsreportsperformance-premium) **(PREMIUM)** | The `performance` report collects Browser Performance metrics. | +| [`artifacts:reports:sast`](../pipelines/job_artifacts.md#artifactsreportssast-ultimate) **(ULTIMATE)** | The `sast` report collects Static Application Security Testing vulnerabilities. | +| [`artifacts:reports:terraform`](../pipelines/job_artifacts.md#artifactsreportsterraform) | The `terraform` report collects Terraform `tfplan.json` files. | #### `dependencies` @@ -3167,7 +3302,7 @@ are passed, but you can use the `dependencies` parameter to define a limited list of jobs (or no jobs) to fetch artifacts from. To use this feature, define `dependencies` in context of the job and pass -a list of all previous jobs from which the artifacts should be downloaded. +a list of all previous jobs the artifacts should be downloaded from. You can only define jobs from stages that are executed before the current one. An error will be shown if you define jobs from the current stage or next ones. Defining an empty array will skip downloading any artifacts for that job. @@ -3251,7 +3386,7 @@ job1: ### `retry` > - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/3442) in GitLab 9.5. -> - [Behavior expanded](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/3515) in GitLab 11.5 to control on which failures to retry. +> - [Behavior expanded](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/3515) in GitLab 11.5 to control which failures to retry on. `retry` allows you to configure how many times a job is going to be retried in case of a failure. @@ -3272,7 +3407,7 @@ test: ``` By default, a job will be retried on all failure cases. To have a better control -on which failures to retry, `retry` can be a hash with the following keys: +over which failures to retry, `retry` can be a hash with the following keys: - `max`: The maximum number of retries. - `when`: The failure cases to retry. @@ -3347,7 +3482,7 @@ test: The job-level timeout can exceed the [project-level timeout](../pipelines/settings.md#timeout) but can't -exceed the Runner-specific timeout. +exceed the runner-specific timeout. ### `parallel` @@ -3398,6 +3533,48 @@ Please be aware that semaphore_test_boosters reports usages statistics to the au You can then navigate to the **Jobs** tab of a new pipeline build and see your RSpec job split into three separate jobs. +#### Parallel `matrix` jobs + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/15356) in GitLab 13.3. + +`matrix:` allows you to configure different variables for jobs that are running in parallel. +There can be from 2 to 50 jobs. + +Every job gets the same `CI_NODE_TOTAL` [environment variable](../variables/README.md#predefined-environment-variables) value, and a unique `CI_NODE_INDEX` value. + +```yaml +deploystacks: + stage: deploy + script: + - bin/deploy + parallel: + matrix: + - PROVIDER: aws + STACK: + - monitoring + - app1 + - app2 + - PROVIDER: ovh + STACK: [monitoring, backup, app] + - PROVIDER: [gcp, vultr] + STACK: [data, processing] +``` + +This generates 10 parallel `deploystacks` jobs, each with different values for `PROVIDER` and `STACK`: + +```plaintext +deploystacks 1/10 with PROVIDER=aws and STACK=monitoring +deploystacks 2/10 with PROVIDER=aws and STACK=app1 +deploystacks 3/10 with PROVIDER=aws and STACK=app2 +deploystacks 4/10 with PROVIDER=ovh and STACK=monitoring +deploystacks 5/10 with PROVIDER=ovh and STACK=backup +deploystacks 6/10 with PROVIDER=ovh and STACK=app +deploystacks 7/10 with PROVIDER=gcp and STACK=data +deploystacks 8/10 with PROVIDER=gcp and STACK=processing +deploystacks 9/10 with PROVIDER=vultr and STACK=data +deploystacks 10/10 with PROVIDER=vultr and STACK=processing +``` + ### `trigger` > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/8997) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.8. @@ -3606,16 +3783,16 @@ Once an uninterruptible job is running, the pipeline will never be canceled, reg > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/15536) in GitLab 12.7. -Sometimes running multiples jobs or pipelines at the same time in an environment +Sometimes running multiple jobs or pipelines at the same time in an environment can lead to errors during the deployment. To avoid these errors, the `resource_group` attribute can be used to ensure that -the Runner won't run certain jobs simultaneously. +the runner doesn't run certain jobs simultaneously. When the `resource_group` key is defined for a job in `.gitlab-ci.yml`, job executions are mutually exclusive across different pipelines for the same project. If multiple jobs belonging to the same resource group are enqueued simultaneously, -only one of the jobs will be picked by the Runner, and the other jobs will wait until the +only one of the jobs is picked by the runner, and the other jobs wait until the `resource_group` is free. Here is a simple example: @@ -3645,8 +3822,7 @@ For more information, see [Deployments Safety](../environments/deployment_safety > [Introduced](https://gitlab.com/gitlab-org/gitlab/merge_requests/19298) in GitLab 13.2. -`release` indicates that the job creates a [Release](../../user/project/releases/index.md), -and optionally includes URLs for Release assets. +`release` indicates that the job creates a [Release](../../user/project/releases/index.md). These methods are supported: @@ -3779,25 +3955,46 @@ tags. These options cannot be used together, so choose one: # or can use a variable. ``` -- To create a release automatically when changes are pushed to the default branch, +- To create a release automatically when commits are pushed or merged to the default branch, using a new Git tag that is defined with variables: +NOTE: **Note:** +Environment variables set in `before_script` or `script` are not available for expanding +in the same job. Read more about +[potentially making variables available for expanding](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/6400). + ```yaml + prepare_job: + stage: prepare # This stage must run before the release stage + rules: + - if: $CI_COMMIT_TAG + when: never # Do not run this job when a tag is created manually + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Run this job when commits are pushed or merged to the default branch + script: + - echo "EXTRA_DESCRIPTION=some message" >> variables.env # Generate the EXTRA_DESCRIPTION and TAG environment variables + - echo "TAG=v$(cat VERSION)" >> variables.env # and append to the variables.env file + artifacts: + reports: + dotenv: variables.env # Use artifacts:reports:dotenv to expose the variables to other jobs + release_job: stage: release image: registry.gitlab.com/gitlab-org/release-cli:latest + needs: + - job: prepare_job + artifacts: true rules: - if: $CI_COMMIT_TAG when: never # Do not run this job when a tag is created manually - - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Run this job when the default branch changes + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Run this job when commits are pushed or merged to the default branch script: - - echo 'running release_job' + - echo 'running release_job for $TAG' release: - name: 'Release $CI_COMMIT_SHA' - description: 'Created using the release-cli $EXTRA_DESCRIPTION' # $EXTRA_DESCRIPTION and the tag_name - tag_name: 'v${MAJOR}.${MINOR}.${REVISION}' # variables must be defined elsewhere - ref: '$CI_COMMIT_SHA' # in the pipeline. - milestones: + name: 'Release $TAG' + description: 'Created using the release-cli $EXTRA_DESCRIPTION' # $EXTRA_DESCRIPTION and the $TAG + tag_name: '$TAG' # variables must be defined elsewhere + ref: '$CI_COMMIT_SHA' # in the pipeline. For example, in the + milestones: # prepare_job - 'm1' - 'm2' - 'm3' @@ -3871,13 +4068,13 @@ These variables can be later used in all executed commands and scripts. The YAML-defined variables are also set to all created service containers, thus allowing to fine tune them. -Except for the user defined variables, there are also the ones [set up by the -Runner itself](../variables/README.md#predefined-environment-variables). -One example would be `CI_COMMIT_REF_NAME` which has the value of -the branch or tag name for which project is built. Apart from the variables -you can set in `.gitlab-ci.yml`, there are also the so called -[Variables](../variables/README.md#gitlab-cicd-environment-variables) -which can be set in GitLab's UI. +Except for the user-defined variables, there are also variables [set up by the +runner itself](../variables/README.md#predefined-environment-variables). +One example would be `CI_COMMIT_REF_NAME`, which has the value of +the branch or tag name the project is built for. Apart from the variables +you can set in `.gitlab-ci.yml`, there are also environment +[variables](../variables/README.md#gitlab-cicd-environment-variables), +which can be set in the GitLab UI. [YAML anchors for variables](#yaml-anchors-for-variables) are available. @@ -3978,7 +4175,7 @@ The `GIT_CHECKOUT` variable can be used when the `GIT_STRATEGY` is set to either specified, it defaults to true. You can set them globally or per-job in the [`variables`](#variables) section. -If set to `false`, the Runner will: +If set to `false`, the runner will: - when doing `fetch` - update the repository and leave working copy on the current revision, @@ -3986,7 +4183,7 @@ If set to `false`, the Runner will: default branch. Having this setting set to `true` will mean that for both `clone` and `fetch` -strategies the Runner will checkout the working copy to a revision related +strategies the runner will checkout the working copy to a revision related to the CI pipeline: ```yaml @@ -4060,7 +4257,7 @@ The configurtion above will result in `git fetch` being called this way: git fetch origin $REFSPECS --depth 50 --prune ``` -Where `$REFSPECS` is a value provided to the Runner internally by GitLab. +Where `$REFSPECS` is a value provided to the runner internally by GitLab. ### Job stages attempts @@ -4069,12 +4266,12 @@ Where `$REFSPECS` is a value provided to the Runner internally by GitLab. You can set the number for attempts the running job will try to execute each of the following stages: -| Variable | Description | -|-----------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **GET_SOURCES_ATTEMPTS** | Number of attempts to fetch sources running a job | -| **ARTIFACT_DOWNLOAD_ATTEMPTS** | Number of attempts to download artifacts running a job | -| **RESTORE_CACHE_ATTEMPTS** | Number of attempts to restore the cache running a job | +| Variable | Description | +|-----------------------------------|--------------------------------------------------------| +| **ARTIFACT_DOWNLOAD_ATTEMPTS** | Number of attempts to download artifacts running a job | | **EXECUTOR_JOB_SECTION_ATTEMPTS** | [Since GitLab 12.10](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/4450), the number of attempts to run a section in a job after a [`No Such Container`](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/4450) error ([Docker executor](https://docs.gitlab.com/runner/executors/docker.html) only). | +| **GET_SOURCES_ATTEMPTS** | Number of attempts to fetch sources running a job | +| **RESTORE_CACHE_ATTEMPTS** | Number of attempts to restore the cache running a job | The default is one single attempt. @@ -4094,8 +4291,8 @@ You can set them globally or per-job in the [`variables`](#variables) section. NOTE: **Note:** As of GitLab 12.0, newly created projects will automatically have a [default `git depth` value of `50`](../pipelines/settings.md#git-shallow-clone). -You can specify the depth of fetching and cloning using `GIT_DEPTH`. This allows -shallow cloning of the repository which can significantly speed up cloning for +You can specify the depth of fetching and cloning using `GIT_DEPTH`. This does a +shallow clone of the repository and can significantly speed up cloning for repositories with a large number of commits or old, large binaries. The value is passed to `git fetch` and `git clone`. @@ -4103,7 +4300,7 @@ NOTE: **Note:** If you use a depth of 1 and have a queue of jobs or retry jobs, jobs may fail. -Since Git fetching and cloning is based on a ref, such as a branch name, Runners +Since Git fetching and cloning is based on a ref, such as a branch name, runners can't clone a specific commit SHA. If there are multiple jobs in the queue, or you're retrying an old job, the commit to be tested needs to be within the Git history that is cloned. Setting too small a value for `GIT_DEPTH` can make @@ -4124,18 +4321,18 @@ You can set it globally or per-job in the [`variables`](#variables) section. ### Custom build directories -> [Introduced](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/2211) in GitLab Runner 11.10 +> [Introduced](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/2211) in GitLab Runner 11.10. NOTE: **Note:** -This can only be used when `custom_build_dir` is enabled in the [Runner's +This can only be used when `custom_build_dir` is enabled in the [runner's configuration](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-runnerscustom_build_dir-section). This is the default configuration for `docker` and `kubernetes` executor. By default, GitLab Runner clones the repository in a unique subpath of the `$CI_BUILDS_DIR` directory. However, your project might require the code in a specific directory (Go projects, for example). In that case, you can specify -the `GIT_CLONE_PATH` variable to tell the Runner in which directory to clone the -repository: +the `GIT_CLONE_PATH` variable to tell the runner the directory to clone the +repository in: ```yaml variables: @@ -4156,9 +4353,9 @@ An executor using a concurrency greater than `1` might lead to failures because multiple jobs might be working on the same directory if the `builds_dir` is shared between jobs. GitLab Runner does not try to prevent this situation. It's up to the administrator -and developers to comply with the requirements of Runner configuration. +and developers to comply with the requirements of runner configuration. -To avoid this scenario, you can use a unique path within `$CI_BUILDS_DIR`, because Runner +To avoid this scenario, you can use a unique path within `$CI_BUILDS_DIR`, because runner exposes two additional variables that provide a unique `ID` of concurrency: - `$CI_CONCURRENT_ID`: Unique ID for all jobs running within the given executor. @@ -4209,7 +4406,7 @@ because `$CI_BUILDS_DIR` is not expanded. ## Special YAML features It's possible to use special YAML features like anchors (`&`), aliases (`*`) -and map merging (`<<`), which will allow you to greatly reduce the complexity +and map merging (`<<`), which allows you to greatly reduce the complexity of `.gitlab-ci.yml`. Read more about the various [YAML features](https://learnxinyminutes.com/docs/yaml/). diff --git a/doc/development/README.md b/doc/development/README.md index ab86c252948..74068db726c 100644 --- a/doc/development/README.md +++ b/doc/development/README.md @@ -5,6 +5,17 @@ description: 'Learn how to contribute to GitLab.' # Contributor and Development Docs +Learn the processes and technical information needed for contributing to GitLab. + +This content is intended for members of the GitLab Team as well as community contributors. +Content specific to the GitLab Team should instead be included in the [Handbook](https://about.gitlab.com/handbook/). + +For information on using GitLab to work on your own software projects, see the [GitLab user documentation](../user/index.md). + +For information on working with GitLab's API, see the [API documentation](../api/README.md). + +For information on how to install, configure, update, and upgrade your own GitLab instance, see the [administration documentation](../administration/index.md). + ## Get started - Set up GitLab's development environment with [GitLab Development Kit (GDK)](https://gitlab.com/gitlab-org/gitlab-development-kit/blob/master/doc/howto/README.md) @@ -45,6 +56,7 @@ Complementary reads: - [Danger bot](dangerbot.md) - [Generate a changelog entry with `bin/changelog`](changelog.md) - [Requesting access to Chatops on GitLab.com](chatops_on_gitlabcom.md#requesting-access) (for GitLab team members) +- [Patch release process for developers](https://gitlab.com/gitlab-org/release/docs/blob/master/general/patch/process.md#process-for-developers) ## UX and Frontend guides @@ -134,6 +146,10 @@ See [database guidelines](database/index.md). - [Refactoring guidelines](refactoring_guide/index.md) +## Deprecation guides + +- [Deprecation guidelines](deprecation_guidelines/index.md) + ## Documentation guides - [Writing documentation](documentation/index.md) diff --git a/doc/development/api_graphql_styleguide.md b/doc/development/api_graphql_styleguide.md index 92e6add9f17..bf2d6400f56 100644 --- a/doc/development/api_graphql_styleguide.md +++ b/doc/development/api_graphql_styleguide.md @@ -36,6 +36,19 @@ can be shared. It is also possible to add a `private_token` to the querystring, or add a `HTTP_PRIVATE_TOKEN` header. +## Global IDs + +GitLab's GraphQL API uses Global IDs (i.e: `"gid://gitlab/MyObject/123"`) +and never database primary key IDs. + +Global ID is [a standard](https://graphql.org/learn/global-object-identification/) +used for caching and fetching in client-side libraries. + +See also: + +- [Exposing Global IDs](#exposing-global-ids). +- [Mutation arguments](#object-identifier-arguments). + ## Types We use a code-first schema, and we declare what type everything is in Ruby. @@ -106,18 +119,28 @@ Further reading: ### Exposing Global IDs -When exposing an `ID` field on a type, we will by default try to -expose a global ID by calling `to_global_id` on the resource being -rendered. +In keeping with GitLab's use of [Global IDs](#global-ids), always convert +database primary key IDs into Global IDs when you expose them. + +All fields named `id` are +[converted automatically](https://gitlab.com/gitlab-org/gitlab/-/blob/b0f56e7/app/graphql/types/base_object.rb#L11-14) +into the object's Global ID. + +Fields that are not named `id` need to be manually converted. We can do this using +[`Gitlab::GlobalID.build`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/global_id.rb), +or by calling `#to_global_id` on an object that has mixed in the +`GlobalID::Identification` module. -To override this behavior, you can implement an `id` method on the -type for which you are exposing an ID. Please make sure that when -exposing a `GraphQL::ID_TYPE` using a custom method that it is -globally unique. +Using an example from +[`Types::Notes::DiscussionType`](https://gitlab.com/gitlab-org/gitlab/-/blob/3c95bd9/app/graphql/types/notes/discussion_type.rb#L24-26): -The records that are exposing a `full_path` as an `ID_TYPE` are one of -these exceptions. Since the full path is a unique identifier for a -`Project` or `Namespace`. +```ruby +field :reply_id, GraphQL::ID_TYPE + +def reply_id + ::Gitlab::GlobalId.build(object, id: object.reply_id) +end +``` ### Connection Types @@ -429,6 +452,52 @@ module Types end ``` +## JSON + +When data to be returned by GraphQL is stored as +[JSON](migration_style_guide.md#storing-json-in-database), we should continue to use +GraphQL types whenever possible. Avoid using the `GraphQL::Types::JSON` type unless +the JSON data returned is _truly_ unstructured. + +If the structure of the JSON data varies, but will be one of a set of known possible +structures, use a +[union](https://graphql-ruby.org/type_definitions/unions.html). +An example of the use of a union for this purpose is +[!30129](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/30129). + +Field names can be mapped to hash data keys using the `hash_key:` keyword if needed. + +For example, given the following simple JSON data: + +```json +{ + "title": "My chart", + "data": [ + { "x": 0, "y": 1 }, + { "x": 1, "y": 1 }, + { "x": 2, "y": 2 } + ] +} +``` + +We can use GraphQL types like this: + +```ruby +module Types + class ChartType < BaseObject + field :title, GraphQL::STRING_TYPE, null: true, description: 'Title of the chart' + field :data, [Types::ChartDatumType], null: true, description: 'Data of the chart' + end +end + +module Types + class ChartDatumType < BaseObject + field :x, GraphQL::INT_TYPE, null: true, description: 'X-axis value of the chart datum' + field :y, GraphQL::INT_TYPE, null: true, description: 'Y-axis value of the chart datum' + end +end +``` + ## Descriptions All fields and arguments @@ -608,15 +677,8 @@ the objects in question. To find objects to display in a field, we can add resolvers to `app/graphql/resolvers`. -Arguments can be defined within the resolver, those arguments will be -made available to the fields using the resolver. When exposing a model -that had an internal ID (`iid`), prefer using that in combination with -the namespace path as arguments in a resolver over a database -ID. Otherwise use a [globally unique ID](#exposing-global-ids). - -We already have a `FullPathLoader` that can be included in other -resolvers to quickly find Projects and Namespaces which will have a -lot of dependent objects. +Arguments can be defined within the resolver in the same way as in a mutation. +See the [Mutation arguments](#object-identifier-arguments) section. To limit the amount of queries performed, we can use `BatchLoader`. @@ -705,10 +767,6 @@ actions. In the same way a GET-request should not modify data, we cannot modify data in a regular GraphQL-query. We can however in a mutation. -To find objects for a mutation, arguments need to be specified. As with -[resolvers](#resolvers), prefer using internal ID or, if needed, a -global ID rather than the database ID. - ### Building Mutations Mutations live in `app/graphql/mutations` ideally grouped per @@ -763,10 +821,34 @@ If you need advice for mutation naming, canvass the Slack `#graphql` channel for ### Arguments -Arguments required by the mutation can be defined as arguments -required for a field. These will be wrapped up in an input type for -the mutation. For example, the `Mutations::MergeRequests::SetWip` -with GraphQL-name `MergeRequestSetWip` defines these arguments: +Arguments for a mutation are defined using `argument`. + +Example: + +```ruby +argument :my_arg, GraphQL::STRING_TYPE, + required: true, + description: "A description of the argument" +``` + +Each GraphQL `argument` defined will be passed to the `#resolve` method +of a mutation as keyword arguments. + +Example: + +```ruby +def resolve(my_arg:) + # Perform mutation ... +end +``` + +`graphql-ruby` will automatically wrap up arguments into an +[input type](https://graphql.org/learn/schema/#input-types). + +For example, the +[`mergeRequestSetWip` mutation](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/graphql/mutations/merge_requests/set_wip.rb) +defines these arguments (some +[through inheritance](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/graphql/mutations/merge_requests/base.rb)): ```ruby argument :project_path, GraphQL::ID_TYPE, @@ -786,12 +868,19 @@ argument :wip, DESC ``` -This would automatically generate an input type called +These arguments automatically generate an input type called `MergeRequestSetWipInput` with the 3 arguments we specified and the `clientMutationId`. -These arguments are then passed to the `resolve` method of a mutation -as keyword arguments. +### Object identifier arguments + +In keeping with GitLab's use of [Global IDs](#global-ids), mutation +arguments should use Global IDs to identify an object and never database +primary key IDs. + +Where an object has an `iid`, prefer to use the `full_path` or `group_path` +of its parent in combination with its `iid` as arguments to identify an +object rather than its `id`. ### Fields @@ -1204,3 +1293,7 @@ See the [schema reference](../api/graphql/reference/index.md) for details. This generated GraphQL documentation needs to be updated when the schema changes. For information on generating GraphQL documentation and schema files, see [updating the schema documentation](rake_tasks.md#update-graphql-documentation-and-schema-definitions). + +To help our readers, you should also add a new page to our [GraphQL API](../api/graphql/index.md) documentation. +For guidance, see the [GraphQL API](documentation/styleguide.md#graphql-api) section +of our documentation style guide. diff --git a/doc/development/api_styleguide.md b/doc/development/api_styleguide.md index 327f919d7f4..06b05f49b12 100644 --- a/doc/development/api_styleguide.md +++ b/doc/development/api_styleguide.md @@ -13,7 +13,7 @@ Always use an [Entity](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/api/ ## Documentation -API endpoints must come with [documentation](documentation/styleguide.md#api), unless it is internal or behind a feature flag. +API endpoints must come with [documentation](documentation/styleguide.md#restful-api), unless it is internal or behind a feature flag. The docs should be in the same merge request, or, if strictly necessary, in a follow-up with the same milestone as the original merge request. @@ -85,7 +85,7 @@ User.create(params) # imagine the user submitted `admin=1`... :) User.create(declared(params, include_parent_namespaces: false).to_h) ``` ->**Note:** +NOTE: **Note:** `declared(params)` return a `Hashie::Mash` object, on which you will have to call `.to_h`. @@ -173,7 +173,8 @@ guide on how you can add a new custom validator. validates the parameter value for different cases. Mainly, it checks whether a path is relative and does it contain `../../` relative traversal using `File::Separator` or not, and whether the path is absolute, for example - `/etc/passwd/`. + `/etc/passwd/`. By default, absolute paths are not allowed. However, you can optionally pass in an allowlist for allowed absolute paths in the following way: + `requires :file_path, type: String, file_path: { allowlist: ['/foo/bar/', '/home/foo/', '/app/home'] }` - `Git SHA`: diff --git a/doc/development/architecture.md b/doc/development/architecture.md index 8b28dd03017..963e1e618a1 100644 --- a/doc/development/architecture.md +++ b/doc/development/architecture.md @@ -46,69 +46,101 @@ https://docs.google.com/drawings/d/1fBzAyklyveF-i-2q-OHUIqDkYfjjxC4mq5shwKSZHLs/ ```mermaid graph TB - HTTP[HTTP/HTTPS] -- TCP 80, 443 --> NGINX[NGINX] - SSH -- TCP 22 --> GitLabShell[GitLab Shell] - SMTP[SMTP Gateway] - Geo[GitLab Geo Node] -- TCP 22, 80, 443 --> NGINX - - GitLabShell --TCP 8080 -->Unicorn["Unicorn (GitLab Rails)"] - GitLabShell --> Praefect - GitLabShell --> Redis - Unicorn --> PgBouncer[PgBouncer] - Unicorn --> Redis - Unicorn --> Praefect - Sidekiq --> Redis - Sidekiq --> PgBouncer - Sidekiq --> Praefect - GitLabWorkhorse[GitLab Workhorse] --> Unicorn - GitLabWorkhorse --> Redis - GitLabWorkhorse --> Praefect - Praefect --> Gitaly - NGINX --> GitLabWorkhorse - NGINX -- TCP 8090 --> GitLabPages[GitLab Pages] - NGINX --> Grafana[Grafana] - Grafana -- TCP 9090 --> Prometheus[Prometheus] - Prometheus -- TCP 80, 443 --> Unicorn - RedisExporter[Redis Exporter] --> Redis - Prometheus -- TCP 9121 --> RedisExporter - PostgreSQLExporter[PostgreSQL Exporter] --> PostgreSQL - PgBouncerExporter[PgBouncer Exporter] --> PgBouncer - Prometheus -- TCP 9187 --> PostgreSQLExporter - Prometheus -- TCP 9100 --> NodeExporter[Node Exporter] - Prometheus -- TCP 9168 --> GitLabExporter[GitLab Exporter] - Prometheus -- TCP 9127 --> PgBouncerExporter - GitLabExporter --> PostgreSQL - GitLabExporter --> GitLabShell - GitLabExporter --> Sidekiq - PgBouncer --> Consul - PostgreSQL --> Consul - PgBouncer --> PostgreSQL - NGINX --> Registry - Unicorn --> Registry - NGINX --> Mattermost - Mattermost --- Unicorn - Prometheus --> Alertmanager - Migrations --> PostgreSQL - Runner -- TCP 443 --> NGINX - Unicorn -- TCP 9200 --> Elasticsearch - Sidekiq -- TCP 9200 --> Elasticsearch - Sidekiq -- TCP 80, 443 --> Sentry - Unicorn -- TCP 80, 443 --> Sentry - Sidekiq -- UDP 6831 --> Jaeger - Unicorn -- UDP 6831 --> Jaeger - Gitaly -- UDP 6831 --> Jaeger - GitLabShell -- UDP 6831 --> Jaeger - GitLabWorkhorse -- UDP 6831 --> Jaeger - Alertmanager -- TCP 25 --> SMTP - Sidekiq -- TCP 25 --> SMTP - Unicorn -- TCP 25 --> SMTP - Unicorn -- TCP 369 --> LDAP - Sidekiq -- TCP 369 --> LDAP - Unicorn -- TCP 443 --> ObjectStorage["Object Storage"] - Sidekiq -- TCP 443 --> ObjectStorage - GitLabWorkhorse -- TCP 443 --> ObjectStorage - Registry -- TCP 443 --> ObjectStorage - Geo -- TCP 5432 --> PostgreSQL +HTTP[HTTP/HTTPS] -- TCP 80, 443 --> NGINX[NGINX] +SSH -- TCP 22 --> GitLabShell[GitLab Shell] +SMTP[SMTP Gateway] +Geo[GitLab Geo Node] -- TCP 22, 80, 443 --> NGINX + +GitLabShell --TCP 8080 -->Unicorn["Unicorn (GitLab Rails)"] +GitLabShell --> Praefect +Unicorn --> PgBouncer[PgBouncer] +Unicorn --> Redis +Unicorn --> Praefect +Sidekiq --> Redis +Sidekiq --> PgBouncer +Sidekiq --> Praefect +GitLabWorkhorse[GitLab Workhorse] --> Unicorn +GitLabWorkhorse --> Redis +GitLabWorkhorse --> Praefect +Praefect --> Gitaly +NGINX --> GitLabWorkhorse +NGINX -- TCP 8090 --> GitLabPages[GitLab Pages] +NGINX --> Grafana[Grafana] +Grafana -- TCP 9090 --> Prometheus[Prometheus] +Prometheus -- TCP 80, 443 --> Unicorn +RedisExporter[Redis Exporter] --> Redis +Prometheus -- TCP 9121 --> RedisExporter +PostgreSQLExporter[PostgreSQL Exporter] --> PostgreSQL +PgBouncerExporter[PgBouncer Exporter] --> PgBouncer +Prometheus -- TCP 9187 --> PostgreSQLExporter +Prometheus -- TCP 9100 --> NodeExporter[Node Exporter] +Prometheus -- TCP 9168 --> GitLabExporter[GitLab Exporter] +Prometheus -- TCP 9127 --> PgBouncerExporter +GitLabExporter --> PostgreSQL +GitLabExporter --> GitLabShell +GitLabExporter --> Sidekiq +PgBouncer --> Consul +PostgreSQL --> Consul +PgBouncer --> PostgreSQL +NGINX --> Registry +Unicorn --> Registry +NGINX --> Mattermost +Mattermost --- Unicorn +Prometheus --> Alertmanager +Migrations --> PostgreSQL +Runner -- TCP 443 --> NGINX +Unicorn -- TCP 9200 --> Elasticsearch +Sidekiq -- TCP 9200 --> Elasticsearch +Sidekiq -- TCP 80, 443 --> Sentry +Unicorn -- TCP 80, 443 --> Sentry +Sidekiq -- UDP 6831 --> Jaeger +Unicorn -- UDP 6831 --> Jaeger +Gitaly -- UDP 6831 --> Jaeger +GitLabShell -- UDP 6831 --> Jaeger +GitLabWorkhorse -- UDP 6831 --> Jaeger +Alertmanager -- TCP 25 --> SMTP +Sidekiq -- TCP 25 --> SMTP +Unicorn -- TCP 25 --> SMTP +Unicorn -- TCP 369 --> LDAP +Sidekiq -- TCP 369 --> LDAP +Unicorn -- TCP 443 --> ObjectStorage["Object Storage"] +Sidekiq -- TCP 443 --> ObjectStorage +GitLabWorkhorse -- TCP 443 --> ObjectStorage +Registry -- TCP 443 --> ObjectStorage +Geo -- TCP 5432 --> PostgreSQL + +click Alertmanager "./architecture.html#alertmanager" +click Praefect "./architecture.html#praefect" +click Geo "./architecture.html#gitlab-geo" +click NGINX "./architecture.html#nginx" +click Runner "./architecture.html#gitlab-runner" +click Registry "./architecture.html#registry" +click ObjectStorage "./architecture.html#minio" +click Mattermost "./architecture.html#mattermost" +click Gitaly "./architecture.html#gitaly" +click Jaeger "./architecture.html#jaeger" +click GitLabWorkhorse "./architecture.html#gitlab-workhorse" +click LDAP "./architecture.html#ldap-authentication" +click Unicorn "./architecture.html#unicorn" +click GitLabShell "./architecture.html#gitlab-shell" +click SSH "./architecture.html#ssh-request-22" +click Sidekiq "./architecture.html#sidekiq" +click Sentry "./architecture.html#sentry" +click GitLabExporter "./architecture.html#gitlab-exporter" +click Elasticsearch "./architecture.html#elasticsearch" +click Migrations "./architecture.html#database-migrations" +click PostgreSQL "./architecture.html#postgresql" +click Consul "./architecture.html#consul" +click PgBouncer "./architecture.html#pgbouncer" +click PgBouncerExporter "./architecture.html#pgbouncer-exporter" +click RedisExporter "./architecture.html#redis-exporter" +click Redis "./architecture.html#redis" +click Prometheus "./architecture.html#prometheus" +click Grafana "./architecture.html#grafana" +click GitLabPages "./architecture.html#gitlab-pages" +click PostgreSQLExporter "./architecture.html#postgresql-exporter" +click SMTP "./architecture.html#outbound-email" +click NodeExporter "./architecture.html#node-exporter" ``` ### Component legend @@ -215,7 +247,7 @@ GitLab can be considered to have two layers from a process perspective: - [Project page](https://github.com/hashicorp/consul/blob/master/README.md) - Configuration: - - [Omnibus](../administration/high_availability/consul.md) + - [Omnibus](../administration/consul.md) - [Charts](https://docs.gitlab.com/charts/installation/deployment.html#postgresql) - Layer: Core Service (Data) - GitLab.com: [Consul](../user/gitlab_com/index.md#consul) @@ -435,7 +467,7 @@ NGINX has an Ingress port for all HTTP requests and routes them to the appropria - [Project page](https://github.com/pgbouncer/pgbouncer/blob/master/README.md) - Configuration: - - [Omnibus](../administration/high_availability/pgbouncer.md) + - [Omnibus](../administration/postgresql/pgbouncer.md) - [Charts](https://docs.gitlab.com/charts/installation/deployment.html#postgresql) - Layer: Core Service (Data) - GitLab.com: [Database Architecture](https://about.gitlab.com/handbook/engineering/infrastructure/production/architecture/#database-architecture) @@ -554,7 +586,7 @@ An external registry can also be configured to use GitLab as an auth endpoint. Sentry fundamentally is a service that helps you monitor and fix crashes in real time. The server is in Python, but it contains a full API for sending events from any language, in any application. -For monitoring deployed apps, see the [Sentry integration docs](../user/project/operations/error_tracking.md) +For monitoring deployed apps, see the [Sentry integration docs](../operations/error_tracking.md) #### Sidekiq diff --git a/doc/development/auto_devops.md b/doc/development/auto_devops.md index 7d0c020ef96..6bdc77fff63 100644 --- a/doc/development/auto_devops.md +++ b/doc/development/auto_devops.md @@ -1,3 +1,9 @@ +--- +stage: Configure +group: Configure +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Auto DevOps development guide This document provides a development guide for contributors to diff --git a/doc/development/background_migrations.md b/doc/development/background_migrations.md index d9e06206961..4b58758b5c7 100644 --- a/doc/development/background_migrations.md +++ b/doc/development/background_migrations.md @@ -29,10 +29,10 @@ Some examples where background migrations can be useful: - Populating one column based on JSON stored in another column. - Migrating data that depends on the output of external services (e.g. an API). -> **Note:** -> If the background migration is part of an important upgrade, make sure it's announced -> in the release post. Discuss with your Project Manager if you're not sure the migration falls -> into this category. +NOTE: **Note:** +If the background migration is part of an important upgrade, make sure it's announced +in the release post. Discuss with your Project Manager if you're not sure the migration falls +into this category. ## Isolation @@ -123,7 +123,7 @@ once. ## Cleaning Up ->**Note:** +NOTE: **Note:** Cleaning up any remaining background migrations _must_ be done in either a major or minor release, you _must not_ do this in a patch release. diff --git a/doc/development/build_test_package.md b/doc/development/build_test_package.md index 858ff41b685..e99915a24d0 100644 --- a/doc/development/build_test_package.md +++ b/doc/development/build_test_package.md @@ -1,3 +1,9 @@ +--- +stage: Enablement +group: Distribution +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Building a package for testing While developing a new feature or modifying an existing one, it is helpful if an diff --git a/doc/development/changelog.md b/doc/development/changelog.md index 00a0573a8ba..e83ce40ef60 100644 --- a/doc/development/changelog.md +++ b/doc/development/changelog.md @@ -37,6 +37,8 @@ the `author` field. GitLab team members **should not**. - Any user-facing change **should** have a changelog entry. Example: "GitLab now uses system fonts for all text." - Performance improvements **should** have a changelog entry. +- Changes that need to be documented in the Telemetry [Event Dictionary](telemetry/event_dictionary.md) + also require a changelog entry. - _Any_ contribution from a community member, no matter how small, **may** have a changelog entry regardless of these guidelines if the contributor wants one. Example: "Fixed a typo on the search results page." diff --git a/doc/development/chatops_on_gitlabcom.md b/doc/development/chatops_on_gitlabcom.md index a3a4f1d7adc..0dd916c37fd 100644 --- a/doc/development/chatops_on_gitlabcom.md +++ b/doc/development/chatops_on_gitlabcom.md @@ -1,3 +1,9 @@ +--- +stage: Configure +group: Configure +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Chatops on GitLab.com ChatOps on GitLab.com allows GitLab team members to run various automation tasks on GitLab.com using Slack. diff --git a/doc/development/cicd/img/ci_architecture.png b/doc/development/cicd/img/ci_architecture.png Binary files differindex b888f2f07aa..0dd8ba57f51 100644 --- a/doc/development/cicd/img/ci_architecture.png +++ b/doc/development/cicd/img/ci_architecture.png diff --git a/doc/development/cicd/img/ci_template_selection_v13_1.png b/doc/development/cicd/img/ci_template_selection_v13_1.png Binary files differindex af9f6dd1a90..32de35f5c1f 100644 --- a/doc/development/cicd/img/ci_template_selection_v13_1.png +++ b/doc/development/cicd/img/ci_template_selection_v13_1.png diff --git a/doc/development/cicd/index.md b/doc/development/cicd/index.md index e0cca00fd69..5b598a19a6e 100644 --- a/doc/development/cicd/index.md +++ b/doc/development/cicd/index.md @@ -1,3 +1,10 @@ +--- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: index, concepts, howto +--- + # CI/CD development documentation Development guides that are specific to CI/CD are listed here. diff --git a/doc/development/cicd/templates.md b/doc/development/cicd/templates.md index 904e7adffe2..0169ca42ac6 100644 --- a/doc/development/cicd/templates.md +++ b/doc/development/cicd/templates.md @@ -1,3 +1,10 @@ +--- +stage: Release +group: Progressive Delivery +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: index, concepts, howto +--- + # Development guide for GitLab CI/CD templates This document explains how to develop [GitLab CI/CD templates](../../ci/examples/README.md). @@ -8,6 +15,7 @@ All template files reside in the `lib/gitlab/ci/templates` directory, and are ca | Sub-directroy | Content | [Selectable in UI](#make-sure-the-new-template-can-be-selected-in-ui) | |---------------|--------------------------------------------------------------|-----------------------------------------------------------------------| +| `/AWS/*` | Cloud Deployment (AWS) related jobs | No | | `/Jobs/*` | Auto DevOps related jobs | Yes | | `/Pages/*` | Static site generators for GitLab Pages (for example Jekyll) | Yes | | `/Security/*` | Security related jobs | Yes | @@ -25,9 +33,37 @@ Also, all templates must be named with the `*.gitlab-ci.yml` suffix. ### Backward compatibility A template might be dynamically included with the `include:template:` keyword. If -you make a change to an *existing* template, you must make sure that it won't break +you make a change to an *existing* template, you **must** make sure that it won't break CI/CD in existing projects. +For example, changing a job name in a template could break pipelines in an existing project. +Let's say there is a template named `Performance.gitlab-ci.yml` with the following content: + +```yaml +performance: + image: registry.gitlab.com/gitlab-org/verify-tools/performance:v0.1.0 + script: ./performance-test $TARGET_URL +``` + +and users include this template with passing an argument to the `performance` job. +This can be done by specifying the environment variable `TARGET_URL` in _their_ `.gitlab-ci.yml`: + +```yaml +include: + template: Performance.gitlab-ci.yml + +performance: + variables: + TARGET_URL: https://awesome-app.com +``` + +If the job name `performance` in the template is renamed to `browser-performance`, +user's `.gitlab-ci.yml` will immediately cause a lint error because there +are no such jobs named `performance` in the included template anymore. Therefore, +users have to fix their `.gitlab-ci.yml` that could annoy their workflow. + +Please read [versioning](#versioning) section for introducing breaking change safely. + ## Testing Each CI/CD template must be tested in order to make sure that it's safe to be published. @@ -64,3 +100,13 @@ You should write an RSpec test to make sure that pipeline jobs will be generated A template could contain malicious code. For example, a template that contains the `export` shell command in a job might accidentally expose project secret variables in a job log. If you're unsure if it's secure or not, you need to ask security experts for cross-validation. + +## Versioning + +Versioning allows you to introduce a new template without modifying the existing +one. This is useful process especially when we need to introduce a breaking change, +but don't want to affect the existing projects that depends on the current template. + +There is an [open issue](https://gitlab.com/gitlab-org/gitlab/-/issues/17716) for +introducing versioning concept in GitLab Ci Template. Please follow the issue for +checking the progress. diff --git a/doc/development/code_review.md b/doc/development/code_review.md index fd53ce79534..2159f7a9ed5 100644 --- a/doc/development/code_review.md +++ b/doc/development/code_review.md @@ -230,7 +230,7 @@ Instead these should be sent to the [Release Manager](https://about.gitlab.com/c - Ask for clarification. ("I didn't understand. Can you clarify?") - Avoid selective ownership of code. ("mine", "not mine", "yours") - Avoid using terms that could be seen as referring to personal traits. ("dumb", - "stupid"). Assume everyone is attractive, intelligent, and well-meaning. + "stupid"). Assume everyone is intelligent and well-meaning. - Be explicit. Remember people don't always understand your intentions online. - Be humble. ("I'm not sure - let's look it up.") - Don't use hyperbole. ("always", "never", "endlessly", "nothing") @@ -281,12 +281,16 @@ first time. ### Assigning a merge request for a review When you are ready to have your merge request reviewed, -you should default to assigning it to a reviewer from your group or team for the first review, -however, you can also assign it to any reviewer. The list of reviewers can be found on [Engineering projects](https://about.gitlab.com/handbook/engineering/projects/) page. +you should request an initial review by assigning it to a reviewer from your group or team. +However, you can also assign it to any reviewer. The list of reviewers can be found on [Engineering projects](https://about.gitlab.com/handbook/engineering/projects/) page. You can also use `workflow::ready for review` label. That means that your merge request is ready to be reviewed and any reviewer can pick it. It is recommended to use that label only if there isn't time pressure and make sure the merge request is assigned to a reviewer. -When your merge request was reviewed and can be passed to a maintainer, you should default to choosing a maintainer with [domain expertise](#domain-experts), and otherwise follow the Reviewer Roulette recommendation or use the label `ready for merge`. +When your merge request receives an approval from the first reviewer it can be passed to a maintainer. You should default to choosing a maintainer with [domain expertise](#domain-experts), and otherwise follow the Reviewer Roulette recommendation or use the label `ready for merge`. + +Sometimes, a maintainer may not be available for review. They could be out of the office or [at capacity](#review-response-slo). +You can and should check the maintainer’s availability in their profile. If the maintainer recommended by +the roulette is not available, choose someone else from that list. It is responsibility of the author of a merge request that the merge request is reviewed. If it stays in `ready for review` state too long it is recommended to assign it to a specific reviewer. diff --git a/doc/development/contributing/issue_workflow.md b/doc/development/contributing/issue_workflow.md index 9feaa485bd2..76175cb7b66 100644 --- a/doc/development/contributing/issue_workflow.md +++ b/doc/development/contributing/issue_workflow.md @@ -49,8 +49,8 @@ Most issues will have labels for at least one of the following: - Team: `~"Technical Writing"`, `~Delivery` - Specialization: `~frontend`, `~backend`, `~documentation` - Release Scoping: `~Deliverable`, `~Stretch`, `~"Next Patch Release"` -- Priority: `~P1`, `~P2`, `~P3`, `~P4` -- Severity: ~`S1`, `~S2`, `~S3`, `~S4` +- Priority: `~P::1`, `~P::2`, `~P::3`, `~P::4` +- Severity: ~`S::1`, `~S::2`, `~S::3`, `~S::4` All labels, their meaning and priority are defined on the [labels page](https://gitlab.com/gitlab-org/gitlab/-/labels). @@ -275,10 +275,10 @@ or ~"Stretch". Any open issue for a previous milestone should be labeled We have the following priority labels: -- ~P1 -- ~P2 -- ~P3 -- ~P4 +- ~P::1 +- ~P::2 +- ~P::3 +- ~P::4 Please refer to the issue triage [priority label](https://about.gitlab.com/handbook/engineering/quality/issue-triage/#priority) section in our handbook to see how it's used. @@ -286,10 +286,10 @@ Please refer to the issue triage [priority label](https://about.gitlab.com/handb We have the following severity labels: -- ~S1 -- ~S2 -- ~S3 -- ~S4 +- ~S::1 +- ~S::2 +- ~S::3 +- ~S::4 Please refer to the issue triage [severity label](https://about.gitlab.com/handbook/engineering/quality/issue-triage/#severity) section in our handbook to see how it's used. diff --git a/doc/development/contributing/style_guides.md b/doc/development/contributing/style_guides.md index ed254052180..f6e64c1f1e6 100644 --- a/doc/development/contributing/style_guides.md +++ b/doc/development/contributing/style_guides.md @@ -10,26 +10,23 @@ we suggest investigating to see if a plugin exists. For instance here is the ## Pre-commit static analysis -You're strongly advised to install -[Overcommit](https://github.com/sds/overcommit) to automatically check for +You should install [`overcommit`](https://github.com/sds/overcommit) to automatically check for static analysis offenses before committing locally. -In your GitLab source directory run: +After installing `overcommit`, run the following in your GitLab source directory: ```shell make -C tooling/overcommit ``` -Then before a commit is created, Overcommit will automatically check for -RuboCop (and other checks) offenses on every modified file. +Then before a commit is created, `overcommit` automatically checks for RuboCop (and other checks) +offenses on every modified file. -This saves you time as you don't have to wait for the same errors to be detected -by the CI. +This saves you time as you don't have to wait for the same errors to be detected by CI/CD. -Overcommit relies on a pre-commit hook to prevent commits that violate its ruleset. -If you wish to override this behavior, it can be done by passing the ENV variable -`OVERCOMMIT_DISABLE`; i.e. `OVERCOMMIT_DISABLE=1 git rebase master` to rebase while -disabling the Git hook. +`overcommit` relies on a pre-commit hook to prevent commits that violate its ruleset. To override +this behavior, pass the `OVERCOMMIT_DISABLE` environment variable. For example, +`OVERCOMMIT_DISABLE=1 git rebase master` to rebase while disabling the Git hook. ## Ruby, Rails, RSpec diff --git a/doc/development/dangerbot.md b/doc/development/dangerbot.md index 93434479846..6fda394c10d 100644 --- a/doc/development/dangerbot.md +++ b/doc/development/dangerbot.md @@ -136,7 +136,7 @@ at GitLab so far: - Their availability: - No "OOO"/"PTO"/"Parental Leave" in their GitLab or Slack status. - No `:red_circle:`/`:palm_tree:`/`:beach:`/`:beach_umbrella:`/`:beach_with_umbrella:` emojis in GitLab or Slack status. - - [Experimental] Their timezone: people for which the local hour is between + - (Experimental) Their timezone: people for which the local hour is between 6 AM and 2 PM are eligible to be picked. This is to ensure they have a good chance to get to perform a review during their current work day. The experimentation is tracked in [this issue](https://gitlab.com/gitlab-org/quality/team-tasks/-/issues/563) diff --git a/doc/development/database_debugging.md b/doc/development/database_debugging.md index 6da0455f392..25b62e0d693 100644 --- a/doc/development/database_debugging.md +++ b/doc/development/database_debugging.md @@ -11,7 +11,7 @@ Available `RAILS_ENV`: - `development` (this is your main GDK db). - `test` (used for tests like RSpec). -## Nuke everything and start over +## Delete everything and start over If you just want to delete everything and start over with an empty DB (approximately 1 minute): diff --git a/doc/development/database_review.md b/doc/development/database_review.md index 967df411db5..f56ffdbad21 100644 --- a/doc/development/database_review.md +++ b/doc/development/database_review.md @@ -78,7 +78,8 @@ the following preparations into account. #### Preparation when adding migrations -- Ensure `db/structure.sql` is updated as [documented](migration_style_guide.md#schema-changes). +- Ensure `db/structure.sql` is updated as [documented](migration_style_guide.md#schema-changes), and additionally ensure that the relevant version files under +`db/schema_migrations` were added or removed. - Make migrations reversible by using the `change` method or include a `down` method when using `up`. - Include either a rollback procedure or describe how to rollback changes. - Add the output of both migrating and rolling back for all migrations into the MR description. @@ -149,6 +150,7 @@ test its execution using `CREATE INDEX CONCURRENTLY` in the `#database-lab` Slac - Ensure it was added in a post-migration. - Maintainer: After the merge request is merged, notify Release Managers about it on `#f_upcoming_release` Slack channel. - Check consistency with `db/structure.sql` and that migrations are [reversible](migration_style_guide.md#reversibility) + - Check that the relevant version files under `db/schema_migrations` were added or removed. - Check queries timing (If any): Queries executed in a migration need to fit comfortably within `15s` - preferably much less than that - on GitLab.com. - For column removals, make sure the column has been [ignored in a previous release](what_requires_downtime.md#dropping-columns) diff --git a/doc/development/deleting_migrations.md b/doc/development/deleting_migrations.md index 8aa16710d55..b8f23019ac2 100644 --- a/doc/development/deleting_migrations.md +++ b/doc/development/deleting_migrations.md @@ -23,7 +23,7 @@ Migrations can be disabled if: In order to disable a migration, the following steps apply to all types of migrations: 1. Turn the migration into a no-op by removing the code inside `#up`, `#down` - or `#perform` methods, and adding `#no-op` comment instead. + or `#perform` methods, and adding `# no-op` comment instead. 1. Add a comment explaining why the code is gone. Disabling migrations requires explicit approval of Database Maintainer. diff --git a/doc/development/deprecation_guidelines/index.md b/doc/development/deprecation_guidelines/index.md new file mode 100644 index 00000000000..1ee22644bbc --- /dev/null +++ b/doc/development/deprecation_guidelines/index.md @@ -0,0 +1,23 @@ +# Deprecation guidelines + +This page includes information about how and when to remove or make breaking +changes to GitLab features. + +## Terminology + +It's important to understand the difference between **deprecation** and +**removal**: + +**Deprecation** is the process of flagging/marking/announcing that a feature +will be removed in a future version of GitLab. + +**Removal** is the process of actually removing a feature that was previously +deprecated. + +## When can a feature be deprecated? + +A feature can be deprecated at any time, provided there is a viable alternative. + +## When can a feature be removed/changed? + +See our [Release and Maintenance policy](../../policy/maintenance.md). diff --git a/doc/development/diffs.md b/doc/development/diffs.md index e065e0acc6f..eb070cbf4d7 100644 --- a/doc/development/diffs.md +++ b/doc/development/diffs.md @@ -93,7 +93,8 @@ Gitlab::Git::DiffCollection.collection_limits[:max_bytes] = Gitlab::Git::DiffCol No more files will be rendered at all if 5 megabytes have already been rendered. -*Note:* All collection limit parameters are currently sent and applied on Gitaly. That is, once the limit is surpassed, +NOTE: **Note:** +All collection limit parameters are currently sent and applied on Gitaly. That is, once the limit is surpassed, Gitaly will only return the safe amount of data to be persisted on `merge_request_diff_files`. ### Individual diff file limits @@ -107,7 +108,8 @@ That is, it's equivalent to 10kb if the maximum allowed value is 100kb. The diff will still be persisted and expandable if the patch size doesn't surpass `ApplicationSettings#diff_max_patch_bytes`. -*Note:* Although this nomenclature (Collapsing) is also used on Gitaly, this limit is only used on GitLab (hardcoded - not sent to Gitaly). +NOTE: **Note:** +Although this nomenclature (Collapsing) is also used on Gitaly, this limit is only used on GitLab (hardcoded - not sent to Gitaly). Gitaly will only return `Diff.Collapsed` (RPC) when surpassing collection limits. #### Not expandable patches (too large) @@ -121,7 +123,8 @@ Commit::DIFF_SAFE_LINES = Gitlab::Git::DiffCollection::DEFAULT_LIMITS[:max_lines File diff will be suppressed (technically different from collapsed, but behaves the same, and is expandable) if it has more than 5000 lines. -*Note:* This limit is currently hardcoded and only applied on GitLab. +NOTE: **Note:** +This limit is currently hardcoded and only applied on GitLab. ## Viewers diff --git a/doc/development/distributed_tracing.md b/doc/development/distributed_tracing.md index 15b3b8ba755..cbbeae47a41 100644 --- a/doc/development/distributed_tracing.md +++ b/doc/development/distributed_tracing.md @@ -1,3 +1,9 @@ +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Distributed Tracing - development guidelines GitLab is instrumented for distributed tracing. diff --git a/doc/development/doc_styleguide.md b/doc/development/doc_styleguide.md index 7d84f8ca86a..4a9f2a626f7 100644 --- a/doc/development/doc_styleguide.md +++ b/doc/development/doc_styleguide.md @@ -1,3 +1,5 @@ --- redirect_to: 'documentation/styleguide.md' --- + +This document was moved to [another location](documentation/styleguide.md). diff --git a/doc/development/documentation/feature_flags.md b/doc/development/documentation/feature_flags.md index f3ce9ce3a83..e2fbf25eb8a 100644 --- a/doc/development/documentation/feature_flags.md +++ b/doc/development/documentation/feature_flags.md @@ -56,7 +56,7 @@ not ready for production use: > - [Introduced](link-to-issue) in GitLab 12.0. > - It's deployed behind a feature flag, disabled by default. > - It's disabled on GitLab.com. -> - It's able to be enabled or disabled per-project +> - It's able to be enabled or disabled per-project. > - It's not recommended for production use. > - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#anchor-to-section). **(CORE ONLY)** @@ -67,7 +67,7 @@ not ready for production use: <Feature Name> is under development and not ready for production use. It is deployed behind a feature flag that is **disabled by default**. [GitLab administrators with access to the GitLab Rails console](../path/to/administration/feature_flags.md) -can enable it for your instance. <Feature Name> can be enabled or disabled per-project +can enable it for your instance. <Feature Name> can be enabled or disabled per-project. To enable it: @@ -109,7 +109,7 @@ For example, for a feature initially deployed disabled by default, that became e > - It was deployed behind a feature flag, disabled by default. > - [Became enabled by default](link-to-issue) on GitLab 12.1. > - It's enabled on GitLab.com. -> - It's not able to be enabled or disabled per-project +> - It's not able to be enabled or disabled per-project. > - It's recommended for production use. > - For GitLab self-managed instances, GitLab administrators can opt to [disable it](#anchor-to-section). **(CORE ONLY)** @@ -155,7 +155,7 @@ For example, for a feature enabled by default, enabled on GitLab.com, cannot be > - [Introduced](link-to-issue) in GitLab 12.0. > - It's deployed behind a feature flag, enabled by default. > - It's enabled on GitLab.com. -> - It's not able to be enabled or disabled per-project +> - It's not able to be enabled or disabled per-project. > - It's recommended for production use. > - For GitLab self-managed instances, GitLab administrators can opt to [disable it](#anchor-to-section). **(CORE ONLY)** diff --git a/doc/development/documentation/index.md b/doc/development/documentation/index.md index 2ea26985fcf..283060ba8d4 100644 --- a/doc/development/documentation/index.md +++ b/doc/development/documentation/index.md @@ -445,8 +445,8 @@ In case the review app URL returns 404, follow these steps to debug: If you want to know the in-depth details, here's what's really happening: 1. You manually run the `review-docs-deploy` job in a merge request. -1. The job runs the [`scripts/trigger-build-docs`](https://gitlab.com/gitlab-org/gitlab/blob/master/scripts/trigger-build-docs) - script with the `deploy` flag, which in turn: +1. The job runs the [`scripts/trigger-build`](https://gitlab.com/gitlab-org/gitlab/blob/master/scripts/trigger-build) + script with the `docs deploy` flag, which in turn: 1. Takes your branch name and applies the following: - The `docs-preview-` prefix is added. - The product slug is used to know the project the review app originated @@ -481,7 +481,7 @@ We treat documentation as code, and so use tests in our CI pipeline to maintain standards and quality of the docs. The current tests, which run in CI jobs when a merge request with new or changed docs is submitted, are: -- [`docs lint`](https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab/ci/docs.gitlab-ci.yml#L48): +- [`docs lint`](https://gitlab.com/gitlab-org/gitlab/-/blob/0b562014f7b71f98540e682c8d662275f0011f2f/.gitlab/ci/docs.gitlab-ci.yml#L41): Runs several tests on the content of the docs themselves: - [`lint-doc.sh` script](https://gitlab.com/gitlab-org/gitlab/blob/master/scripts/lint-doc.sh) runs the following checks and linters: @@ -492,33 +492,20 @@ merge request with new or changed docs is submitted, are: - [markdownlint](#markdownlint). - [Vale](#vale). - Nanoc tests: - - [`internal_links`](https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab/ci/docs.gitlab-ci.yml#L67) + - [`internal_links`](https://gitlab.com/gitlab-org/gitlab/-/blob/0b562014f7b71f98540e682c8d662275f0011f2f/.gitlab/ci/docs.gitlab-ci.yml#L58) checks that all internal links (ex: `[link](../index.md)`) are valid. - - [`internal_anchors`](https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab/ci/docs.gitlab-ci.yml#L69) + - [`internal_anchors`](https://gitlab.com/gitlab-org/gitlab/-/blob/0b562014f7b71f98540e682c8d662275f0011f2f/.gitlab/ci/docs.gitlab-ci.yml#L60) checks that all internal anchors (ex: `[link](../index.md#internal_anchor)`) are valid. + - [`ui-docs-links lint`](https://gitlab.com/gitlab-org/gitlab/-/blob/0b562014f7b71f98540e682c8d662275f0011f2f/.gitlab/ci/docs.gitlab-ci.yml#L62) + checks that all links to docs from UI elements (`app/views` files, for example) + are linking to valid docs and anchors. -### Running tests +### Run tests locally Apart from [previewing your changes locally](#previewing-the-changes-live), you can also run all lint checks and Nanoc tests locally. -#### Nanoc tests - -To execute Nanoc tests locally: - -1. Navigate to the [`gitlab-docs`](https://gitlab.com/gitlab-org/gitlab-docs) directory. -1. Run: - - ```shell - # Check for broken internal links - bundle exec nanoc check internal_links - - # Check for broken external links (might take a lot of time to complete). - # This test is set to be allowed to fail and is run only in the gitlab-docs project CI - bundle exec nanoc check internal_anchors - ``` - #### Lint checks Lint checks are performed by the [`lint-doc.sh`](https://gitlab.com/gitlab-org/gitlab/blob/master/scripts/lint-doc.sh) @@ -550,6 +537,57 @@ The output should be similar to: Note that this requires you to either have the required lint tools installed on your machine, or a working Docker installation, in which case an image with these tools pre-installed will be used. +#### Nanoc tests + +To execute Nanoc tests locally: + +1. Navigate to the [`gitlab-docs`](https://gitlab.com/gitlab-org/gitlab-docs) directory. +1. Run: + + ```shell + # Check for broken internal links + bundle exec nanoc check internal_links + + # Check for broken external links (might take a lot of time to complete). + # This test is set to be allowed to fail and is run only in the gitlab-docs project CI + bundle exec nanoc check internal_anchors + ``` + +#### `ui-docs-links` test + +The `ui-docs-links lint` job uses `haml-lint` to test that all links to docs from +UI elements (`app/views` files, for example) are linking to valid docs and anchors. + +To run the `ui-docs-links` test locally: + +1. Open the `gitlab` directory in a terminal window. +1. Run: + + ```shell + bundle exec haml-lint -i DocumentationLinks + ``` + +If you receive an error the first time you run this test, run `bundle install`, which +installs GitLab's dependencies, and try again. + +If you don't want to install all of GitLab's dependencies to test the links, you can: + +1. Open the `gitlab` directory in a terminal window. +1. Install `haml-lint`: + + ```shell + gem install haml_lint + ``` + +1. Run: + + ```shell + haml-lint -i DocumentationLinks + ``` + +If you manually install `haml-lint` with this process, it will not update automatically +and you should make sure your version matches the version used by GitLab. + ### Local linters To help adhere to the [documentation style guidelines](styleguide.md), and improve the content @@ -586,6 +624,7 @@ You can use markdownlint: - [On the command line](https://github.com/igorshubovych/markdownlint-cli#markdownlint-cli--). - [Within a code editor](#configure-editors). +- [In a `pre-commit` hook](#configure-pre-commit-hooks). #### Vale @@ -612,6 +651,9 @@ You can use Vale: - [On the command line](https://errata-ai.gitbook.io/vale/getting-started/usage). - [Within a code editor](#configure-editors). +- [In a `pre-commit` hook](#configure-pre-commit-hooks). Vale only reports errors in the + `pre-commit` hook (the same configuration as the CI/CD pipelines), and does not report suggestions + or warnings. #### Install linters @@ -655,14 +697,32 @@ To configure markdownlint within your editor, install one of the following as ap - [Sublime Text](https://packagecontrol.io/packages/SublimeLinter-contrib-markdownlint) - [Visual Studio Code](https://marketplace.visualstudio.com/items?itemName=DavidAnson.vscode-markdownlint) - [Atom](https://atom.io/packages/linter-node-markdownlint) +- [Vim](https://github.com/dense-analysis/ale) To configure Vale within your editor, install one of the following as appropriate: - The Sublime Text [`SublimeLinter-contrib-vale` plugin](https://packagecontrol.io/packages/SublimeLinter-contrib-vale) - The Visual Studio Code [`testthedocs.vale` extension](https://marketplace.visualstudio.com/items?itemName=testthedocs.vale) +- [Vim](https://github.com/dense-analysis/ale) We don't use [Vale Server](https://errata-ai.github.io/vale/#using-vale-with-a-text-editor-or-another-third-party-application). +#### Configure pre-commit hooks + +Git [pre-commit hooks](https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks) allow Git users to +run tests or other processes before committing to a branch, with the ability to not commit to the branch if +failures occur with these tests. + +[`overcommit`](https://github.com/sds/overcommit) is a Git hooks manager, making configuring, +installing, and removing Git hooks easy. + +Sample configuration for `overcommit` is available in the +[`.overcommit.yml.example`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.overcommit.yml.example) +file for the [`gitlab`](https://gitlab.com/gitlab-org/gitlab) project. + +To set up `overcommit` for documentation linting, see +[Pre-commit static analysis](../contributing/style_guides.md#pre-commit-static-analysis). + #### Disable Vale tests You can disable a specific Vale linting rule or all Vale linting rules for any portion of a diff --git a/doc/development/documentation/site_architecture/deployment_process.md b/doc/development/documentation/site_architecture/deployment_process.md new file mode 100644 index 00000000000..00cdc69d422 --- /dev/null +++ b/doc/development/documentation/site_architecture/deployment_process.md @@ -0,0 +1,58 @@ +# Documentation deployment process + +The [`dockerfiles` directory](https://gitlab.com/gitlab-org/gitlab-docs/blob/master/dockerfiles/) +contains all needed Dockerfiles to build and deploy <https://docs.gitlab.com>. It +is heavily inspired by Docker's +[Dockerfile](https://github.com/docker/docker.github.io/blob/06ed03db13895bfe867761b6fc2ad40acf6026dd/Dockerfile). + +The following Dockerfiles are used. + +| Dockerfile | Docker image | Description | +| ---------- | ------------ | ----------- | +| [`Dockerfile.bootstrap`](https://gitlab.com/gitlab-org/gitlab-docs/blob/master/dockerfiles/Dockerfile.bootstrap) | `gitlab-docs:bootstrap` | Contains all the dependencies that are needed to build the website. If the gems are updated and `Gemfile{,.lock}` changes, the image must be rebuilt. | +| [`Dockerfile.builder.onbuild`](https://gitlab.com/gitlab-org/gitlab-docs/blob/master/dockerfiles/Dockerfile.builder.onbuild) | `gitlab-docs:builder-onbuild` | Base image to build the docs website. It uses `ONBUILD` to perform all steps and depends on `gitlab-docs:bootstrap`. | +| [`Dockerfile.nginx.onbuild`](https://gitlab.com/gitlab-org/gitlab-docs/blob/master/dockerfiles/Dockerfile.nginx.onbuild) | `gitlab-docs:nginx-onbuild` | Base image to use for building documentation archives. It uses `ONBUILD` to perform all required steps to copy the archive, and relies upon its parent `Dockerfile.builder.onbuild` that is invoked when building single documentation archives (see the `Dockerfile` of each branch. | +| [`Dockerfile.archives`](https://gitlab.com/gitlab-org/gitlab-docs/blob/master/dockerfiles/Dockerfile.archives) | `gitlab-docs:archives` | Contains all the versions of the website in one archive. It copies all generated HTML files from every version in one location. | + +## How to build the images + +Although build images are built automatically via GitLab CI/CD, you can build +and tag all tooling images locally: + +1. Make sure you have [Docker installed](https://docs.docker.com/install/). +1. Make sure you're in the `dockerfiles/` directory of the `gitlab-docs` repository. +1. Build the images: + + ```shell + docker build -t registry.gitlab.com/gitlab-org/gitlab-docs:bootstrap -f Dockerfile.bootstrap ../ + docker build -t registry.gitlab.com/gitlab-org/gitlab-docs:builder-onbuild -f Dockerfile.builder.onbuild ../ + docker build -t registry.gitlab.com/gitlab-org/gitlab-docs:nginx-onbuild -f Dockerfile.nginx.onbuild ../ + ``` + +For each image, there's a manual job under the `images` stage in +[`.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab-docs/blob/master/.gitlab-ci.yml) which can be invoked at will. + +## Update an old Docker image with new upstream docs content + +If there are any changes to any of the stable branches of the products that are +not included in the single Docker image, just rerun the pipeline (`https://gitlab.com/gitlab-org/gitlab-docs/pipelines/new`) +for the version in question. + +## Porting new website changes to old versions + +CAUTION: **Warning:** +Porting changes to older branches can have unintended effects as we're constantly +changing the backend of the website. Use only when you know what you're doing +and make sure to test locally. + +The website will keep changing and being improved. In order to consolidate +those changes to the stable branches, we'd need to pick certain changes +from time to time. + +If this is not possible or there are many changes, merge master into them: + +```shell +git branch 12.0 +git fetch origin master +git merge origin/master +``` diff --git a/doc/development/documentation/site_architecture/global_nav.md b/doc/development/documentation/site_architecture/global_nav.md index 2625fbe0415..22d97a9e2cf 100644 --- a/doc/development/documentation/site_architecture/global_nav.md +++ b/doc/development/documentation/site_architecture/global_nav.md @@ -304,9 +304,9 @@ Examples: # does not include index.html at the end docs: - - doc_title: Service Desk - doc_url: 'user/project/service_desk.html' - ee_only: false + - doc_title: Container Scanning + doc_url: 'user/application_security/container_scanning/' + ee_only: true # note that the URL above ends in html and, as the # document is EE-only, the attribute ee_only is set to true. ``` diff --git a/doc/development/documentation/site_architecture/index.md b/doc/development/documentation/site_architecture/index.md index 60e6d4bcb13..63cd9959985 100644 --- a/doc/development/documentation/site_architecture/index.md +++ b/doc/development/documentation/site_architecture/index.md @@ -138,6 +138,8 @@ If you need to build and deploy the site immediately (must have maintainer level 1. In [`gitlab-docs`](https://gitlab.com/gitlab-org/gitlab-docs), go to **{rocket}** **CI / CD > Schedules**. 1. For the `Build docs.gitlab.com every 4 hours` scheduled pipeline, click the **play** (**{play}**) button. +Read more about the [deployment process](deployment_process.md). + ## Using YAML data files The easiest way to achieve something similar to diff --git a/doc/development/documentation/site_architecture/release_process.md b/doc/development/documentation/site_architecture/release_process.md index d100ab8afa8..d04d34ff786 100644 --- a/doc/development/documentation/site_architecture/release_process.md +++ b/doc/development/documentation/site_architecture/release_process.md @@ -1,51 +1,21 @@ # GitLab Docs monthly release process -The [`dockerfiles` directory](https://gitlab.com/gitlab-org/gitlab-docs/blob/master/dockerfiles/) -contains all needed Dockerfiles to build and deploy the versioned website. It -is heavily inspired by Docker's -[Dockerfile](https://github.com/docker/docker.github.io/blob/06ed03db13895bfe867761b6fc2ad40acf6026dd/Dockerfile). - -The following Dockerfiles are used. - -| Dockerfile | Docker image | Description | -| ---------- | ------------ | ----------- | -| [`Dockerfile.bootstrap`](https://gitlab.com/gitlab-org/gitlab-docs/blob/master/dockerfiles/Dockerfile.bootstrap) | `gitlab-docs:bootstrap` | Contains all the dependencies that are needed to build the website. If the gems are updated and `Gemfile{,.lock}` changes, the image must be rebuilt. | -| [`Dockerfile.builder.onbuild`](https://gitlab.com/gitlab-org/gitlab-docs/blob/master/dockerfiles/Dockerfile.builder.onbuild) | `gitlab-docs:builder-onbuild` | Base image to build the docs website. It uses `ONBUILD` to perform all steps and depends on `gitlab-docs:bootstrap`. | -| [`Dockerfile.nginx.onbuild`](https://gitlab.com/gitlab-org/gitlab-docs/blob/master/dockerfiles/Dockerfile.nginx.onbuild) | `gitlab-docs:nginx-onbuild` | Base image to use for building documentation archives. It uses `ONBUILD` to perform all required steps to copy the archive, and relies upon its parent `Dockerfile.builder.onbuild` that is invoked when building single documentation archives (see the `Dockerfile` of each branch. | -| [`Dockerfile.archives`](https://gitlab.com/gitlab-org/gitlab-docs/blob/master/dockerfiles/Dockerfile.archives) | `gitlab-docs:archives` | Contains all the versions of the website in one archive. It copies all generated HTML files from every version in one location. | - -## How to build the images - -Although build images are built automatically via GitLab CI/CD, you can build -and tag all tooling images locally: - -1. Make sure you have [Docker installed](https://docs.docker.com/install/). -1. Make sure you're in the `dockerfiles/` directory of the `gitlab-docs` repository. -1. Build the images: - - ```shell - docker build -t registry.gitlab.com/gitlab-org/gitlab-docs:bootstrap -f Dockerfile.bootstrap ../ - docker build -t registry.gitlab.com/gitlab-org/gitlab-docs:builder-onbuild -f Dockerfile.builder.onbuild ../ - docker build -t registry.gitlab.com/gitlab-org/gitlab-docs:nginx-onbuild -f Dockerfile.nginx.onbuild ../ - ``` - -For each image, there's a manual job under the `images` stage in -[`.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab-docs/blob/master/.gitlab-ci.yml) which can be invoked at will. - -## Monthly release process - When a new GitLab version is released on the 22nd, we need to create the respective single Docker image, and update some files so that the dropdown works correctly. -### 1. Add the chart version +## 1. Add the chart version Since the charts use a different version number than all the other GitLab products, we need to add a [version mapping](https://docs.gitlab.com/charts/installation/version_mappings.html): -1. Check that there is a [stable branch created](https://gitlab.com/gitlab-org/charts/gitlab/-/branches) - for the new chart version. If you're unsure or can't find it, drop a line in - the `#g_delivery` channel. +NOTE: **Note:** +The charts stable branch is not created automatically like the other products. +There's an [issue to track this](https://gitlab.com/gitlab-org/charts/gitlab/-/issues/1442). +It is usually created on the 21st or the 22nd. + +To add a new charts version: + 1. Make sure you're in the root path of the `gitlab-docs` repository. 1. Open `content/_data/chart_versions.yaml` and add the new stable branch version using the version mapping. Note that only the `major.minor` version is needed. @@ -56,7 +26,7 @@ It can be handy to create the future mappings since they are pretty much known. In that case, when a new GitLab version is released, you don't have to repeat this first step. -### 2. Create an image for a single version +## 2. Create an image for a single version The single docs version must be created before the release merge request, but this needs to happen when the stable branches for all products have been created. @@ -89,7 +59,7 @@ docker run -it --rm -p 4000:4000 docs:12.0 Visit `http://localhost:4000/12.0/` to see if everything works correctly. -### 3. Create the release merge request +## 3. Create the release merge request NOTE: **Note:** To be [automated](https://gitlab.com/gitlab-org/gitlab-docs/-/issues/750). @@ -135,7 +105,7 @@ version and rotates the old one: git push origin release-12-0 ``` -### 4. Update the dropdown for all online versions +## 4. Update the dropdown for all online versions The versions dropdown is in a way "hardcoded". When the site is built, it looks at the contents of `content/_data/versions.yaml` and based on that, the dropdown @@ -144,14 +114,18 @@ dropdown will list one or more releases behind. Remember that the new changes of the dropdown are included in the unmerged `release-X-Y` branch. The content of `content/_data/versions.yaml` needs to change for all online -versions: +versions (stable branches `X.Y` of the `gitlab-docs` project): 1. Run the Rake task that will create all the respective merge requests needed to update the dropdowns and will be set to automatically be merged when their - pipelines succeed. The `release-X-Y` branch needs to be present locally, - and you need to have switched to it, otherwise the Rake task will fail: + pipelines succeed: + + NOTE: **Note:** + The `release-X-Y` branch needs to be present locally, + and you need to have switched to it, otherwise the Rake task will fail. ```shell + git checkout release-X-Y ./bin/rake release:dropdowns ``` @@ -162,46 +136,22 @@ versions: TIP: **Tip:** In case a pipeline fails, see [troubleshooting](#troubleshooting). -### 5. Merge the release merge request +## 5. Merge the release merge request The dropdown merge requests should have now been merged into their respective -version (stable branch), which will trigger another pipeline. At this point, +version (stable `X.Y` branch), which will trigger another pipeline. At this point, you need to only babysit the pipelines and make sure they don't fail: -1. Check the pipelines page: `https://gitlab.com/gitlab-org/gitlab-docs/pipelines` +1. Check the [pipelines page](https://gitlab.com/gitlab-org/gitlab-docs/pipelines) and make sure all stable branches have green pipelines. 1. After all the pipelines of the online versions succeed, merge the release merge request. -1. Finally, from `https://gitlab.com/gitlab-org/gitlab-docs/pipeline_schedules` run - the `Build docker images weekly` pipeline that will build the `:latest` and `:archives` Docker images. +1. Finally, run the + [`Build docker images weekly` pipeline](https://gitlab.com/gitlab-org/gitlab-docs/pipeline_schedules) + that will build the `:latest` and `:archives` Docker images. Once the scheduled pipeline succeeds, the docs site will be deployed with all new versions online. -## Update an old Docker image with new upstream docs content - -If there are any changes to any of the stable branches of the products that are -not included in the single Docker image, just rerun the pipeline (`https://gitlab.com/gitlab-org/gitlab-docs/pipelines/new`) -for the version in question. - -## Porting new website changes to old versions - -CAUTION: **Warning:** -Porting changes to older branches can have unintended effects as we're constantly -changing the backend of the website. Use only when you know what you're doing -and make sure to test locally. - -The website will keep changing and being improved. In order to consolidate -those changes to the stable branches, we'd need to pick certain changes -from time to time. - -If this is not possible or there are many changes, merge master into them: - -```shell -git branch 12.0 -git fetch origin master -git merge origin/master -``` - ## Troubleshooting Releasing a new version is a long process that involves many moving parts. diff --git a/doc/development/documentation/structure.md b/doc/development/documentation/structure.md index 4cfc57aa57b..e13b2f4d031 100644 --- a/doc/development/documentation/structure.md +++ b/doc/development/documentation/structure.md @@ -27,7 +27,7 @@ be used. There is no need to add a title called "Introduction" or "Overview," be search for these terms. Just put this information after the title. - **Use cases**: describes real use case scenarios for that feature/configuration. - **Requirements**: describes what software, configuration, account, or knowledge is required. -- **Instructions**: One or more sets of detailed instructions to follow. +- **Instructions**: one or more sets of detailed instructions to follow. - **Troubleshooting** guide (recommended but not required). For additional details on each, see the [template for new docs](#template-for-new-docs), @@ -42,40 +42,48 @@ To start a new document, respect the file tree and file name guidelines, as well as the style guidelines. Use the following template: ```markdown -<!--Follow the Style Guide when working on this document. https://docs.gitlab.com/ee/development/documentation/styleguide.html -When done, remove all of this commented-out text, except a commented-out Troubleshooting section, -which, if empty, can be left in place to encourage future use.--> +<!--Follow the Style Guide when working on this document. +https://docs.gitlab.com/ee/development/documentation/styleguide.html +When done, remove all of this commented-out text, except a commented-out +Troubleshooting section, which, if empty, can be left in place to encourage future use.--> --- -description: "Short document description." # Up to ~200 chars long. They will be displayed in Google Search snippets. It may help to write the page intro first, and then reuse it here. +description: "Short document description." # Up to ~200 chars long. They will be displayed +in Google Search snippets. It may help to write the page intro first, and then reuse it here. stage: "Add the stage name here, and remove the quotation marks" group: "Add the group name here, and remove the quotation marks" -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +info: To determine the technical writer assigned to the Stage/Group associated with this page, +see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # Feature Name or Use Case Name **[TIER]** (1) -<!--If writing about a use case, drop the tier, and start with a verb, e.g. "Configure", "Implement", + the goal/scenario--> +<!--If writing about a use case, drop the tier, and start with a verb, +for example, "Configure", "Implement", + the goal/scenario--> -<!--For pages on newly introduced features, add the following line. If only some aspects of the feature have been introduced, specify what parts of the feature.--> +<!--For pages on newly-introduced features, add the following line. +If only some aspects of the feature have been introduced, specify which parts of the feature.--> > [Introduced](link_to_issue_or_mr) in GitLab (Tier) X.Y (2). An introduction -- without its own additional header -- goes here. Offer a description of the feature or use case, and what to expect on this page. -(You can reuse this content, or part of it, for the front matter's `description` at the top of this file). +(You can reuse this content, or part of it, for the front matter's `description` at the top +of this file). The introduction should answer the following questions: - What is this feature or use case? - Who is it for? - What is the context in which it is used and are there any prerequisites/requirements? -- What can the audience do with this? (Be sure to consider all applicable audiences, like GitLab admin and developer-user.) +- What can the audience do with this? (Be sure to consider all applicable audiences, like + GitLab admin and developer-user.) - What are the benefits to using this over any alternatives? ## Use cases Describe some use cases, typically in bulleted form. Include real-life examples for each. -If the page itself is dedicated to a use case, this section can usually include more specific scenarios -for use (e.g. variations on the main use case), but if that's not applicable, the section can be omitted. +If the page itself is dedicated to a use case, this section can usually include more specific +scenarios for use (for example, variations on the main use case), but if that's not applicable, +the section can be omitted. Examples of use cases on feature pages: - CE and EE: [Issues](../../user/project/issues/index.md#use-cases) @@ -88,27 +96,60 @@ Examples of use cases on feature pages: State any requirements for using the feature and/or following along with the instructions. These can include both: -- technical requirements (e.g. an account on a third party service, an amount of storage space, prior configuration of another feature) -- prerequisite knowledge (e.g. familiarity with certain GitLab features, cloud technologies) +- technical requirements (for example, an account on a third party service, an amount of storage space, + prior configuration of another feature) +- prerequisite knowledge (for example, familiarity with certain GitLab features, cloud technologies) Link each one to an appropriate place for more information. ## Instructions -"Instructions" is usually not the name of the heading. -This is the part of the document where you can include one or more sets of instructions, each to accomplish a specific task. -Headers should describe the task the reader will achieve by following the instructions within, typically starting with a verb. +This is the part of the document where you can include one or more sets of instructions. +Each topic should help users accomplish a specific task. + +Headers should describe the task the reader will achieve by following the instructions within, +typically starting with a verb. For example, `Create a package` or `Configure a pipeline`. + Larger instruction sets may have subsections covering specific phases of the process. -Where appropriate, provide examples of code or configuration files to better clarify intended usage. +Where appropriate, provide examples of code or configuration files to better clarify +intended usage. - Write a step-by-step guide, with no gaps between the steps. -- Include example code or configurations as part of the relevant step. Use appropriate Markdown to [wrap code blocks with syntax highlighting](../../user/markdown.md#colored-code-and-syntax-highlighting). +- Include example code or configurations as part of the relevant step. + Use appropriate Markdown to wrap code blocks with + [syntax highlighting](../../user/markdown.md#colored-code-and-syntax-highlighting). - Start with an h2 (`##`), break complex steps into small steps using -subheadings h3 > h4 > h5 > h6. _Never skip a hierarchy level, such -as h2 > h4_, as it will break the TOC and may affect the breadcrumbs. + subheadings h3 > h4 > h5 > h6. _Never skip a hierarchy level, such + as h2 > h4_, as it will break the TOC and may affect the breadcrumbs. - Use short and descriptive headings (up to ~50 chars). You can use one -single heading like `## Configure X` for instructions when the feature -is simple and the document is short. + single heading like `## Configure X` for instructions when the feature + is simple and the document is short. + +Example topic: + +## Create a teddy bear + +Start by writing a sentence or two about _why_ someone would want to perform this task. +It's not always possible, but is a good practice. For example: + +Create a teddy bear when you need something to hug. + +Follow this information with the task steps. + +To create a teddy bear: + +1. Go to **Settings > CI/CD**. +1. Expand **This** and click **This**. +1. Do another step. + +After the numbered list, add a sentence with the expected result, if it +is not obvious, and any next steps. For example: + +The teddy bear is now in the kitchen, in the cupboard above the sink. + +You can retrieve the teddy bear and put it on the couch with the other animals. + +Screenshots are not necessary. They are difficult to keep up-to-date and can clutter the page. <!-- ## Troubleshooting @@ -118,7 +159,7 @@ important to describe those, too. Think of things that may go wrong and include This is important to minimize requests for support, and to avoid doc comments with questions that you know someone might ask. -Each scenario can be a third-level heading, e.g. `### Getting error message X`. +Each scenario can be a third-level heading, for example, `### Getting error message X`. If you have none to add when creating a doc, leave this section in place but commented out to help encourage others to add to it in the future. --> @@ -127,7 +168,8 @@ but commented out to help encourage others to add to it in the future. --> Notes: - (1): Apply the [tier badges](styleguide.md#product-badges) accordingly -- (2): Apply the correct format for the [GitLab version introducing the feature](styleguide.md#gitlab-versions-and-tiers) +- (2): Apply the correct format for the + [GitLab version that introduces the feature](styleguide.md#gitlab-versions-and-tiers) ``` ## Help and feedback section @@ -167,3 +209,28 @@ Disqus, therefore, don't add both keys to the same document. The click events in the feedback section are tracked with Google Tag Manager. The conversions can be viewed on Google Analytics by navigating to **Behavior > Events > Top events > docs**. + +## Guidelines for good practices + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/36576/) in GitLab 13.2 as GitLab Development documentation. + +"Good practice" examples demonstrate encouraged ways of writing code while comparing with examples of practices to avoid. +These examples are labeled as "Bad" or "Good". +In GitLab development guidelines, when presenting the cases, it is recommended +to follow a **first-bad-then-good** strategy. First demonstrate the "Bad" practice (how things _could_ be done, which is often still working code), +and then how things _should_ be done better, using a "Good" example. This is typically an improved example of the same code. + +Consider the following guidelines when offering examples: + +- First, offer the "Bad" example, then the "Good" one. +- When only one bad case and one good case is given, use the same code block. +- When more than one bad case or one good case is offered, use separated code blocks for each. +With many examples being presented, a clear separation helps the reader to go directly to the good part. +Consider offering an explanation (for example, a comment, a link to a resource, etc.) on why something is bad practice. +- Better and best cases can be considered part of the good case(s) code block. +In the same code block, precede each with comments: `# Better` and `# Best`. + +NOTE: **Note:** +While the bad-then-good approach is acceptable for the GitLab development guidelines, do not use it +for user documentation. For user documentation, use "Do" and "Don't." For example, see the +[Pajamas Design System](https://design.gitlab.com/content/punctuation/). diff --git a/doc/development/documentation/styleguide.md b/doc/development/documentation/styleguide.md index f2c90e71bd5..c252f6425d0 100644 --- a/doc/development/documentation/styleguide.md +++ b/doc/development/documentation/styleguide.md @@ -13,6 +13,8 @@ For programmatic help adhering to the guidelines, see [Testing](index.md#testing See the GitLab handbook for further [writing style guidelines](https://about.gitlab.com/handbook/communication/#writing-style-guidelines) that apply to all GitLab content, not just documentation. +View [a list of recent style guide updates](https://gitlab.com/dashboard/merge_requests?scope=all&utf8=%E2%9C%93&state=merged&label_name[]=tw-style¬[label_name][]=docs%3A%3Afix). + ## Documentation is the single source of truth (SSOT) ### Why a single source of truth @@ -249,7 +251,7 @@ GitLab documentation should be clear and easy to understand. - Be clear, concise, and stick to the goal of the documentation. - Write in US English with US grammar. (Tested in [`British.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/British.yml).) -- Use inclusive language. +- Use [inclusive language](#inclusive-language). ### Point of view @@ -261,37 +263,117 @@ because it’s friendly and easy to understand. ### Capitalization -- Capitalize "G" and "L" in GitLab. -- Use sentence case for: - - Titles. - - Labels. - - Menu items. - - Buttons. - - Headings. Don't capitalize other words in the title, unless - it refers to a product feature. For example: - - Capitalizing "issues" is acceptable in - `## What you can do with GitLab Issues`, but not in `## Closing multiple issues`. -- Use title case when referring to: - - [GitLab Features](https://about.gitlab.com/features/). For example, Issue Board, - Geo, and Runner. - - GitLab [product tiers](https://about.gitlab.com/pricing/). For example, GitLab Core - and GitLab Ultimate. (Tested in [`BadgeCapitalization.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/BadgeCapitalization.yml).) - - Third-party products. For example, Prometheus, Kubernetes, and Git. - - Methods or methodologies. For example, Continuous Integration, Continuous - Deployment, Scrum, and Agile. - (Tested in [`.markdownlint.json`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.markdownlint.json).) - - NOTE: **Note:** - Some features are also objects. For example, "GitLab's Merge Requests support X" and - "Create a new merge request for Z." +#### Headings + +Use sentence case. For example: + +- `# Use variables to configure pipelines` +- `## Use the To-Do List` + +#### UI text + +When referring to specific user interface text, like a button label or menu item, use the same capitalization that is displayed in the UI. +Standards for this content are listed in the [Pajamas Design System Content section](https://design.gitlab.com/content/punctuation/) and typically +match what is called for in this Documentation Style Guide. + +If you think there is a mistake in the way the UI text is styled, +create an issue or an MR to propose a change to the UI text. + +#### Feature names + +- **Feature names are typically lowercase**, like those describing actions and types of objects in GitLab. For example: + - epics + - issues + - issue weights + - merge requests + - milestones + - reorder issues + - runner, runners, shared runners +- **Some features are capitalized**, typically nouns naming GitLab-specific capabilities or tools. For example: + - GitLab CI/CD + - Repository Mirroring + - Value Stream Analytics + - the To-Do List + - the Web IDE + - Geo + - GitLab Runner (see [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/233529) for details) + +Document any exceptions in this style guide. If you're not sure, ask a GitLab Technical Writer so that they can help decide and document the result. + +Do not match the capitalization of terms or phrases on the [Features page](https://about.gitlab.com/features/) or [features.yml](https://gitlab.com/gitlab-com/www-gitlab-com/blob/master/data/features.yml) by default. + +#### Other terms + +Capitalize names of: + +- GitLab [product tiers](https://about.gitlab.com/pricing/). For example, GitLab Core + and GitLab Ultimate. (Tested in [`BadgeCapitalization.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/BadgeCapitalization.yml).) +- Third-party organizations, software, and products. For example, Prometheus, Kubernetes, Git, and The Linux Foundation. +- Methods or methodologies. For example, Continuous Integration, Continuous Deployment, Scrum, and Agile. (Tested in [`.markdownlint.json`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.markdownlint.json).) + +Follow the capitalization style listed at the [authoritative source](#links-to-external-documentation) for the entity, which may use non-standard case styles. For example: GitLab and npm. + +### Inclusive language + +We strive to create documentation that is inclusive. This section includes guidance and examples in the +following categories: + +- [Gender-specific wording](#avoid-gender-specific-wording). +- [Ableist language](#avoid-ableist-language). +- [Cultural sensitivity](#culturally-sensitive-language). + +We write our developer documentation with inclusivity and diversity in mind. This page is not an exhaustive reference, but describes some general guidelines and examples that illustrate some best practices to follow. + +#### Avoid gender-specific wording + +When possible, use gender-neutral pronouns. For example, you can use a singular +[they](https://developers.google.com/style/pronouns#gender-neutral-pronouns) as a gender-neutral +pronoun. + +Avoid the use of gender-specific pronouns, unless referring to a specific person. + +| Use | Avoid | +|-----------------------------------|-----------------| +| People, humanity | Mankind | +| GitLab Team Members | Manpower | +| You can install; They can install | He can install; She can install | + +If you need to set up [Fake user information](#fake-user-information), use diverse or non-gendered +names with common surnames. + +#### Avoid ableist language + +Avoid terms that are also used in negative stereotypes for different groups. + +| Use | Avoid | +|------------------------|----------------------| +| Check for completeness | Sanity check | +| Uncertain outliers | Crazy outliers | +| Slows the service | Cripples the service | +| Placeholder variable | Dummy variable | +| Active/Inactive | Enabled/Disabled | +| On/Off | Enabled/Disabled | + +Credit: [Avoid ableist language](https://developers.google.com/style/inclusive-documentation#ableist-language) in the Google Developer Style Guide. + +#### Culturally sensitive language + +Avoid terms that reflect negative cultural stereotypes and history. In most cases, you can replace terms such as `master` and `slave` with terms that are more precise and functional, such as `primary` and `secondary`. + +| Use | Avoid | +|----------------------|-----------------------| +| Primary / secondary | Master / slave | +| Allowlist / denylist | Blacklist / whitelist | + +For more information see the following [Internet Draft specification](https://tools.ietf.org/html/draft-knodel-terminology-02). ### Language to avoid When creating documentation, limit or avoid the use of the following verb tenses, words, and phrases: -- Avoid jargon. -- Avoid uncommon words. +- Avoid jargon when possible, and when not possible, define the term or [link to a definition](#links-to-external-documentation). +- Avoid uncommon words when a more-common alternative is possible, ensuring that content is accessible to more readers. - Don't write in the first person singular. (Tested in [`FirstPerson.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/FirstPerson.yml).) - Instead of "I" or "me," use "we," "you," "us," or "one." @@ -431,6 +513,7 @@ tenses, words, and phrases: Check the general punctuation rules for the GitLab documentation on the table below. Check specific punctuation rules for [lists](#lists) below. +Additional examples are available in the [Pajamas guide for punctuation](https://design.gitlab.com/content/punctuation/). | Rule | Example | | ---- | ------- | @@ -806,9 +889,40 @@ Using the Markdown extension is necessary for the [`/help`](index.md#gitlab-help ### Links to external documentation When describing interactions with external software, it's often helpful to include links to external -documentation. When possible, make sure that you are linking to an **authoritative** source. +documentation. When possible, make sure that you're linking to an [**authoritative** source](#authoritative-sources). For example, if you're describing a feature in Microsoft's Active Directory, include a link to official Microsoft documentation. +### Authoritative sources + +When citing external information, use sources that are written by the people who created +the item or product in question. These sources are the most likely to +be accurate and remain up to date. + +Examples of authoritative sources include: + +- Specifications, such as a [Request for Comments](https://www.ietf.org/standards/rfcs/) document +from the Internet Engineering Task Force. +- Official documentation for a product. For example, if you're setting up an interface with the +Google OAuth 2 authorization server, include a link to Google's documentation. +- Official documentation for a project. For example, if you're citing NodeJS functionality, +refer directly to [NodeJS documentation](https://nodejs.org/en/docs/). +- Books from an authoritative publisher. + +Examples of sources to avoid include: + +- Personal blog posts. +- Wikipedia. +- Non-trustworthy articles. +- Discussions on forums such as Stack Overflow. +- Documentation from a company that describes another company's product. + +While many of these sources to avoid can help you learn skills and or features, they can become +obsolete quickly. Nobody is obliged to maintain any of these sites. Therefore, we should avoid using them as reference literature. + +NOTE: **Note:** +Non-authoritative sources are acceptable only if there is no equivalent authoritative source. +Even then, focus on non-authoritative sources that are extensively cited or peer-reviewed. + ### Links requiring permissions Don't link directly to: @@ -1135,48 +1249,39 @@ Usage examples: [Bootstrap utility class](https://getbootstrap.com/docs/4.4/utilities/float/): `**{tanuki, 32, float-right}**` renders as: **{tanuki, 32, float-right}** -### Use GitLab SVGs to describe UI elements +### When to use icons -When using GitLab SVGs to describe screen elements, also include the name or tooltip of the element as text. +Icons should be used sparingly, and only in ways that aid and do not hinder the readability of the +text. -For example, for references to the Admin Area: +For example, the following adds little to the accompanying text: -- Correct: `**{admin}** **Admin Area > Settings**` (**{admin}** **Admin Area > Settings**) -- Incorrect: `**{admin}** **> Settings**` (**{admin}** **> Settings**) +```markdown +1. Go to **{home}** **Project overview > Details** +``` -This will ensure that the source Markdown remains readable and should help with accessibility. +1. Go to **{home}** **Project overview > Details** -The following are examples of source Markdown for menu items with their published output: +However, the following might help the reader connect the text to the user interface: ```markdown -1. Go to **{home}** **Project overview > Details** -1. Go to **{doc-text}** **Repository > Branches** -1. Go to **{issues}** **Issues > List** -1. Go to **{merge-request}** **Merge Requests** -1. Go to **{rocket}** **CI/CD > Pipelines** -1. Go to **{shield}** **Security & Compliance > Configuration** -1. Go to **{cloud-gear}** **Operations > Metrics** -1. Go to **{package}** **Packages > Container Registry** -1. Go to **{chart}** **Project Analytics > Code Review** -1. Go to **{book}** **Wiki** -1. Go to **{snippet}** **Snippets** -1. Go to **{users}** **Members** -1. Select the **More actions** **{ellipsis_v}** icon > **Hide stage** +| Section | Description | +|:-------------------------|:----------------------------------------------------------------------------------------------------------------------------| +| **{overview}** Overview | View your GitLab Dashboard, and administer projects, users, groups, jobs, Runners, and Gitaly servers. | +| **{monitor}** Monitoring | View GitLab system information, and information on background jobs, logs, health checks, requests profiles, and audit logs. | +| **{messages}** Messages | Send and manage broadcast messages for your users. | ``` -1. Go to **{home}** **Project overview > Details** -1. Go to **{doc-text}** **Repository > Branches** -1. Go to **{issues}** **Issues > List** -1. Go to **{merge-request}** **Merge Requests** -1. Go to **{rocket}** **CI/CD > Pipelines** -1. Go to **{shield}** **Security & Compliance > Configuration** -1. Go to **{cloud-gear}** **Operations > Metrics** -1. Go to **{package}** **Packages > Container Registry** -1. Go to **{chart}** **Project Analytics > Code Review** -1. Go to **{book}** **Wiki** -1. Go to **{snippet}** **Snippets** -1. Go to **{users}** **Members** -1. Select the **More actions** **{ellipsis_v}** icon > **Hide stage** +| Section | Description | +|:-------------------------|:----------------------------------------------------------------------------------------------------------------------------| +| **{overview}** Overview | View your GitLab Dashboard, and administer projects, users, groups, jobs, Runners, and Gitaly servers. | +| **{monitor}** Monitoring | View GitLab system information, and information on background jobs, logs, health checks, requests profiles, and audit logs. | +| **{messages}** Messages | Send and manage broadcast messages for your users. | + +Use an icon when you find youself having to describe an interface element. For example: + +- Do: Click the Admin Area icon ( **{admin}** ). +- Don't: Click the Admin Area icon (the wrench icon). ## Alert boxes @@ -1294,21 +1399,20 @@ Which renders to: To maintain consistency through GitLab documentation, the following guides documentation authors on agreed styles and usage of terms. -### Merge Requests (MRs) +### Merge requests (MRs) Merge requests allow you to exchange changes you made to source code and collaborate with other people on the same project. You'll see this term used in the following ways: -- If you're referring to the feature, use **Merge Request**. -- In any other context, use **merge request**. +- Use lowercase **merge requests** regardless of whether referring to the feature or individual merge requests. -As noted in our corporate [Writing Style Guidelines](https://about.gitlab.com/handbook/communication/#writing-style-guidelines), +As noted in the GitLab [Writing Style Guidelines](https://about.gitlab.com/handbook/communication/#writing-style-guidelines), if you use the **MR** acronym, expand it at least once per document page. -For example, the first time you specify a MR, specify either _Merge Request (MR)_ or _merge request (MR)_. +Typically, the first use would be phrased as _merge request (MR)_ with subsequent instances being _MR_. Examples: -- "We prefer GitLab Merge Requests". +- "We prefer GitLab merge requests". - "Open a merge request to fix a broken link". - "After you open a merge request (MR), submit your MR for review and approval". @@ -1476,43 +1580,43 @@ GitLab Community Edition), don't split the product or feature name across lines. ### Product badges -When a feature is available in EE-only tiers, add the corresponding tier according to the -feature availability: +When a feature is available in paid tiers, add the corresponding tier to the +header or other page element according to the feature's availability: -- For GitLab Core and GitLab.com Free: `**(CORE)**`. -- For GitLab Starter and GitLab.com Bronze: `**(STARTER)**`. -- For GitLab Premium and GitLab.com Silver: `**(PREMIUM)**`. -- For GitLab Ultimate and GitLab.com Gold: `**(ULTIMATE)**`. +| Tier in which feature is available | Tier markup | +|:-----------------------------------------------------------------------|:----------------------| +| GitLab Core and GitLab.com Free, and their higher tiers | `**(CORE)**` | +| GitLab Starter and GitLab.com Bronze, and their higher tiers | `**(STARTER)**` | +| GitLab Premium and GitLab.com Silver, and their higher tiers | `**(PREMIUM)**` | +| GitLab Ultimate and GitLab.com Gold | `**(ULTIMATE)**` | +| *Only* GitLab Core and higher tiers (no GitLab.com-based tiers) | `**(CORE ONLY)**` | +| *Only* GitLab Starter and higher tiers (no GitLab.com-based tiers) | `**(STARTER ONLY)**` | +| *Only* GitLab Premium and higher tiers (no GitLab.com-based tiers) | `**(PREMIUM ONLY)**` | +| *Only* GitLab Ultimate (no GitLab.com-based tiers) | `**(ULTIMATE ONLY)**` | +| *Only* GitLab.com Free and higher tiers (no self-managed instances) | `**(FREE ONLY)**` | +| *Only* GitLab.com Bronze and higher tiers (no self-managed instances) | `**(BRONZE ONLY)**` | +| *Only* GitLab.com Silver and higher tiers (no self-managed instances) | `**(SILVER ONLY)**` | +| *Only* GitLab.com Gold (no self-managed instances) | `**(GOLD ONLY)**` | -To exclude GitLab.com tiers (when the feature is not available in GitLab.com), add the -keyword "only": +For clarity, all page title headers (H1s) must be have a tier markup for +the lowest tier that has information on the documentation page. -- For GitLab Core: `**(CORE ONLY)**`. -- For GitLab Starter: `**(STARTER ONLY)**`. -- For GitLab Premium: `**(PREMIUM ONLY)**`. -- For GitLab Ultimate: `**(ULTIMATE ONLY)**`. +If sections of a page apply to higher tier levels, they can be separately +labeled with their own tier markup. -For GitLab.com only tiers (when the feature is not available for self-managed instances): +#### Product badge display behavior -- For GitLab Free and higher tiers: `**(FREE ONLY)**`. -- For GitLab Bronze and higher tiers: `**(BRONZE ONLY)**`. -- For GitLab Silver and higher tiers: `**(SILVER ONLY)**`. -- For GitLab Gold: `**(GOLD ONLY)**`. +When using the tier markup with headers, the documentation page will +display the full tier badge with the header line. -The tier should be ideally added to headers, so that the full badge will be displayed. -However, it can be also mentioned from paragraphs, list items, and table cells. For these cases, -the tier mention will be represented by an orange info icon **(information)** that will show the tiers on hover. - -Use the lowest tier at the page level, even if higher-level tiers exist on the page. For example, you might have a page that is marked as Starter but a section badged as Premium. - -For example: +You can also use the tier markup with paragraphs, list items, +and table cells. For these cases, the tier mention will be represented by an +orange info icon **{information}** that will display the tiers when visitors +point to the icon. For example: -- `**(STARTER)**` renders as **(STARTER)** -- `**(STARTER ONLY)**` renders as **(STARTER ONLY)** -- `**(SILVER ONLY)**` renders as **(SILVER ONLY)** - -The absence of tiers' mentions mean that the feature is available in GitLab Core, -GitLab.com Free, and all higher tiers. +- `**(STARTER)**` displays as **(STARTER)** +- `**(STARTER ONLY)**` displays as **(STARTER ONLY)** +- `**(SILVER ONLY)**` displays as **(SILVER ONLY)** #### How it works @@ -1622,10 +1726,10 @@ Learn how to [document features deployed behind flags](feature_flags.md). For guidance on developing GitLab with feature flags, see [Feature flags in development of GitLab](../feature_flags/index.md). -## API +## RESTful API -Here is a list of must-have items. Use them in the exact order that appears -on this document. Further explanation is given below. +Here is a list of must-have items for RESTful API documentation. Use them in the +exact order that appears on this document. Further explanation is given below. - Every method must have the REST API request. For example: @@ -1776,7 +1880,8 @@ curl --data "name=foo" --header "PRIVATE-TOKEN: <your_access_token>" "https://gi #### Post data using JSON content -> **Note:** In this example we create a new group. Watch carefully the single +NOTE: **Note:** +In this example we create a new group. Watch carefully the single and double quotes. ```shell @@ -1816,3 +1921,80 @@ exclude specific users when requesting a list of users for a project, you would ```shell curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" --data "skip_users[]=<user_id>" --data "skip_users[]=<user_id>" https://gitlab.example.com/api/v4/projects/<project_id>/users ``` + +## GraphQL API + +GraphQL APIs are different from [RESTful APIs](#restful-api). Reference information is +generated automatically in our [GraphQL reference](../../api/graphql/reference/index.md). + +However, it's helpful to include examples on how to use GraphQL for different "use cases", +with samples that readers can use directly in the [GraphiQL explorer](../api_graphql_styleguide.md#graphiql). + +This section describes the steps required to add your GraphQL examples to GitLab documentation. + +### Add a dedicated GraphQL page + +To create a dedicated GraphQL page, create a new `.md` file in the `doc/api/graphql/` directory. +Give that file a functional name, such as `import_from_specific_location.md`. + +### Start the page with an explanation + +Include a page title that describes the GraphQL functionality in a few words, such as: + +```markdown +# Search for [substitute kind of data] +``` + +Describe the search. One sentence may be all you need. More information may help +readers learn how to use the example for their GitLab deployments. + +### Include a procedure using the GraphiQL explorer + +The GraphiQL explorer can help readers test queries with working deployments. Set up the section with the following: + +- Use the following title: + + ```markdown + ## Set up the GraphiQL explorer + ``` + +- Include a code block with the query that anyone can include in their instance of + the GraphiQL explorer: + + ````markdown + ```graphql + query { + <insert queries here> + } + ``` + ```` + +- Tell the user what to do: + + ```markdown + 1. Open the GraphiQL explorer tool in the following URL: `https://gitlab.com/-/graphql-explorer`. + 1. Paste the `query` listed above into the left window of your GraphiQL explorer tool. + 1. Click Play to get the result shown here: + ``` + +- Include a screenshot of the result in the GraphiQL explorer. Follow the naming + convention described in the [Save the image](#save-the-image) section. +- Follow up with an example of what you can do with the output. + Make sure the example is something that readers can do on their own deployments. +- Include a link to the [GraphQL API resources](../../api/graphql/reference/index.md). + +### Add the GraphQL example to the Table of Contents + +You'll need to open a second MR, against the [GitLab Docs repository](https://gitlab.com/gitlab-org/gitlab-docs/). + +We store our Table of Contents in the `default-nav.yaml` file, in the `content/_data` +subdirectory. You can find the GraphQL section under the following line: + +```yaml + - category_title: GraphQL +``` + +Be aware that CI tests for that second MR will fail with a bad link until the main MR +that adds the new GraphQL page is merged. + +And that's all you need! diff --git a/doc/development/ee_features.md b/doc/development/ee_features.md index ac544113cbd..e7954fa910b 100644 --- a/doc/development/ee_features.md +++ b/doc/development/ee_features.md @@ -56,6 +56,12 @@ This works because for every path that is present in CE's eager-load/auto-load paths, we add the same `ee/`-prepended path in [`config/application.rb`](https://gitlab.com/gitlab-org/gitlab/blob/925d3d4ebc7a2c72964ce97623ae41b8af12538d/config/application.rb#L42-52). This also applies to views. +#### Testing EE-only features + +To test an EE class that doesn't exist in CE, create the spec file as you normally +would in the `ee/spec` directory, but without the second `ee/` subdirectory. +For example, a class `ee/app/models/vulnerability.rb` would have its tests in `ee/spec/models/vulnerability_spec.rb`. + ### EE features based on CE features For features that build on existing CE features, write a module in the `EE` @@ -96,6 +102,21 @@ This is also not just applied to models. Here's a list of other examples: - `ee/app/validators/ee/foo_attr_validator.rb` - `ee/app/workers/ee/foo_worker.rb` +#### Testing EE features based on CE features + +To test an `EE` namespaced module that extends a CE class with EE features, +create the spec file as you normally would in the `ee/spec` directory, including the second `ee/` subdirectory. +For example, an extension `ee/app/models/ee/user.rb` would have its tests in `ee/app/models/ee/user_spec.rb`. + +In the `RSpec.describe` call, use the CE class name where the EE module would be used. +For example, in `ee/app/models/ee/user_spec.rb`, the test would start with: + +```ruby +RSpec.describe User do + describe 'ee feature added through extension' +end +``` + #### Overriding CE methods To override a method present in the CE codebase, use `prepend`. It @@ -904,7 +925,7 @@ export default { - Please do not use mixins unless ABSOLUTELY NECESSARY. Please try to find an alternative pattern. -##### Reccomended alternative approach (named/scoped slots) +##### Recommended alternative approach (named/scoped slots) - We can use slots and/or scoped slots to achieve the same thing as we did with mixins. If you only need an EE component there is no need to create the CE component. diff --git a/doc/development/elasticsearch.md b/doc/development/elasticsearch.md index 90debab3b5c..2f01692e944 100644 --- a/doc/development/elasticsearch.md +++ b/doc/development/elasticsearch.md @@ -1,3 +1,9 @@ +--- +stage: Enablement +group: Global Search +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Elasticsearch knowledge **(STARTER ONLY)** This area is to maintain a compendium of useful information when working with Elasticsearch. @@ -121,6 +127,9 @@ Patterns: ## Zero downtime reindexing with multiple indices +NOTE: **Note:** +This is not applicable yet as multiple indices functionality is not fully implemented. + Currently GitLab can only handle a single version of setting. Any setting/schema changes would require reindexing everything from scratch. Since reindexing can take a long time, this can cause search functionality downtime. To avoid downtime, GitLab is working to support multiple indices that diff --git a/doc/development/fe_guide/frontend_faq.md b/doc/development/fe_guide/frontend_faq.md index 3c0845a9aaa..71436a7c7fb 100644 --- a/doc/development/fe_guide/frontend_faq.md +++ b/doc/development/fe_guide/frontend_faq.md @@ -163,3 +163,33 @@ To return to the normal development mode: 1. Run `yarn clean` to remove the production assets and free some space (optional). 1. Start webpack again: `gdk start webpack`. 1. Restart GDK: `gdk-restart rails-web`. + +### 8. Babel polyfills + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/28837) in GitLab 12.8. + +GitLab has enabled the Babel `preset-env` option +[`useBuiltIns: 'usage'`](https://babeljs.io/docs/en/babel-preset-env#usebuiltins-usage), +which adds the appropriate `core-js` polyfills once for each JavaScript feature +we're using that our target browsers don't support. You don't need to add `core-js` +polyfills manually. + +NOTE: **Note:** +GitLab still manually adds non-`core-js` polyfills for extending browser features +(such as GitLab's SVG polyfill) that allow us reference SVGs by using `<use xlink:href>`. +These polyfills should be added to `app/assets/javascripts/commons/polyfills.js`. + +To see what polyfills are being used: + +1. Navigate to your merge request. +1. In the secondary menu below the title of the merge request, click **Pipelines**, then + click the pipeline you want to view, to display the jobs in that pipeline. +1. Click the [`compile-production-assets`](https://gitlab.com/gitlab-org/gitlab/-/jobs/641770154) job. +1. In the right-hand sidebar, scroll to **Job Artifacts**, and click **Browse**. +1. Click the **webpack-report** folder to open it, and click **index.html**. +1. In the upper left corner of the page, click the right arrow **{angle-right}** + to display the explorer. +1. In the **Search modules** field, enter `gitlab/node_modules/core-js` to see + which polyfills are being loaded and where: + +  diff --git a/doc/development/fe_guide/graphql.md b/doc/development/fe_guide/graphql.md index 3d74ec94ae4..f5e16d377f1 100644 --- a/doc/development/fe_guide/graphql.md +++ b/doc/development/fe_guide/graphql.md @@ -75,7 +75,7 @@ their execution by clicking **Execute query** button on the top left: ## Apollo Client To save duplicated clients getting created in different apps, we have a -[default client](https://gitlab.com/gitlab-org/gitlab/blob/master/app/assets/javascripts/lib/graphql.js) that should be used. This setups the +[default client](https://gitlab.com/gitlab-org/gitlab/blob/master/app/assets/javascripts/lib/graphql.js) that should be used. This sets up the Apollo client with the correct URL and also sets the CSRF headers. Default client accepts two parameters: `resolvers` and `config`. @@ -85,6 +85,7 @@ Default client accepts two parameters: `resolvers` and `config`. - `cacheConfig` field accepts an optional object of settings to [customize Apollo cache](https://www.apollographql.com/docs/react/caching/cache-configuration/#configuring-the-cache) - `baseUrl` allows us to pass a URL for GraphQL endpoint different from our main endpoint (i.e.`${gon.relative_url_root}/api/graphql`) - `assumeImmutableResults` (set to `false` by default) - this setting, when set to `true`, will assume that every single operation on updating Apollo Cache is immutable. It also sets `freezeResults` to `true`, so any attempt on mutating Apollo Cache will throw a console warning in development environment. Please ensure you're following the immutability pattern on cache update operations before setting this option to `true`. + - `fetchPolicy` determines how you want your component to interact with the Apollo cache. Defaults to "cache-first". ## GraphQL Queries @@ -167,9 +168,7 @@ import VueApollo from 'vue-apollo'; import createDefaultClient from '~/lib/graphql'; Vue.use(VueApollo); -const defaultClient = createDefaultClient({ - resolvers: {} -}); +const defaultClient = createDefaultClient(); defaultClient.cache.writeData({ data: { @@ -257,10 +256,7 @@ We need to pass resolvers object to our existing Apollo Client: import createDefaultClient from '~/lib/graphql'; import resolvers from './graphql/resolvers'; -const defaultClient = createDefaultClient( - {}, - resolvers, -); +const defaultClient = createDefaultClient(resolvers); ``` Now every single time on attempt to fetch a version, our client will fetch `id` and `sha` from the remote API endpoint and will assign our hardcoded values to `author` and `createdAt` version properties. With this data, frontend developers are able to work on UI part without being blocked by backend. When actual response is added to the API, a custom local resolver can be removed fast and the only change to query/fragment is `@client` directive removal. @@ -470,6 +466,28 @@ fetchNextPage() { Please note we don't have to save `pageInfo` one more time; `fetchMore` triggers a query `result` hook as well. +### Managing performance + +The Apollo client will batch queries by default. This means that if you have 3 queries defined, +Apollo will group them into one request, send the single request off to the server and only +respond once all 3 queries have completed. + +If you need to have queries sent as individual requests, additional context can be provided +to tell Apollo to do this. + +```javascript +export default { + apollo: { + user: { + query: QUERY_IMPORT, + context: { + isSingleRequest: true, + } + } + }, +}; +``` + ### Testing #### Mocking response as component data @@ -501,6 +519,7 @@ If we need to test how our component renders when results from the GraphQL API a designs: { loading, }, + }, }; wrapper = shallowMount(Index, { @@ -595,7 +614,7 @@ These errors are located at the "top level" of a GraphQL response. These are non #### Handling top-level errors -Apollo is aware of top-level errors, so we are able to leverage Apollo's various error-handling mechanisms to handle these errors (e.g. handling Promise rejections after invoking the [`mutate`](https://www.apollographql.com/docs/react/api/apollo-client/#ApolloClient.mutate) method, or handling the `error` event emitted from the [`ApolloMutation`](https://apollo.vuejs.org/api/apollo-mutation.html#events) component). +Apollo is aware of top-level errors, so we are able to leverage Apollo's various error-handling mechanisms to handle these errors (e.g. handling Promise rejections after invoking the [`mutate`](https://www.apollographql.com/docs/react/api/core/ApolloClient/#ApolloClient.mutate) method, or handling the `error` event emitted from the [`ApolloMutation`](https://apollo.vuejs.org/api/apollo-mutation.html#events) component). Because these errors are not intended for users, error messages for top-level errors should be defined client-side. diff --git a/doc/development/fe_guide/icons.md b/doc/development/fe_guide/icons.md index 7078b5e9b2f..b539293e9cf 100644 --- a/doc/development/fe_guide/icons.md +++ b/doc/development/fe_guide/icons.md @@ -76,6 +76,89 @@ export default { Please use the following function inside JS to render an icon: `gl.utils.spriteIcon(iconName)` +## Loading icon + +### Usage in HAML/Rails + +DANGER: **Danger:** +Do not use the `spinner` or `icon('spinner spin')` rails helpers to insert +loading icons. These helpers rely on the Font Awesome icon library which is +deprecated. + +To insert a loading spinner in HAML or Rails use the `loading_icon` helper: + +```haml += loading_icon +``` + +You can include one or more of the following properties with the `loading_icon` helper, as demonstrated +by the examples that follow: + +- `container` (optional): wraps the loading icon in a container, which centers the loading icon using the `text-center` CSS property. +- `color` (optional): either `orange` (default), `light`, or `dark`. +- `size` (optional): either `sm` (default), `md`, `lg`, or `xl`. +- `css_class` (optional): defaults to an empty string, but can be useful for utility classes to fine-tune alignment or spacing. + +**Example 1:** + +The following HAML expression generates a loading icon’s markup and +centers the icon by wrapping it in a `gl-spinner-container` element. + +```haml += loading_icon(container: true) +``` + +**Output from example 1:** + +```html +<div class="gl-spinner-container"> + <span class="gl-spinner gl-spinner-orange gl-spinner-sm" aria-label="Loading"></span> +</div> +``` + +**Example 2:** + +The following HAML expression generates a loading icon’s markup +with a custom size. It also appends a margin utility class. + +```haml += loading_icon(size: 'lg', css_class: 'gl-mr-2') +``` + +**Output from example 2:** + +```html +<span class="gl-spinner gl-spinner-orange gl-spinner-lg gl-mr-2" aria-label="Loading"></span> +``` + +### Usage in Vue + +The [GitLab UI](https://gitlab-org.gitlab.io/gitlab-ui/) components library provides a +`GlLoadingIcon` component. See the component’s +[storybook](https://gitlab-org.gitlab.io/gitlab-ui/?path=/story/base-loading-icon--default) +for more information about its usage. + +**Example:** + +The following code snippet demonstrates how to use `GlLoadingIcon` in +a Vue component. + +```html +<script> +import { GlLoadingIcon } from "@gitlab/ui"; + +export default { + components: { + GlLoadingIcon, + }, +}; +<script> + +<template> + <gl-loading-icon inline /> +</template> +``` + ## SVG Illustrations Please use from now on for any SVG based illustrations simple `img` tags to show an illustration by simply using either `image_tag` or `image_path` helpers. diff --git a/doc/development/fe_guide/img/webpack_report_v12_8.png b/doc/development/fe_guide/img/webpack_report_v12_8.png Binary files differnew file mode 100644 index 00000000000..99c6f7d0eec --- /dev/null +++ b/doc/development/fe_guide/img/webpack_report_v12_8.png diff --git a/doc/development/fe_guide/index.md b/doc/development/fe_guide/index.md index 3a2b3cac9bf..ef23b6c4ed2 100644 --- a/doc/development/fe_guide/index.md +++ b/doc/development/fe_guide/index.md @@ -5,7 +5,7 @@ across GitLab's frontend team. ## Overview -GitLab is built on top of [Ruby on Rails](https://rubyonrails.org) using [Haml](http://haml.info/) and also a JavaScript based Frontend with [Vue.js](https://vuejs.org). +GitLab is built on top of [Ruby on Rails](https://rubyonrails.org) using [Haml](https://haml.info/) and also a JavaScript based Frontend with [Vue.js](https://vuejs.org). Be wary of [the limitations that come with using Hamlit](https://github.com/k0kubun/hamlit/blob/master/REFERENCE.md#limitations). We also use [SCSS](https://sass-lang.com) and plain JavaScript with modern ECMAScript standards supported through [Babel](https://babeljs.io/) and ES module support through [webpack](https://webpack.js.org/). diff --git a/doc/development/fe_guide/style/javascript.md b/doc/development/fe_guide/style/javascript.md index b69a6f1941c..2a06a473878 100644 --- a/doc/development/fe_guide/style/javascript.md +++ b/doc/development/fe_guide/style/javascript.md @@ -10,7 +10,7 @@ linter to manage most of our JavaScript style guidelines. In addition to the style guidelines set by Airbnb, we also have a few specific rules listed below. -> **Tip:** +TIP: **Tip:** You can run eslint locally by running `yarn eslint` ## Avoid forEach diff --git a/doc/development/fe_guide/style/scss.md b/doc/development/fe_guide/style/scss.md index 336c9b8ca35..dba39eeb98c 100644 --- a/doc/development/fe_guide/style/scss.md +++ b/doc/development/fe_guide/style/scss.md @@ -23,6 +23,14 @@ Classes in [`utilities.scss`](https://gitlab.com/gitlab-org/gitlab/blob/master/a Avoid [Bootstrap's Utility Classes](https://getbootstrap.com/docs/4.3/utilities/). +NOTE: **Note:** +While migrating [Bootstrap's Utility Classes](https://getbootstrap.com/docs/4.3/utilities/) +to the [GitLab UI](https://gitlab.com/gitlab-org/gitlab-ui/-/blob/master/doc/css.md#utilities) +utility classes, note both the classes for margin and padding differ. The size scale used at +GitLab differs from the scale used in the Bootstrap library. For a Bootstrap padding or margin +utility, you may need to double the size of the applied utility to achieve the same visual +result (such as `ml-1` becoming `gl-ml-2`). + #### Where should I put new utility classes? If a class you need has not been added to GitLab UI, you get to add it! Follow the naming patterns documented in the [utility files](https://gitlab.com/gitlab-org/gitlab-ui/-/tree/master/src/scss/utility-mixins) and refer to [GitLab UI's CSS documentation](https://gitlab.com/gitlab-org/gitlab-ui/-/blob/master/doc/contributing/adding_css.md#adding-utility-mixins) for more details, especially about adding responsive and stateful rules. @@ -45,8 +53,8 @@ Examples of component classes that were created using "utility-first" include: Inspiration: -- <https://tailwindcss.com/docs/utility-first/> -- <https://tailwindcss.com/docs/extracting-components/> +- <https://tailwindcss.com/docs/utility-first> +- <https://tailwindcss.com/docs/extracting-components> ### Naming diff --git a/doc/development/fe_guide/tooling.md b/doc/development/fe_guide/tooling.md index 28deb7d95f9..5685ac5abcd 100644 --- a/doc/development/fe_guide/tooling.md +++ b/doc/development/fe_guide/tooling.md @@ -40,7 +40,8 @@ yarn eslint-fix _If manual changes are required, a list of changes will be sent to the console._ -**Caution:** Limit use to global rule updates. Otherwise, the changes can lead to huge Merge Requests. +CAUTION: **Caution:** +Limit use to global rule updates. Otherwise, the changes can lead to huge Merge Requests. ### Disabling ESLint in new files diff --git a/doc/development/fe_guide/vue.md b/doc/development/fe_guide/vue.md index 2a0556c6cda..58a8332589d 100644 --- a/doc/development/fe_guide/vue.md +++ b/doc/development/fe_guide/vue.md @@ -64,11 +64,11 @@ which will make the tests easier. See the following example: ```javascript // haml -.js-vue-app{ data: { endpoint: 'foo' }} +#js-vue-app{ data: { endpoint: 'foo' }} // index.js document.addEventListener('DOMContentLoaded', () => new Vue({ - el: '.js-vue-app', + el: '#js-vue-app', data() { const dataset = this.$options.el.dataset; return { @@ -85,6 +85,8 @@ document.addEventListener('DOMContentLoaded', () => new Vue({ })); ``` +> When adding an `id` attribute to mount a Vue application, please make sure this `id` is unique across the codebase + #### Accessing the `gl` object When we need to query the `gl` object for data that won't change during the application's life cycle, we should do it in the same place where we query the DOM. @@ -283,7 +285,7 @@ describe('~/todos/app.vue', () => { ### Test the component's output The main return value of a Vue component is the rendered output. In order to test the component we -need to test the rendered output. [Vue](https://vuejs.org/v2/guide/unit-testing.html) guide's to unit test show us exactly that: +need to test the rendered output. Visit the [Vue testing guide](https://vuejs.org/v2/guide/testing.html#Unit-Testing). ### Events diff --git a/doc/development/fe_guide/vue3_migration.md b/doc/development/fe_guide/vue3_migration.md index 7ab48db7f76..afdb2690c02 100644 --- a/doc/development/fe_guide/vue3_migration.md +++ b/doc/development/fe_guide/vue3_migration.md @@ -76,6 +76,9 @@ const FunctionalComp = (props, slots) => { } ``` +NOTE: **Note:** +It is not recommended to replace stateful components with functional components unless you absolutely need a performance improvement right now. In Vue 3, performance gains for functional components will be negligible. + ## Old slots syntax with `slot` attribute **Why?** diff --git a/doc/development/fe_guide/vuex.md b/doc/development/fe_guide/vuex.md index 02387c15951..9573dd36e63 100644 --- a/doc/development/fe_guide/vuex.md +++ b/doc/development/fe_guide/vuex.md @@ -40,24 +40,25 @@ The following example shows an application that lists and adds users to the stat This is the entry point for our store. You can use the following as a guide: ```javascript -import Vue from 'vue'; import Vuex from 'vuex'; import * as actions from './actions'; import * as getters from './getters'; import mutations from './mutations'; import state from './state'; -Vue.use(Vuex); - -export const createStore = () => new Vuex.Store({ - actions, - getters, - mutations, - state, -}); -export default createStore(); +export const createStore = () => + new Vuex.Store({ + actions, + getters, + mutations, + state, + }); ``` +_Note:_ Until this +[RFC](https://gitlab.com/gitlab-org/frontend/rfcs/-/issues/20) is implemented, +the above will need to disable the `import/prefer-default-export` ESLint rule. + ### `state.js` The first thing you should do before writing any code is to design the state. @@ -137,44 +138,12 @@ import { mapActions } from 'vuex'; ### `mutations.js` The mutations specify how the application state changes in response to actions sent to the store. -The only way to change state in a Vuex store should be by committing a mutation. - -**It's a good idea to think of the state before writing any code.** - -Remember that actions only describe that something happened, they don't describe how the application state changes. +The only way to change state in a Vuex store is by committing a mutation. -**Never commit a mutation directly from a component** - -Instead, you should create an action that will commit a mutation. - -```javascript - import * as types from './mutation_types'; +Most mutations are committed from an action using `commit`. If you don't have any +asynchronous operations, you can call mutations from a component using the `mapMutations` helper. - export default { - [types.REQUEST_USERS](state) { - state.isLoading = true; - }, - [types.RECEIVE_USERS_SUCCESS](state, data) { - // Do any needed data transformation to the received payload here - state.users = data; - state.isLoading = false; - }, - [types.RECEIVE_USERS_ERROR](state, error) { - state.isLoading = false; - }, - [types.REQUEST_ADD_USER](state, user) { - state.isAddingUser = true; - }, - [types.RECEIVE_ADD_USER_SUCCESS](state, user) { - state.isAddingUser = false; - state.users.push(user); - }, - [types.REQUEST_ADD_USER_ERROR](state, error) { - state.isAddingUser = false; - state.errorAddingUser = error; - }, - }; -``` +See the Vuex docs for examples of [committing mutations from components](https://vuex.vuejs.org/guide/mutations.html#committing-mutations-in-components). #### Naming Pattern: `REQUEST` and `RECEIVE` namespaces @@ -316,9 +285,12 @@ function when mounting your Vue component: // in the Vue app's initialization script (e.g. mount_show.js) import Vue from 'vue'; -import createStore from './stores'; +import Vuex from 'vuex'; +import { createStore } from './stores'; import AwesomeVueApp from './components/awesome_vue_app.vue' +Vue.use(Vuex); + export default () => { const el = document.getElementById('js-awesome-vue-app'); @@ -398,10 +370,8 @@ discussion](https://gitlab.com/gitlab-org/frontend/rfcs/-/issues/56#note_3025148 ```javascript <script> import { mapActions, mapState, mapGetters } from 'vuex'; -import store from './store'; export default { - store, computed: { ...mapGetters([ 'getUsersWithPets' @@ -417,12 +387,10 @@ export default { 'fetchUsers', 'addUser', ]), - onClickAddUser(data) { this.addUser(data); } }, - created() { this.fetchUsers() } @@ -448,29 +416,6 @@ export default { </template> ``` -### Vuex Gotchas - -1. Do not call a mutation directly. Always use an action to commit a mutation. Doing so will keep consistency throughout the application. From Vuex docs: - - > Why don't we just call store.commit('action') directly? Well, remember that mutations must be synchronous? Actions aren't. We can perform asynchronous operations inside an action. - - ```javascript - // component.vue - - // bad - created() { - this.$store.commit('mutation'); - } - - // good - created() { - this.$store.dispatch('action'); - } - ``` - -1. Use mutation types instead of hardcoding strings. It will be less error prone. -1. The State will be accessible in all components descending from the use where the store is instantiated. - ### Testing Vuex #### Testing Vuex concerns @@ -485,55 +430,50 @@ In order to write unit tests for those components, we need to include the store ```javascript //component_spec.js import Vue from 'vue'; +import Vuex from 'vuex'; +import { mount, createLocalVue } from '@vue/test-utils'; import { createStore } from './store'; -import component from './component.vue' +import Component from './component.vue' + +const localVue = createLocalVue(); +localVue.use(Vuex); describe('component', () => { let store; - let vm; - let Component; + let wrapper; + + const createComponent = () => { + store = createStore(); + + wrapper = mount(Component, { + localVue, + store, + }); + }; beforeEach(() => { - Component = Vue.extend(issueActions); + createComponent(); }); afterEach(() => { - vm.$destroy(); + wrapper.destroy(); + wrapper = null; }); - it('should show a user', () => { + it('should show a user', async () => { const user = { name: 'Foo', age: '30', }; - store = createStore(); - // populate the store - store.dispatch('addUser', user); + await store.dispatch('addUser', user); - vm = new Component({ - store, - propsData: props, - }).$mount(); + expect(wrapper.text()).toContain(user.name); }); }); ``` -#### Testing Vuex actions and getters - -Because we're currently using [`babel-plugin-rewire`](https://github.com/speedskater/babel-plugin-rewire), you may encounter the following error when testing your Vuex actions and getters: -`[vuex] actions should be function or object with "handler" function` - -To prevent this error from happening, you need to export an empty function as `default`: - -```javascript -// getters.js or actions.js - -// prevent babel-plugin-rewire from generating an invalid default during karma tests -export default () => {}; -``` - ### Two way data binding When storing form data in Vuex, it is sometimes necessary to update the value stored. The store should never be mutated directly, and an action should be used instead. diff --git a/doc/development/feature_categorization/index.md b/doc/development/feature_categorization/index.md index bcd5e16cc2b..3fd402ebe84 100644 --- a/doc/development/feature_categorization/index.md +++ b/doc/development/feature_categorization/index.md @@ -24,7 +24,7 @@ and generate a new version of the file, which needs to be committed to the repository. The [Scalabilitity -team](https://about.gitlab.com/handbook/engineering/infrastructure/team/scalability) +team](https://about.gitlab.com/handbook/engineering/infrastructure/team/scalability/) currently maintains the `stages.yml` file. They will automatically be notified on Slack when the file becomes outdated. diff --git a/doc/development/feature_flags.md b/doc/development/feature_flags.md index 6bad91d6287..cff88388ba0 100644 --- a/doc/development/feature_flags.md +++ b/doc/development/feature_flags.md @@ -1 +1,5 @@ +--- +redirect_to: 'feature_flags/index.md' +--- + This document was moved to [another location](feature_flags/index.md). diff --git a/doc/development/feature_flags/controls.md b/doc/development/feature_flags/controls.md index 9e7ce74cc0c..09260be1264 100644 --- a/doc/development/feature_flags/controls.md +++ b/doc/development/feature_flags/controls.md @@ -140,7 +140,7 @@ run the following in Slack: This sets a feature flag to `true` based on the following formula: ```ruby -feature_flag_state = Zlib.crc32("some_feature<Actor>:#{actor.id}") % (100 * 1_000) < 25 * 1_000] +feature_flag_state = Zlib.crc32("some_feature<Actor>:#{actor.id}") % (100 * 1_000) < 25 * 1_000 # where <Actor>: is a `User`, `Group`, `Project` and actor is an instance ``` diff --git a/doc/development/feature_flags/development.md b/doc/development/feature_flags/development.md index 0b918478668..9b30187fcd1 100644 --- a/doc/development/feature_flags/development.md +++ b/doc/development/feature_flags/development.md @@ -1,19 +1,219 @@ # Developing with feature flags -In general, it's better to have a group- or user-based gate, and you should prefer -it over the use of percentage gates. This would make debugging easier, as you -filter for example logs and errors based on actors too. Furthermore, this allows -for enabling for the `gitlab-org` or `gitlab-com` group first, while the rest of +This document provides guidelines on how to use feature flags +in the GitLab codebase to conditionally enable features +and test them. + +Features that are developed and merged behind a feature flag +should not include a changelog entry. The entry should be added either in the merge +request removing the feature flag or the merge request where the default value of +the feature flag is set to enabled. If the feature contains any database migrations, it +*should* include a changelog entry for the database changes. + +CAUTION: **Caution:** +All newly-introduced feature flags should be [disabled by default](process.md#feature-flags-in-gitlab-development). + +NOTE: **Note:** +This document is the subject of continued work as part of an epic to [improve internal usage of Feature Flags](https://gitlab.com/groups/gitlab-org/-/epics/3551). Raise any suggestions as new issues and attach them to the epic. + +## Types of feature flags + +Currently, only a single type of feature flag is available. +Additional feature flag types will be provided in the future, +with descriptions for their usage. + +### `development` type + +`development` feature flags are short-lived feature flags, +used so that unfinished code can be deployed in production. + +A `development` feature flag should have a rollout issue, +ideally created using the [Feature Flag Roll Out template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/issue_templates/Feature%20Flag%20Roll%20Out.md). + +NOTE: **Note:** +This is the default type used when calling `Feature.enabled?`. + +## Feature flag definition and validation + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/229161) in GitLab 13.3. + +During development (`RAILS_ENV=development`) or testing (`RAILS_ENV=test`) all feature flag usage is being strictly validated. + +This process is meant to ensure consistent feature flag usage in the codebase. All feature flags **must**: + +- Be known. Only use feature flags that are explicitly defined. +- Not be defined twice. They have to be defined either in FOSS or EE, but not both. +- Use a valid and consistent `type:` across all invocations. +- Use the same `default_enabled:` across all invocations. +- Have an owner. + +All feature flags known to GitLab are self-documented in YAML files stored in: + +- [`config/feature_flags`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/config/feature_flags) +- [`ee/config/feature_flags`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/ee/config/feature_flags) + +Each feature flag is defined in a separate YAML file consisting of a number of fields: + +| Field | Required | Description | +|---------------------|----------|----------------------------------------------------------------| +| `name` | yes | Name of the feature flag. | +| `type` | yes | Type of feature flag. | +| `default_enabled` | yes | The default state of the feature flag that is strongly validated, with `default_enabled:` passed as an argument. | +| `introduced_by_url` | no | The URL to the Merge Request that introduced the feature flag. | +| `rollout_issue_url` | no | The URL to the Issue covering the feature flag rollout. | +| `group` | no | The [group](https://about.gitlab.com/handbook/product/product-categories/#devops-stages) that owns the feature flag. | + +TIP: **Tip:** +All validations are skipped when running in `RAILS_ENV=production`. + +## Create a new feature flag + +The GitLab codebase provides [`bin/feature-flag`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/bin/feature-flag), +a dedicated tool to create new feature flag definitions. +The tool asks various questions about the new feature flag, then creates +a YAML definition in `config/feature_flags` or `ee/config/feature_flags`. + +Only feature flags that have a YAML definition file can be used when running the development or testing environments. + +```shell +$ bin/feature-flag my-feature-flag +>> Please specify the group introducing feature flag, like `group::apm`: +?> group::memory + +>> If you have MR open, can you paste the URL here? (or enter to skip) +?> https://gitlab.com/gitlab-org/gitlab/-/merge_requests/38602 + +>> Open this URL and fill the rest of details: +https://gitlab.com/gitlab-org/gitlab/-/issues/new?issue%5Btitle%5D=%5BFeature+flag%5D+Rollout+of+%60test-flag%60&issuable_template=Feature+Flag+Roll+Out + +>> Paste URL of `rollout issue` here, or enter to skip: +?> https://gitlab.com/gitlab-org/gitlab/-/issues/232533 +create config/feature_flags/development/test-flag.yml +--- +name: test-flag +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/38602 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/232533 +group: group::memory +type: development +default_enabled: false +``` + +TIP: **Tip:** +To create a feature flag that is only used in EE, add the `--ee` flag: `bin/feature-flag --ee` + +## Develop with a feature flag + +There are two main ways of using Feature Flags in the GitLab codebase: + +- [Backend code (Rails)](#backend) +- [Frontend code (VueJS)](#frontend) + +### Backend + +The feature flag interface is defined in [`lib/feature.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/feature.rb). +This interface provides a set of methods to check if the feature flag is enabled or disabled: + +```ruby +if Feature.enabled?(:my_feature_flag, project) + # execute code if feature flag is enabled +else + # execute code if feature flag is disabled +end + +if Feature.disabled?(:my_feature_flag, project) + # execute code if feature flag is disabled +end +``` + +In rare cases you may want to make a feature enabled by default. If so, explain the reasoning +in the merge request. Use `default_enabled: true` when checking the feature flag state: + +```ruby +if Feature.enabled?(:feature_flag, project, default_enabled: true) + # execute code if feature flag is enabled +else + # execute code if feature flag is disabled +end + +if Feature.disabled?(:my_feature_flag, project, default_enabled: true) + # execute code if feature flag is disabled +end +``` + +### Frontend + +Use the `push_frontend_feature_flag` method for frontend code, which is +available to all controllers that inherit from `ApplicationController`. You can use +this method to expose the state of a feature flag, for example: + +```ruby +before_action do + # Prefer to scope it per project or user e.g. + push_frontend_feature_flag(:vim_bindings, project) +end + +def index + # ... +end + +def edit + # ... +end +``` + +You can then check the state of the feature flag in JavaScript as follows: + +```javascript +if ( gon.features.vimBindings ) { + // ... +} +``` + +The name of the feature flag in JavaScript is always camelCase, +so checking for `gon.features.vim_bindings` would not work. + +See the [Vue guide](../fe_guide/vue.md#accessing-feature-flags) for details about +how to access feature flags in a Vue component. + +In rare cases you may want to make a feature enabled by default. If so, explain the reasoning +in the merge request. Use `default_enabled: true` when checking the feature flag state: + +```ruby +before_action do + # Prefer to scope it per project or user e.g. + push_frontend_feature_flag(:vim_bindings, project, default_enabled: true) +end +``` + +### Feature actors + +**It is strongly advised to use actors with feature flags.** Actors provide a simple +way to enable a feature flag only for a given project, group or user. This makes debugging +easier, as you can filter logs and errors for example, based on actors. This also makes it possible +to enable the feature on the `gitlab-org` or `gitlab-com` groups first, while the rest of the users aren't impacted. +Actors also provide an easy way to do a percentage rollout of a feature in a sticky way. +If a 1% rollout enabled a feature for a specific actor, that actor will continue to have the feature enabled at +10%, 50%, and 100%. + +GitLab currently supports the following models as feature flag actors: + +- `User` +- `Project` +- `Group` + +The actor is a second parameter of the `Feature.enabled?` call. The +same actor type must be used consistently for all invocations of `Feature.enabled?`. + ```ruby -# Good Feature.enabled?(:feature_flag, project) - -# Avoid, if possible -Feature.enabled?(:feature_flag) +Feature.enabled?(:feature_flag, group) +Feature.enabled?(:feature_flag, user) ``` +### Enable additional objects as actors + To use feature gates based on actors, the model needs to respond to `flipper_id`. For example, to enable for the Foo model: @@ -26,29 +226,18 @@ end Only models that `include FeatureGate` or expose `flipper_id` method can be used as an actor for `Feature.enabled?`. -Features that are developed and are intended to be merged behind a feature flag -should not include a changelog entry. The entry should either be added in the merge -request removing the feature flag or the merge request where the default value of -the feature flag is set to true. If the feature contains any DB migration it -should include a changelog entry for DB changes. - -If you need the feature flag to be on automatically, use `default_enabled: true` -when checking: +### Feature flags for licensed features -```ruby -Feature.enabled?(:feature_flag, project, default_enabled: true) -``` +If a feature is license-gated, there's no need to add an additional +explicit feature flag check since the flag will be checked as part of the +`License.feature_available?` call. Similarly, there's no need to "clean up" a +feature flag once the feature has reached general availability. The [`Project#feature_available?`](https://gitlab.com/gitlab-org/gitlab/blob/4cc1c62918aa4c31750cb21dfb1a6c3492d71080/app/models/project_feature.rb#L63-68), [`Namespace#feature_available?`](https://gitlab.com/gitlab-org/gitlab/blob/4cc1c62918aa4c31750cb21dfb1a6c3492d71080/ee/app/models/ee/namespace.rb#L71-85) (EE), and [`License.feature_available?`](https://gitlab.com/gitlab-org/gitlab/blob/4cc1c62918aa4c31750cb21dfb1a6c3492d71080/ee/app/models/license.rb#L293-300) (EE) methods all implicitly check for a by default enabled feature flag with the same name as the provided argument. -For example if a feature is license-gated, there's no need to add an additional -explicit feature flag check since the flag will be checked as part of the -`License.feature_available?` call. Similarly, there's no need to "clean up" a -feature flag once the feature has reached general availability. - You'd still want to use an explicit `Feature.enabled?` check if your new feature isn't gated by a License or Plan. @@ -58,7 +247,7 @@ the feature flag check will default to `true`.** This is relevant when developing the feature using [several smaller merge requests](https://about.gitlab.com/handbook/values/#make-small-merge-requests), or when the feature is considered to be an -[alpha or beta](https://about.gitlab.com/handbook/product/#alpha-beta-ga), and +[alpha or beta](https://about.gitlab.com/handbook/product/gitlab-the-product/#alpha-beta-ga), and should not be available by default. As an example, if you were to ship the frontend half of a feature without the @@ -67,13 +256,10 @@ also ready to be shipped. To make sure this feature is disabled for both GitLab.com and self-managed instances, you should use the [`Namespace#alpha_feature_available?`](https://gitlab.com/gitlab-org/gitlab/blob/458749872f4a8f27abe8add930dbb958044cb926/ee/app/models/ee/namespace.rb#L113) or [`Namespace#beta_feature_available?`](https://gitlab.com/gitlab-org/gitlab/blob/458749872f4a8f27abe8add930dbb958044cb926/ee/app/models/ee/namespace.rb#L100-112) -method, according to our [definitions](https://about.gitlab.com/handbook/product/#alpha-beta-ga). This ensures the feature is disabled unless the feature flag is +method, according to our [definitions](https://about.gitlab.com/handbook/product/gitlab-the-product/#alpha-beta-ga). This ensures the feature is disabled unless the feature flag is _explicitly_ enabled. -## Feature groups - -Starting from GitLab 9.4 we support feature groups via -[Flipper groups](https://github.com/jnunemaker/flipper/blob/v0.10.2/docs/Gates.md#2-group). +### Feature groups Feature groups must be defined statically in `lib/feature.rb` (in the `.register_feature_groups` method), but their implementation can obviously be @@ -82,50 +268,144 @@ dynamic (querying the DB etc.). Once defined in `lib/feature.rb`, you will be able to activate a feature for a given feature group via the [`feature_group` parameter of the features API](../../api/features.md#set-or-create-a-feature) -### Frontend +### Enabling a feature flag locally (in development) -For frontend code you can use the method `push_frontend_feature_flag`, which is -available to all controllers that inherit from `ApplicationController`. Using -this method you can expose the state of a feature flag as follows: +In the rails console (`rails c`), enter the following command to enable a feature flag: ```ruby -before_action do - # Prefer to scope it per project or user e.g. - push_frontend_feature_flag(:vim_bindings, project) +Feature.enable(:feature_flag_name) +``` - # Avoid, if possible - push_frontend_feature_flag(:vim_bindings) -end +Similarly, the following command will disable a feature flag: -def index - # ... -end +```ruby +Feature.disable(:feature_flag_name) +``` -def edit - # ... -end +You can also enable a feature flag for a given gate: + +```ruby +Feature.enable(:feature_flag_name, Project.find_by_full_path("root/my-project")) ``` -You can then check for the state of the feature flag in JavaScript as follows: +## Feature flags in tests -```javascript -if ( gon.features.vimBindings ) { - // ... -} +Introducing a feature flag into the codebase creates an additional codepath that should be tested. +It is strongly advised to test all code affected by a feature flag, both when **enabled** and **disabled** +to ensure the feature works properly. + +NOTE: **Note:** +When using the testing environment, all feature flags are enabled by default. + +To disable a feature flag in a test, use the `stub_feature_flags` +helper. For example, to globally disable the `ci_live_trace` feature +flag in a test: + +```ruby +stub_feature_flags(ci_live_trace: false) + +Feature.enabled?(:ci_live_trace) # => false ``` -The name of the feature flag in JavaScript will always be camelCased, meaning -that checking for `gon.features.vim_bindings` would not work. +If you wish to set up a test where a feature flag is enabled only +for some actors and not others, you can specify this in options +passed to the helper. For example, to enable the `ci_live_trace` +feature flag for a specific project: -See the [Vue guide](../fe_guide/vue.md#accessing-feature-flags) for details about -how to access feature flags in a Vue component. +```ruby +project1, project2 = build_list(:project, 2) -### Specs +# Feature will only be enabled for project1 +stub_feature_flags(ci_live_trace: project1) + +Feature.enabled?(:ci_live_trace) # => false +Feature.enabled?(:ci_live_trace, project1) # => true +Feature.enabled?(:ci_live_trace, project2) # => false +``` + +The behavior of FlipperGate is as follows: + +1. You can enable an override for a specified actor to be enabled +1. You can disable (remove) an override for a specified actor, + falling back to default state +1. There's no way to model that you explicitly disable a specified actor + +```ruby +Feature.enable(:my_feature) +Feature.disable(:my_feature, project1) +Feature.enabled?(:my_feature) # => true +Feature.enabled?(:my_feature, project1) # => true + +Feature.disable(:my_feature2) +Feature.enable(:my_feature2, project1) +Feature.enabled?(:my_feature2) # => false +Feature.enabled?(:my_feature2, project1) # => true +``` + +### `stub_feature_flags` vs `Feature.enable*` + +It is preferred to use `stub_feature_flags` to enable feature flags +in the testing environment. This method provides a simple and well described +interface for simple use cases. + +However, in some cases more complex behavior needs to be tested, +like percentage rollouts of feature flags. This can be done using +`.enable_percentage_of_time` or `.enable_percentage_of_actors`: + +```ruby +# Good: feature needs to be explicitly disabled, as it is enabled by default if not defined +stub_feature_flags(my_feature: false) +stub_feature_flags(my_feature: true) +stub_feature_flags(my_feature: project) +stub_feature_flags(my_feature: [project, project2]) + +# Bad +Feature.enable(:my_feature_2) + +# Good: enable my_feature for 50% of time +Feature.enable_percentage_of_time(:my_feature_3, 50) + +# Good: enable my_feature for 50% of actors/gates/things +Feature.enable_percentage_of_actors(:my_feature_4, 50) +``` + +Each feature flag that has a defined state will be persisted +during test execution time: + +```ruby +Feature.persisted_names.include?('my_feature') => true +Feature.persisted_names.include?('my_feature_2') => true +Feature.persisted_names.include?('my_feature_3') => true +Feature.persisted_names.include?('my_feature_4') => true +``` + +### Stubbing actor + +When you want to enable a feature flag for a specific actor only, +you can stub its representation. A gate that is passed +as an argument to `Feature.enabled?` and `Feature.disabled?` must be an object +that includes `FeatureGate`. + +In specs you can use the `stub_feature_flag_gate` method that allows you to +quickly create a custom actor: + +```ruby +gate = stub_feature_flag_gate('CustomActor') + +stub_feature_flags(ci_live_trace: gate) + +Feature.enabled?(:ci_live_trace) # => false +Feature.enabled?(:ci_live_trace, gate) # => true +``` + +### Controlling feature flags engine in tests Our Flipper engine in the test environment works in a memory mode `Flipper::Adapters::Memory`. `production` and `development` modes use `Flipper::Adapters::ActiveRecord`. -### `stub_feature_flags: true` (default and preferred) +You can control whether the `Flipper::Adapters::Memory` or `ActiveRecord` mode is being used. + +#### `stub_feature_flags: true` (default and preferred) In this mode Flipper is configured to use `Flipper::Adapters::Memory` and mark all feature flags to be on-by-default and persisted on a first use. This overwrites the `default_enabled:` @@ -145,23 +425,3 @@ a mode that is used by `production` and `development`. You should use this mode only when you really want to tests aspects of Flipper with how it interacts with `ActiveRecord`. - -### Enabling a feature flag (in development) - -In the rails console (`rails c`), enter the following command to enable your feature flag - -```ruby -Feature.enable(:feature_flag_name) -``` - -Similarly, the following command will disable a feature flag: - -```ruby -Feature.disable(:feature_flag_name) -``` - -You can as well enable feature flag for a given gate: - -```ruby -Feature.enable(:feature_flag_name, Project.find_by_full_path("root/my-project")) -``` diff --git a/doc/development/feature_flags/process.md b/doc/development/feature_flags/process.md index b053838964b..5dc3cf44a6e 100644 --- a/doc/development/feature_flags/process.md +++ b/doc/development/feature_flags/process.md @@ -24,7 +24,8 @@ should be leveraged: - When development of a feature will be spread across multiple merge requests, you can use the following workflow: - 1. Introduce a feature flag which is **off** by default, in the first merge request. + 1. [Create a new feature flag](development.md#create-a-new-feature-flag) + which is **off** by default, in the first merge request. 1. Submit incremental changes via one or more merge requests, ensuring that any new code added can only be reached if the feature flag is **on**. You can keep the feature flag enabled on your local GDK during development. diff --git a/doc/development/filtering_by_label.md b/doc/development/filtering_by_label.md index ef92bd35985..9c1993fdf7f 100644 --- a/doc/development/filtering_by_label.md +++ b/doc/development/filtering_by_label.md @@ -1,3 +1,8 @@ +--- +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- # Filtering by label ## Introduction diff --git a/doc/development/geo.md b/doc/development/geo.md index 5c781a60bac..57959b07e49 100644 --- a/doc/development/geo.md +++ b/doc/development/geo.md @@ -209,121 +209,12 @@ To migrate the tracking database, run: bundle exec rake geo:db:migrate ``` -### Foreign Data Wrapper - -> Introduced in GitLab 10.1. - -Foreign Data Wrapper ([FDW](#fdw)) is used by the [Geo Log Cursor](#geo-log-cursor) and improves -the performance of many synchronization operations. - -FDW is a PostgreSQL extension ([`postgres_fdw`](https://www.postgresql.org/docs/11/postgres-fdw.html)) that is enabled within -the Geo Tracking Database (on a **secondary** node), which allows it -to connect to the read-only database replica and perform queries and filter -data from both instances. - -This persistent connection is configured as an FDW server -named `gitlab_secondary`. This configuration exists within the database's user -context only. To access the `gitlab_secondary`, GitLab needs to use the -same database user that had previously been configured. - -The Geo Tracking Database accesses the read-only database replica via FDW as a regular user, -limited by its own restrictions. The credentials are configured as a -`USER MAPPING` associated with the `SERVER` mapped previously -(`gitlab_secondary`). - -FDW configuration and credentials definition are managed automatically by the -Omnibus GitLab `gitlab-ctl reconfigure` command. - -#### Refreshing the Foreign Tables - -Whenever a new Geo node is configured or the database schema changes on the -**primary** node, you must refresh the foreign tables on the **secondary** node -by running the following: - -```shell -bundle exec rake geo:db:refresh_foreign_tables -``` - -Failure to do this will prevent the **secondary** node from -functioning properly. The **secondary** node will generate error -messages, as the following PostgreSQL error: - -```sql -ERROR: relation "gitlab_secondary.ci_job_artifacts" does not exist at character 323 -STATEMENT: SELECT a.attname, format_type(a.atttypid, a.atttypmod), - pg_get_expr(d.adbin, d.adrelid), a.attnotnull, a.atttypid, a.atttypmod - FROM pg_attribute a LEFT JOIN pg_attrdef d - ON a.attrelid = d.adrelid AND a.attnum = d.adnum - WHERE a.attrelid = '"gitlab_secondary"."ci_job_artifacts"'::regclass - AND a.attnum > 0 AND NOT a.attisdropped - ORDER BY a.attnum -``` - -#### Accessing data from a Foreign Table - -At the SQL level, all you have to do is `SELECT` data from `gitlab_secondary.*`. - -Here's an example of how to access all projects from the Geo Tracking Database's FDW: - -```sql -SELECT * FROM gitlab_secondary.projects; -``` - -As a more real-world example, this is how you filter for unarchived projects -on the Tracking Database: - -```sql -SELECT project_registry.* - FROM project_registry - JOIN gitlab_secondary.projects - ON (project_registry.project_id = gitlab_secondary.projects.id - AND gitlab_secondary.projects.archived IS FALSE) -``` - -At the ActiveRecord level, we have additional Models that represent the -foreign tables. They must be mapped in a slightly different way, and they are read-only. - -Check the existing FDW models in `ee/app/models/geo/fdw` for reference. - -From a developer's perspective, it's no different than creating a model that -represents a Database View. - -With the examples above, you can access the projects with: - -```ruby -Geo::Fdw::Project.all -``` - -and to access the `ProjectRegistry` filtering by unarchived projects: - -```ruby -# We have to use Arel here: -project_registry_table = Geo::ProjectRegistry.arel_table -fdw_project_table = Geo::Fdw::Project.arel_table - -project_registry_table.join(fdw_project_table) - .on(project_registry_table[:project_id].eq(fdw_project_table[:id])) - .where((fdw_project_table[:archived]).eq(true)) # if you append `.to_sql` you can check generated query -``` - ## Finders Geo uses [Finders](https://gitlab.com/gitlab-org/gitlab/tree/master/app/finders), which are classes take care of the heavy lifting of looking up projects/attachments/etc. in the tracking database and main database. -### Finders Performance - -The Finders need to compare data from the main database with data in -the tracking database. For example, counting the number of synced -projects normally involves retrieving the project IDs from one -database and checking their state in the other database. This is slow -and requires a lot of memory. - -To overcome this, the Finders use [FDW](#fdw), or Foreign Data -Wrappers. This allows a regular `JOIN` between the main database and -the tracking database. - ## Redis Redis on the **secondary** node works the same as on the **primary** @@ -397,12 +288,6 @@ migration do not need to run on the secondary nodes. A database on each Geo **secondary** node that keeps state for the node on which it resides. Read more in [Using the Tracking database](#using-the-tracking-database). -### FDW - -Foreign Data Wrapper, or FDW, is a feature built-in in PostgreSQL. It -allows data to be queried from different data sources. In Geo, it's -used to query data from different PostgreSQL instances. - ## Geo Event Log The Geo **primary** stores events in the `geo_event_log` table. Each diff --git a/doc/development/geo/framework.md b/doc/development/geo/framework.md index de4d6fb869d..64c9030e3dd 100644 --- a/doc/development/geo/framework.md +++ b/doc/development/geo/framework.md @@ -90,6 +90,14 @@ module Geo def self.model ::Packages::PackageFile end + + # Change this to `true` to release replication of this model. Then remove + # this override in the next release. + # The feature flag follows the format `geo_#{replicable_name}_replication`, + # so here it would be `geo_package_file_replication` + def self.replication_enabled_by_default? + false + end end end ``` @@ -180,6 +188,11 @@ For example, to add support for files referenced by a `Widget` model with a mount_uploader :file, WidgetUploader + def local? + # Must to be implemented, Check the uploader's storage types + file_store == ObjectStorage::Store::LOCAL + end + def self.replicables_for_geo_node # Should be implemented. The idea of the method is to restrict # the set of synced items depending on synchronization settings @@ -206,10 +219,30 @@ For example, to add support for files referenced by a `Widget` model with a def carrierwave_uploader model_record.file end + + # Change this to `true` to release replication of this model. Then remove + # this override in the next release. + # The feature flag follows the format `geo_#{replicable_name}_replication`, + # so here it would be `geo_widget_replication` + def self.replication_enabled_by_default? + false + end end end ``` +1. Add this replicator class to the method `replicator_classes` in +`ee/lib/gitlab/geo.rb`: + + ```ruby + def self.replicator_classes + classes = [::Geo::PackageFileReplicator, + ::Geo::WidgetReplicator] + + classes.select(&:enabled?) + end + ``` + 1. Create `ee/spec/replicators/geo/widget_replicator_spec.rb` and perform the setup necessary to define the `model_record` variable for the shared examples. @@ -226,13 +259,15 @@ For example, to add support for files referenced by a `Widget` model with a end ``` -1. Create the `widget_registry` table so Geo secondaries can track the sync and - verification state of each Widget's file: +1. Create the `widget_registry` table, with columns ordered according to [our guidelines](../ordering_table_columns.md) so Geo secondaries can track the sync and + verification state of each Widget's file. This migration belongs in `ee/db/geo/migrate`: ```ruby # frozen_string_literal: true class CreateWidgetRegistry < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + DOWNTIME = false disable_ddl_transaction! @@ -244,12 +279,12 @@ For example, to add support for files referenced by a `Widget` model with a t.integer :widget_id, null: false t.integer :state, default: 0, null: false, limit: 2 t.integer :retry_count, default: 0, limit: 2 - t.text :last_sync_failure t.datetime_with_timezone :retry_at t.datetime_with_timezone :last_synced_at t.datetime_with_timezone :created_at, null: false + t.text :last_sync_failure - t.index :widget_id + t.index :widget_id, name: :index_widget_registry_on_widget_id t.index :retry_at t.index :state end @@ -286,6 +321,8 @@ For example, to add support for files referenced by a `Widget` model with a 1. Update `REGISTRY_CLASSES` in `ee/app/workers/geo/secondary/registry_consistency_worker.rb`. +1. Add `widget_registry` to `ActiveSupport::Inflector.inflections` in `config/initializers_before_autoloader/000_inflections.rb`. + 1. Create `ee/spec/factories/geo/widget_registry.rb`: ```ruby @@ -343,8 +380,13 @@ Widgets should now be replicated by Geo! #### Verification -1. Add verification state fields to the `widgets` table so the Geo primary can - track verification state: +There are two ways to add verification related fields so that the Geo primary +can track verification state: + +##### Option 1: Add verification state fields to the existing `widgets` table itself + +1. Add a migration to add columns ordered according to [our guidelines](../ordering_table_columns.md) +for verification state to the widgets table: ```ruby # frozen_string_literal: true @@ -353,17 +395,39 @@ Widgets should now be replicated by Geo! DOWNTIME = false def change - add_column :widgets, :verification_retry_at, :datetime_with_timezone - add_column :widgets, :verified_at, :datetime_with_timezone - add_column :widgets, :verification_checksum, :binary, using: 'verification_checksum::bytea' - add_column :widgets, :verification_failure, :string - add_column :widgets, :verification_retry_count, :integer + change_table(:widgets) do |t| + t.integer :verification_retry_count, limit: 2 + t.column :verification_retry_at, :datetime_with_timezone + t.column :verified_at, :datetime_with_timezone + t.binary :verification_checksum, using: 'verification_checksum::bytea' + + # rubocop:disable Migration/AddLimitToTextColumns + t.text :verification_failure + # rubocop:enable Migration/AddLimitToTextColumns + end + end + end + ``` + +1. Adding a `text` column also [requires](../database/strings_and_the_text_data_type.md#add-a-text-column-to-an-existing-table) + setting a limit: + + ```ruby + # frozen_string_literal: true + + class AddVerificationFailureLimitToWidgets < ActiveRecord::Migration[6.0] + DOWNTIME = false + + disable_ddl_transaction! + + def change + add_text_limit :widgets, :verification_failure, 255 end end ``` 1. Add a partial index on `verification_failure` and `verification_checksum` to ensure - re-verification can be performed efficiently: + re-verification can be performed efficiently. Add a migration in `ee/db/geo/migrate/`: ```ruby # frozen_string_literal: true @@ -387,6 +451,65 @@ Widgets should now be replicated by Geo! end ``` +##### Option 2: Create a separate `widget_states` table with verification state fields + +1. Add a migration in `ee/db/geo/migrate/` to create a `widget_states` table and add a + partial index on `verification_failure` and `verification_checksum` to ensure + re-verification can be performed efficiently. Order the columns according to [our guidelines](../ordering_table_columns.md): + + ```ruby + # frozen_string_literal: true + + class CreateWidgetStates < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + unless table_exists?(:widget_states) + with_lock_retries do + create_table :widget_states, id: false do |t| + t.references :widget, primary_key: true, null: false, foreign_key: { on_delete: :cascade } + t.datetime_with_timezone :verification_retry_at + t.datetime_with_timezone :verified_at + t.integer :verification_retry_count, limit: 2 + t.binary :verification_checksum, using: 'verification_checksum::bytea' + t.text :verification_failure + + t.index :verification_failure, where: "(verification_failure IS NOT NULL)", name: "widgets_verification_failure_partial" + t.index :verification_checksum, where: "(verification_checksum IS NOT NULL)", name: "widgets_verification_checksum_partial" + end + end + end + + add_text_limit :widget_states, :verification_failure, 255 + end + + def down + drop_table :widget_states + end + end + ``` + +1. Add the following lines to the `widget` model: + + ```ruby + class Widget < ApplicationRecord + ... + has_one :widget_state, inverse_of: :widget + + delegate :verification_retry_at, :verification_retry_at=, + :verified_at, :verified_at=, + :verification_checksum, :verification_checksum=, + :verification_failure, :verification_failure=, + :verification_retry_count, :verification_retry_count=, + to: :widget_state + ... + end + ``` + To do: Add verification on secondaries. This should be done as part of [Geo: Self Service Framework - First Implementation for Package File verification](https://gitlab.com/groups/gitlab-org/-/epics/1817) diff --git a/doc/development/go_guide/index.md b/doc/development/go_guide/index.md index 779dcd9b7a2..6954b2bd6dc 100644 --- a/doc/development/go_guide/index.md +++ b/doc/development/go_guide/index.md @@ -108,13 +108,13 @@ lint: - '[ -e .golangci.yml ] || cp /golangci/.golangci.yml .' # Write the code coverage report to gl-code-quality-report.json # and print linting issues to stdout in the format: path/to/file:line description - - golangci-lint run --out-format code-climate | tee gl-code-quality-report.json | jq -r '.[] | "\(.location.path):\(.location.lines.begin) \(.description)"' + # remove `--issues-exit-code 0` or set to non-zero to fail the job if linting issues are detected + - golangci-lint run --issues-exit-code 0 --out-format code-climate | tee gl-code-quality-report.json | jq -r '.[] | "\(.location.path):\(.location.lines.begin) \(.description)"' artifacts: reports: codequality: gl-code-quality-report.json paths: - gl-code-quality-report.json - allow_failure: true ``` Including a `.golangci.yml` in the root directory of the project allows for diff --git a/doc/development/gotchas.md b/doc/development/gotchas.md index 34fe3f11489..6dff9deb59d 100644 --- a/doc/development/gotchas.md +++ b/doc/development/gotchas.md @@ -238,3 +238,34 @@ This problem will disappear as soon as we upgrade to Rails 6 and use the Zeitwer - Rails Guides: [Autoloading and Reloading Constants (Classic Mode)](https://guides.rubyonrails.org/autoloading_and_reloading_constants_classic_mode.html) - Ruby Constant lookup: [Everything you ever wanted to know about constant lookup in Ruby](http://cirw.in/blog/constant-lookup) - Rails 6 and Zeitwerk autoloader: [Understanding Zeitwerk in Rails 6](https://medium.com/cedarcode/understanding-zeitwerk-in-rails-6-f168a9f09a1f) + +## Storing assets that do not require pre-compiling + +Assets that need to be served to the user are stored under the `app/assets` directory, which is later pre-compiled and placed in the `public/` directory. + +However, you cannot access the content of any file from within `app/assets` from the application code, as we do not include that folder in production installations as a [space saving measure](https://gitlab.com/gitlab-org/omnibus-gitlab/-/commit/ca049f990b223f5e1e412830510a7516222810be). + +```ruby +support_bot = User.support_bot + +# accessing a file from the `app/assets` folder +support_bot.avatar = Rails.root.join('app', 'assets', 'images', 'bot_avatars', 'support_bot.png').open + +support_bot.save! +``` + +While the code above works in local environments, it errors out in production installations as the `app/assets` folder is not included. + +### Solution + +The alternative is the `lib/assets` folder. Use it if you need to add assets (like images) to the repo that meet the following conditions: + +- The assets do not need to be directly served to the user (and hence need not be pre-compiled). +- The assets do need to be accessed via application code. + +In short: + +Use `app/assets` for storing any asset that needs to be precompiled and served to the end user. +Use `lib/assets` for storing any asset that does not need to be served to the end user directly, but is still required to be accessed by the application code. + +MR for reference: [!37671](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/37671) diff --git a/doc/development/graphql_guide/index.md b/doc/development/graphql_guide/index.md new file mode 100644 index 00000000000..62650f7cd3f --- /dev/null +++ b/doc/development/graphql_guide/index.md @@ -0,0 +1,13 @@ +# GraphQL development guidelines + +This guide contains all the information to successfully contribute to GitLab's +GraphQL API. This is a living document, and we welcome contributions, +feedback, and suggestions. + +## Resources + +- [GraphQL API development style guide](../api_graphql_styleguide.md): development style guide for + GraphQL. +- [GraphQL API documentation style guide](../documentation/styleguide.md#graphql-api): documentation + style guide for GraphQL. +- [GraphQL API](../../api/graphql/index.md): user documentation for the GitLab GraphQL API. diff --git a/doc/development/i18n/externalization.md b/doc/development/i18n/externalization.md index bce95dfc24e..1374ca92256 100644 --- a/doc/development/i18n/externalization.md +++ b/doc/development/i18n/externalization.md @@ -196,7 +196,7 @@ For example use `%{created_at}` in Ruby but `%{createdAt}` in JavaScript. Make s // => 'This is <strong><script>alert('evil')</script></strong>' // OK: - sprintf(__('This is %{value}'), { value: `<strong>${escape(someDynamicValue)}</strong>`, false); + sprintf(__('This is %{value}'), { value: `<strong>${escape(someDynamicValue)}</strong>` }, false); // => 'This is <strong><script>alert('evil')</script></strong>' ``` @@ -296,6 +296,74 @@ Namespaces should be PascalCase. Note: The namespace should be removed from the translation. See the [translation guidelines for more details](translation.md#namespaced-strings). +### HTML + +We no longer include HTML directly in the strings that are submitted for translation. This is for a couple of reasons: + +1. It introduces a chance for the translated string to accidentally include invalid HTML. +1. It introduces a security risk where translated strings become an attack vector for XSS, as noted by the + [Open Web Application Security Project (OWASP)](https://owasp.org/www-community/attacks/xss/). + +To include formatting in the translated string, we can do the following: + +- In Ruby/HAML: + + ```ruby + html_escape(_('Some %{strongOpen}bold%{strongClose} text.')) % { strongOpen: '<strong>'.html_safe, strongClose: '</strong>'.html_safe } + + # => 'Some <strong>bold</strong> text.' + ``` + +- In JavaScript: + + ```javascript + sprintf(__('Some %{strongOpen}bold%{strongClose} text.'), { strongOpen: '<strong>', strongClose: '</strong>'}, false); + + // => 'Some <strong>bold</strong> text.' + ``` + +- In Vue + + See the section on [interpolation](#interpolation). + +When [this translation helper issue](https://gitlab.com/gitlab-org/gitlab/-/issues/217935) is complete, we'll update the +process of including formatting in translated strings. + +#### Including Angle Brackets + +If a string contains angles brackets (`<`/`>`) that are not used for HTML, it will still be flagged by the +`rake gettext:lint` linter. +To avoid this error, use the applicable HTML entity code (`<` or `>`) instead: + +- In Ruby/HAML: + + ```ruby + html_escape_once(_('In < 1 hour')).html_safe + + # => 'In < 1 hour' + ``` + +- In JavaScript: + + ```javascript + import sanitize from 'sanitize-html'; + + const i18n = { LESS_THAN_ONE_HOUR: sanitize(__('In < 1 hours'), { allowedTags: [] }) }; + + // ... using the string + element.innerHTML = i18n.LESS_THAN_ONE_HOUR; + + // => 'In < 1 hour' + ``` + +- In Vue: + + ```vue + <gl-sprintf :message="s__('In < 1 hours')"/> + + // => 'In < 1 hour' + ``` + ### Dates / times - In JavaScript: @@ -555,6 +623,7 @@ The linter will take the following into account: - There should be no variables used in a translation that aren't in the message ID - Errors during translation. +- Presence of angle brackets (`<` or `>`) The errors are grouped per file, and per message ID: @@ -567,7 +636,7 @@ Errors in `locale/zh_HK/gitlab.po`: Syntax error in msgstr Syntax error in message_line There should be only whitespace until the end of line after the double quote character of a message text. - Parsing result before error: '{:msgid=>["", "You are going to remove %{project_name_with_namespace}.\\n", "Removed project CANNOT be restored!\\n", "Are you ABSOLUTELY sure?"]}' + Parsing result before error: '{:msgid=>["", "You are going to delete %{project_name_with_namespace}.\\n", "Deleted projects CANNOT be restored!\\n", "Are you ABSOLUTELY sure?"]}' SimplePoParser filtered backtrace: SimplePoParser::ParserError Errors in `locale/zh_TW/gitlab.po`: 1 pipeline @@ -581,6 +650,10 @@ aren't in the message with ID `1 pipeline`. ## Adding a new language +NOTE: **Note:** +[Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/221012) in GitLab 13.3: +Languages with less than 2% of translations won't be available in the UI. + Let's suppose you want to add translations for a new language, let's say French. 1. The first step is to register the new language in `lib/gitlab/i18n.rb`: diff --git a/doc/development/i18n/merging_translations.md b/doc/development/i18n/merging_translations.md index e0fcaddda4c..8b3357c41d3 100644 --- a/doc/development/i18n/merging_translations.md +++ b/doc/development/i18n/merging_translations.md @@ -25,14 +25,15 @@ suggesting to automate this process. Disapproving will exclude the invalid translation, the merge request will be updated within a few minutes. +If the translation has failed validation due to angle brackets `<` or `>` +it should be disapproved on CrowdIn as our strings should be +using [variables](externalization.md#html) for HTML instead. + It might be handy to pause the integration on the CrowdIn side for a little while so translations don't keep coming. This can be done by clicking `Pause sync` on the [CrowdIn integration settings page](https://translate.gitlab.com/project/gitlab-ee/settings#integration). -When all failures are resolved, the translations need to be double -checked once more as discussed in [confidential issue](../../user/project/issues/confidential_issues.md) `https://gitlab.com/gitlab-org/gitlab/-/issues/19485`. - ## Merging translations When all translations are found good and pipelines pass the @@ -56,7 +57,7 @@ have been fixed on master. ## Recreate the GitLab integration in CrowdIn -NOTE: ***Note:** +NOTE: **Note:** These instructions work only for GitLab Team Members. If for some reason the GitLab integration in CrowdIn does not exist, it can be diff --git a/doc/development/i18n/proofreader.md b/doc/development/i18n/proofreader.md index 935a171f34c..9d2997379c1 100644 --- a/doc/development/i18n/proofreader.md +++ b/doc/development/i18n/proofreader.md @@ -115,9 +115,10 @@ are very appreciative of the work done by translators and proofreaders! ## Become a proofreader -> **Note:** Before requesting Proofreader permissions in CrowdIn please make -> sure that you have a history of contributing translations to the GitLab -> project. +NOTE: **Note:** +Before requesting Proofreader permissions in CrowdIn please make +sure that you have a history of contributing translations to the GitLab +project. 1. Contribute translations to GitLab. See instructions for [translating GitLab](translation.md). diff --git a/doc/development/img/bullet_v13_0.png b/doc/development/img/bullet_v13_0.png Binary files differindex 04b476db581..e185bdef76d 100644 --- a/doc/development/img/bullet_v13_0.png +++ b/doc/development/img/bullet_v13_0.png diff --git a/doc/development/img/deployment_pipeline_v13_3.png b/doc/development/img/deployment_pipeline_v13_3.png Binary files differnew file mode 100644 index 00000000000..e4fd470fc3d --- /dev/null +++ b/doc/development/img/deployment_pipeline_v13_3.png diff --git a/doc/development/img/telemetry_system_overview.png b/doc/development/img/telemetry_system_overview.png Binary files differindex 1667039e8cd..f2e6b300e94 100644 --- a/doc/development/img/telemetry_system_overview.png +++ b/doc/development/img/telemetry_system_overview.png diff --git a/doc/development/instrumentation.md b/doc/development/instrumentation.md index ee1aab1456e..e420ae0c54f 100644 --- a/doc/development/instrumentation.md +++ b/doc/development/instrumentation.md @@ -1,3 +1,9 @@ +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Instrumenting Ruby code [GitLab Performance Monitoring](../administration/monitoring/performance/index.md) allows instrumenting of both methods and custom diff --git a/doc/development/integrations/elasticsearch_for_paid_tiers_on_gitlabcom.md b/doc/development/integrations/elasticsearch_for_paid_tiers_on_gitlabcom.md deleted file mode 100644 index 8289be47253..00000000000 --- a/doc/development/integrations/elasticsearch_for_paid_tiers_on_gitlabcom.md +++ /dev/null @@ -1,28 +0,0 @@ -# Elasticsearch for paid tiers on GitLab.com - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/220246) in GitLab 13.2 -> - It's deployed behind a feature flag, disabled by default. -> - It's disabled on GitLab.com. -> - It's not recommended for use in GitLab self-managed instances. - -This document describes how to enable Elasticsearch with GitLab for all paid tiers on GitLab.com. Once enabled, -all paid tiers will have access to the [Advanced Global Search feature](../../integration/elasticsearch.md) on GitLab.com. - -## Enable or disable Elasticsearch for all paid tiers on GitLab.com - -Since we're still in the process of rolling this out and want to control the timing this is behind a feature flag -which defaults to off. - -To enable it: - -```ruby -# Instance-wide -Feature.enable(:elasticsearch_index_only_paid_groups) -``` - -To disable it: - -```ruby -# Instance-wide -Feature.disable(:elasticsearch_index_only_paid_groups) -``` diff --git a/doc/development/integrations/example_vuln.png b/doc/development/integrations/example_vuln.png Binary files differindex f7a3c8b38f2..bbfeb3c805f 100644 --- a/doc/development/integrations/example_vuln.png +++ b/doc/development/integrations/example_vuln.png diff --git a/doc/development/integrations/jenkins.md b/doc/development/integrations/jenkins.md index f2bc6532dde..5d3c869d922 100644 --- a/doc/development/integrations/jenkins.md +++ b/doc/development/integrations/jenkins.md @@ -16,7 +16,7 @@ brew services start jenkins GitLab does not allow requests to localhost or the local network by default. When running Jenkins on your local machine, you need to enable local access. 1. Log into your GitLab instance as an admin. -1. Go to **{admin}** **Admin Area > Settings > Network**. +1. Go to **Admin Area > Settings > Network**. 1. Expand **Outbound requests** and check the following checkboxes: - **Allow requests to the local network from web hooks and services** diff --git a/doc/development/integrations/jira_connect.md b/doc/development/integrations/jira_connect.md index 374cc976caa..1f9b03075f0 100644 --- a/doc/development/integrations/jira_connect.md +++ b/doc/development/integrations/jira_connect.md @@ -11,7 +11,7 @@ The following are required to install and test the app: For the app to work, Jira Cloud should be able to connect to the GitLab instance through the internet. To easily expose your local development environment, you can use tools like - [serveo](https://medium.com/testautomator/how-to-forward-my-local-port-to-public-using-serveo-4979f352a3bf) + [serveo](https://medium.com/automationmaster/how-to-forward-my-local-port-to-public-using-serveo-4979f352a3bf) or [ngrok](https://ngrok.com). These also take care of SSL for you because Jira requires all connections to the app host to be over SSL. diff --git a/doc/development/integrations/secure.md b/doc/development/integrations/secure.md index 22da57400e0..21076fa681f 100644 --- a/doc/development/integrations/secure.md +++ b/doc/development/integrations/secure.md @@ -358,7 +358,7 @@ It is recommended to reuse the identifiers the GitLab scanners already define: | [CVE](https://cve.mitre.org/cve/) | `cve` | CVE-2019-10086 | | [CWE](https://cwe.mitre.org/data/index.html) | `cwe` | CWE-1026 | | [OSVD](https://cve.mitre.org/data/refs/refmap/source-OSVDB.html) | `osvdb` | OSVDB-113928 | -| [USN](https://usn.ubuntu.com/) | `usn` | USN-4234-1 | +| [USN](https://ubuntu.com/security/notices) | `usn` | USN-4234-1 | | [WASC](http://projects.webappsec.org/Threat-Classification-Reference-Grid) | `wasc` | WASC-19 | | [RHSA](https://access.redhat.com/errata/#/) | `rhsa` | RHSA-2020:0111 | | [ELSA](https://linux.oracle.com/security/) | `elsa` | ELSA-2020-0085 | diff --git a/doc/development/internal_api.md b/doc/development/internal_api.md index d220a2d46fb..c51bf66be46 100644 --- a/doc/development/internal_api.md +++ b/doc/development/internal_api.md @@ -1,3 +1,10 @@ +--- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference, api +--- + # Internal API The internal API is used by different GitLab components, it can not be @@ -24,10 +31,11 @@ authentication. ## Git Authentication -This is called by Gitaly and GitLab-shell to check access to a +This is called by [Gitaly](https://gitlab.com/gitlab-org/gitaly) and +[GitLab Shell](https://gitlab.com/gitlab-org/gitlab-shell) to check access to a repository. -When called from GitLab-shell no changes are passed and the internal +When called from GitLab Shell no changes are passed and the internal API replies with the information needed to pass the request on to Gitaly. @@ -40,13 +48,13 @@ POST /internal/allowed | Attribute | Type | Required | Description | |:----------|:-------|:---------|:------------| -| `key_id` | string | no | ID of the SSH-key used to connect to GitLab-shell | -| `username` | string | no | Username from the certificate used to connect to GitLab-Shell | +| `key_id` | string | no | ID of the SSH-key used to connect to GitLab Shell | +| `username` | string | no | Username from the certificate used to connect to GitLab Shell | | `project` | string | no (if `gl_repository` is passed) | Path to the project | | `gl_repository` | string | no (if `project` is passed) | Repository identifier (e.g. `project-7`) | | `protocol` | string | yes | SSH when called from GitLab-shell, HTTP or SSH when called from Gitaly | | `action` | string | yes | Git command being run (`git-upload-pack`, `git-receive-pack`, `git-upload-archive`) | -| `changes` | string | yes | `<oldrev> <newrev> <refname>` when called from Gitaly, The magic string `_any` when called from GitLab Shell | +| `changes` | string | yes | `<oldrev> <newrev> <refname>` when called from Gitaly, the magic string `_any` when called from GitLab Shell | | `check_ip` | string | no | IP address from which call to GitLab Shell was made | Example request: @@ -84,17 +92,17 @@ Example response: ### Known consumers - Gitaly -- GitLab-shell +- GitLab Shell ## LFS Authentication -This is the endpoint that gets called from GitLab-shell to provide +This is the endpoint that gets called from GitLab Shell to provide information for LFS clients when the repository is accessed over SSH. | Attribute | Type | Required | Description | |:----------|:-------|:---------|:------------| -| `key_id` | string | no | ID of the SSH-key used to connect to GitLab-shell | -| `username`| string | no | Username from the certificate used to connect to GitLab-Shell | +| `key_id` | string | no | ID of the SSH-key used to connect to GitLab Shell | +| `username`| string | no | Username from the certificate used to connect to GitLab Shell | | `project` | string | no | Path to the project | Example request: @@ -114,17 +122,17 @@ curl --request POST --header "Gitlab-Shared-Secret: <Base64 encoded token>" --da ### Known consumers -- GitLab-shell +- GitLab Shell ## Authorized Keys Check -This endpoint is called by the GitLab-shell authorized keys +This endpoint is called by the GitLab Shell authorized keys check. Which is called by OpenSSH for [fast SSH key lookup](../administration/operations/fast_ssh_key_lookup.md). | Attribute | Type | Required | Description | |:----------|:-------|:---------|:------------| -| `key` | string | yes | SSH key as passed by OpenSSH to GitLab-shell | +| `key` | string | yes | SSH key as passed by OpenSSH to GitLab Shell | ```plaintext GET /internal/authorized_keys @@ -149,7 +157,7 @@ Example response: ### Known consumers -- GitLab-shell +- GitLab Shell ## Get user for user ID or key @@ -159,7 +167,7 @@ discovers the user associated with an SSH key. | Attribute | Type | Required | Description | |:----------|:-------|:---------|:------------| | `key_id` | integer | no | The ID of the SSH key used as found in the authorized-keys file or through the `/authorized_keys` check | -| `username` | string | no | Username of the user being looked up, used by GitLab-shell when authenticating using a certificate | +| `username` | string | no | Username of the user being looked up, used by GitLab Shell when authenticating using a certificate | ```plaintext GET /internal/discover @@ -183,12 +191,12 @@ Example response: ### Known consumers -- GitLab-shell +- GitLab Shell ## Instance information This gets some generic information about the instance. This is used -by Geo nodes to get information about each other +by Geo nodes to get information about each other. ```plaintext GET /internal/check @@ -214,12 +222,12 @@ Example response: ### Known consumers - GitLab Geo -- GitLab-shell's `bin/check` +- GitLab Shell's `bin/check` ## Get new 2FA recovery codes using an SSH key -This is called from GitLab-shell and allows users to get new 2FA -recovery codes based on their SSH key +This is called from GitLab Shell and allows users to get new 2FA +recovery codes based on their SSH key. | Attribute | Type | Required | Description | |:----------|:-------|:---------|:------------| @@ -258,7 +266,45 @@ Example response: ### Known consumers -- GitLab-shell +- GitLab Shell + +## Get new personal access-token + +This is called from GitLab Shell and allows users to generate a new +personal access token. + +| Attribute | Type | Required | Description | +|:----------|:-------|:---------|:------------| +| `name` | string | yes | The name of the new token | +| `scopes` | string array | yes | The authorization scopes for the new token, these must be valid token scopes | +| `expires_at` | string | no | The expiry date for the new token | +| `key_id` | integer | no | The ID of the SSH key used as found in the authorized-keys file or through the `/authorized_keys` check | +| `user_id` | integer | no | User\_id for which to generate the new token | + +```plaintext +POST /internal/personal_access_token +``` + +Example request: + +```shell +curl --request POST --header "Gitlab-Shared-Secret: <Base64 encoded secret>" --data "user_id=29&name=mytokenname&scopes[]=read_user&scopes[]=read_repository&expires_at=2020-07-24" http://localhost:3001/api/v4/internal/personal_access_token +``` + +Example response: + +```json +{ + "success": true, + "token": "Hf_79B288hRv_3-TSD1R", + "scopes": ["read_user","read_repository"], + "expires_at": "2020-07-24" +} +``` + +### Known consumers + +- GitLab Shell ## Incrementing counter on pre-receive diff --git a/doc/development/issuable-like-models.md b/doc/development/issuable-like-models.md index d252735dbd8..9029886c334 100644 --- a/doc/development/issuable-like-models.md +++ b/doc/development/issuable-like-models.md @@ -1,3 +1,8 @@ +--- +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- # Issuable-like Rails models utilities GitLab Rails codebase contains several models that hold common functionality and behave similarly to @@ -9,9 +14,9 @@ This guide accumulates guidelines on working with such Rails models. ## Important text fields -There are max length constraints for the most important text fields for `Issuable`s: +There are maximum length constraints for the most important text fields for issuables: -- `title`: 255 chars -- `title_html`: 800 chars +- `title`: 255 characters +- `title_html`: 800 characters - `description`: 1 megabyte - `description_html`: 5 megabytes diff --git a/doc/development/issue_types.md b/doc/development/issue_types.md index 028d42b27fc..416aa65b13f 100644 --- a/doc/development/issue_types.md +++ b/doc/development/issue_types.md @@ -1,3 +1,9 @@ +--- +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Issue Types Sometimes when a new resource type is added it's not clear if it should be only an diff --git a/doc/development/kubernetes.md b/doc/development/kubernetes.md index 9e0e686f447..64089d51c7b 100644 --- a/doc/development/kubernetes.md +++ b/doc/development/kubernetes.md @@ -1,3 +1,9 @@ +--- +stage: Configure +group: Configure +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Kubernetes integration - development guidelines This document provides various guidelines when developing for GitLab's diff --git a/doc/development/licensed_feature_availability.md b/doc/development/licensed_feature_availability.md index 4f962b6f5e2..cf479544eea 100644 --- a/doc/development/licensed_feature_availability.md +++ b/doc/development/licensed_feature_availability.md @@ -8,7 +8,7 @@ on-premise or GitLab.com plans and features. GitLab.com plans are persisted on user groups and namespaces, therefore, if you're adding a feature such as [Related issues](../user/project/issues/related_issues.md) or -[Service desk](../user/project/service_desk.md), +[Service Desk](../user/project/service_desk.md), it should be restricted on namespace scope. 1. Add the feature symbol on `EES_FEATURES`, `EEP_FEATURES` or `EEU_FEATURES` constants in diff --git a/doc/development/logging.md b/doc/development/logging.md index ccd4adf7cef..474a500da61 100644 --- a/doc/development/logging.md +++ b/doc/development/logging.md @@ -1,3 +1,9 @@ +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # GitLab Developers Guide to Logging [GitLab Logs](../administration/logs.md) play a critical role for both @@ -281,7 +287,7 @@ method or variable shouldn't be evaluated right away) See our [HOWTO: Use Sidekiq metadata logs](https://www.youtube.com/watch?v=_wDllvO_IY0) for further knowledge on creating visualizations in Kibana. -**Note:** +NOTE: **Note:** The fields of the context are currently only logged for Sidekiq jobs triggered through web requests. See the [follow-up work](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/68) diff --git a/doc/development/merge_request_performance_guidelines.md b/doc/development/merge_request_performance_guidelines.md index ed6d08a9894..b3829a82d59 100644 --- a/doc/development/merge_request_performance_guidelines.md +++ b/doc/development/merge_request_performance_guidelines.md @@ -211,7 +211,7 @@ For keeping transaction as minimal as possible, please consider using `AfterComm module or `after_commit` AR hook. Here is [an example](https://gitlab.com/gitlab-org/gitlab/-/issues/36154#note_247228859) -that one request to Gitaly instance during transaction triggered a P1 issue. +that one request to Gitaly instance during transaction triggered a P::1 issue. ## Eager Loading diff --git a/doc/development/migration_style_guide.md b/doc/development/migration_style_guide.md index d1f956f957e..ebaceac6d17 100644 --- a/doc/development/migration_style_guide.md +++ b/doc/development/migration_style_guide.md @@ -30,7 +30,7 @@ versions. If needed copy-paste GitLab code into the migration to make it forward compatible. For GitLab.com, please take into consideration that regular migrations (under `db/migrate`) -are run before [Canary is deployed](https://about.gitlab.com/handbook/engineering/infrastructure/library/canary/#configuration-and-deployment), +are run before [Canary is deployed](https://gitlab.com/gitlab-com/gl-infra/readiness/-/tree/master/library/canary/#configuration-and-deployment), and post-deployment migrations (`db/post_migrate`) are run after the deployment to production has finished. ## Schema Changes @@ -380,7 +380,8 @@ Example changes: - `change_column_default` - `create_table` / `drop_table` -**Note:** `with_lock_retries` method **cannot** be used within the `change` method, you must manually define the `up` and `down` methods to make the migration reversible. +NOTE: **Note:** +`with_lock_retries` method **cannot** be used within the `change` method, you must manually define the `up` and `down` methods to make the migration reversible. ### How the helper method works @@ -440,7 +441,8 @@ the `with_multiple_threads` block, instead of re-using the global connection pool. This ensures each thread has its own connection object, and won't time out when trying to obtain one. -**NOTE:** PostgreSQL has a maximum amount of connections that it allows. This +NOTE: **Note:** +PostgreSQL has a maximum amount of connections that it allows. This limit can vary from installation to installation. As a result, it's recommended you do not use more than 32 threads in a single migration. Usually, 4-8 threads should be more than enough. @@ -873,14 +875,7 @@ end When doing so be sure to explicitly set the model's table name, so it's not derived from the class name or namespace. -Finally, make sure that `reset_column_information` is run in the `up` method of -the migration for all local Models that update the database. - -The reason for that is that all migration classes are loaded at the beginning -(when `db:migrate` starts), so they can get out of sync with the table schema -they map to in case another migration updates that schema. That makes the data -migration fail when trying to insert or make updates to the underlying table, -as the new columns are reported as `unknown attribute` by `ActiveRecord`. +Be aware of the limitations [when using models in migrations](#using-models-in-migrations-discouraged). ### Renaming reserved paths @@ -906,3 +901,78 @@ _namespaces_ that have a `project_id`. The `path` column for these rows will be renamed to their previous value followed by an integer. For example: `users` would turn into `users0` + +## Using models in migrations (discouraged) + +The use of models in migrations is generally discouraged. As such models are +[contraindicated for background migrations](background_migrations.md#isolation), +the model needs to be declared in the migration. + +If using a model in the migrations, you should first +[clear the column cache](https://api.rubyonrails.org/classes/ActiveRecord/ModelSchema/ClassMethods.html#method-i-reset_column_information) +using `reset_column_information`. + +This avoids problems where a column that you are using was altered and cached +in a previous migration. + +### Example: Add a column `my_column` to the users table + +NOTE: **Note:** +It is important not to leave out the `User.reset_column_information` command, in order to ensure that the old schema is dropped from the cache and ActiveRecord loads the updated schema information. + +```ruby +class AddAndSeedMyColumn < ActiveRecord::Migration[6.0] + class User < ActiveRecord::Base + self.table_name = 'users' + end + + def up + User.count # Any ActiveRecord calls on the model that caches the column information. + + add_column :users, :my_column, :integer, default: 1 + + User.reset_column_information # The old schema is dropped from the cache. + User.find_each do |user| + user.my_column = 42 if some_condition # ActiveRecord sees the correct schema here. + user.save! + end + end +end +``` + +The underlying table is modified and then accessed via ActiveRecord. + +Note that this also needs to be used if the table is modified in a previous, different migration, +if both migrations are run in the same `db:migrate` process. + +This results in the following. Note the inclusion of `my_column`: + +```shell +== 20200705232821 AddAndSeedMyColumn: migrating ============================== +D, [2020-07-06T00:37:12.483876 #130101] DEBUG -- : (0.2ms) BEGIN +D, [2020-07-06T00:37:12.521660 #130101] DEBUG -- : (0.4ms) SELECT COUNT(*) FROM "user" +-- add_column(:users, :my_column, :integer, {:default=>1}) +D, [2020-07-06T00:37:12.523309 #130101] DEBUG -- : (0.8ms) ALTER TABLE "users" ADD "my_column" integer DEFAULT 1 + -> 0.0016s +D, [2020-07-06T00:37:12.650641 #130101] DEBUG -- : AddAndSeedMyColumn::User Load (0.7ms) SELECT "users".* FROM "users" ORDER BY "users"."id" ASC LIMIT $1 [["LIMIT", 1000]] +D, [2020-07-18T00:41:26.851769 #459802] DEBUG -- : AddAndSeedMyColumn::User Update (1.1ms) UPDATE "users" SET "my_column" = $1, "updated_at" = $2 WHERE "users"."id" = $3 [["my_column", 42], ["updated_at", "2020-07-17 23:41:26.849044"], ["id", 1]] +D, [2020-07-06T00:37:12.653648 #130101] DEBUG -- : ↳ config/initializers/config_initializers_active_record_locking.rb:13:in `_update_row' +== 20200705232821 AddAndSeedMyColumn: migrated (0.1706s) ===================== +``` + +If you skip clearing the schema cache (`User.reset_column_information`), the column is not +used by ActiveRecord and the intended changes are not made, leading to the result below, +where `my_column` is missing from the query. + +```shell +== 20200705232821 AddAndSeedMyColumn: migrating ============================== +D, [2020-07-06T00:37:12.483876 #130101] DEBUG -- : (0.2ms) BEGIN +D, [2020-07-06T00:37:12.521660 #130101] DEBUG -- : (0.4ms) SELECT COUNT(*) FROM "user" +-- add_column(:users, :my_column, :integer, {:default=>1}) +D, [2020-07-06T00:37:12.523309 #130101] DEBUG -- : (0.8ms) ALTER TABLE "users" ADD "my_column" integer DEFAULT 1 + -> 0.0016s +D, [2020-07-06T00:37:12.650641 #130101] DEBUG -- : AddAndSeedMyColumn::User Load (0.7ms) SELECT "users".* FROM "users" ORDER BY "users"."id" ASC LIMIT $1 [["LIMIT", 1000]] +D, [2020-07-06T00:37:12.653459 #130101] DEBUG -- : AddAndSeedMyColumn::User Update (0.5ms) UPDATE "users" SET "updated_at" = $1 WHERE "users"."id" = $2 [["updated_at", "2020-07-05 23:37:12.652297"], ["id", 1]] +D, [2020-07-06T00:37:12.653648 #130101] DEBUG -- : ↳ config/initializers/config_initializers_active_record_locking.rb:13:in `_update_row' +== 20200705232821 AddAndSeedMyColumn: migrated (0.1706s) ===================== +``` diff --git a/doc/development/multi_version_compatibility.md b/doc/development/multi_version_compatibility.md index aedd5c1ffb7..5ca43b9b818 100644 --- a/doc/development/multi_version_compatibility.md +++ b/doc/development/multi_version_compatibility.md @@ -20,6 +20,121 @@ but AJAX requests to URLs (like the GraphQL endpoint) won't match the pattern. With this canary setup, we'd be in this mixed-versions state for an extended period of time until canary is promoted to production and post-deployment migrations run. +Also be aware that during a deployment to production, Web, API, and +Sidekiq nodes are updated in parallel, but they may finish at +different times. That means there may be a window of time when the +application code is not in sync across the whole fleet. Changes that +cut across Sidekiq, Web, and/or the API may [introduce unexpected +errors until the deployment is complete](#builds-failing-due-to-varying-deployment-times-across-node-types). + +One way to handle this is to use a feature flag that is disabled by +default. The feature flag can be enabled when the deployment is in a +consistent state. However, this method of synchronization doesn't +guarantee that customers with on-premise instances can [upgrade with +zero downtime](https://docs.gitlab.com/omnibus/update/#zero-downtime-updates) +since point releases bundle many changes together. Minimizing the time +between when versions are out of sync across the fleet may help mitigate +errors caused by upgrades. + +## Requirements for zero downtime upgrades + +One way to guarantee zero downtime upgrades for on-premise instances is following the +[expand and contract pattern](https://martinfowler.com/bliki/ParallelChange.html). + +This means that every breaking change is broken down in three phases: expand, migrate, and contract. + +1. **expand**: a breaking change is introduced keeping the software backward-compatible. +1. **migrate**: all consumers are updated to make use of the new implementation. +1. **contract**: backward compatibility is removed. + +Those three phases **must be part of different milestones**, to allow zero downtime upgrades. + +Depending on the support level for the feature, the contract phase could be delayed until the next major release. + +## Expand and contract examples + +Route changes, changing Sidekiq worker parameters, and database migrations are all perfect examples of a breaking change. +Let's see how we can handle them safely. + +### Route changes + +When changing routing we should pay attention to make sure a route generated from the new version can be served by the old one and vice versa. +As you can see in [an example later on this page](#some-links-to-issues-and-mrs-were-broken), not doing it can lead to an outage. +This type of change may look like an immediate switch between the two implementations. However, +especially with the canary stage, there is an extended period of time where both version of the code +coexists in production. + +1. **expand**: a new route is added, pointing to the same controller as the old one. But nothing in the application will generate links for the new routes. +1. **migrate**: now that every machine in the fleet can understand the new route, we can generate links with the new routing. +1. **contract**: the old route can be safely removed. (If the old route was likely to be widely shared, like the link to a repository file, we might want to add redirects and keep the old route for a longer period.) + +### Changing Sidekiq worker's parameters + +This topic is explained in detail in [Sidekiq Compatibility across Updates](sidekiq_style_guide.md#sidekiq-compatibility-across-updates). + +When we need to add a new parameter to a Sidekiq worker class, we can split this into the following steps: + +1. **expand**: the worker class adds a new parameter with a default value. +1. **migrate**: we add the new parameter to all the invocations of the worker. +1. **contract**: we remove the default value. + +At a first look, it may seem safe to bundle expand and migrate into a single milestone, but this will cause an outage if Puma restarts before Sidekiq. +Puma enqueues jobs with an extra parameter that the old Sidekiq cannot handle. + +### Database migrations + +The following graph is a simplified visual representation of a deployment, this will guide us in understanding how expand and contract is implemented in our migrations strategy. + +There's a special consideration here. Using our post-deployment migrations framework allows us to bundle all three phases into one milestone. + +```mermaid +gantt + title Deployment + dateFormat HH:mm + + section Deploy box + Run migrations :done, migr, after schemaA, 2m + Run post-deployment migrations :postmigr, after mcvn , 2m + + section Database + Schema A :done, schemaA, 00:00 , 1h + Schema B :crit, schemaB, after migr, 58m + Schema C. : schmeaC, after postmigr, 1h + + section Machine A + Version N :done, mavn, 00:00 , 75m + Version N+1 : after mavn, 105m + + section Machine B + Version N :done, mbvn, 00:00 , 105m + Version N+1 : mbdone, after mbvn, 75m + + section Machine C + Version N :done, mcvn, 00:00 , 2h + Version N+1 : mbcdone, after mcvn, 1h +``` + +If we look at this schema from a database point of view, we can see two deployments feed into a single GitLab deployment: + +1. from `Schema A` to `Schema B` +1. from `Schema B` to `Schema C` + +And these deployments align perfectly with application changes. + +1. At the beginning we have `Version N` on `Schema A`. +1. Then we have a _long_ transition periond with both `Version N` and `Version N+1` on `Schema B`. +1. When we only have `Version N+1` on `Schema B` the schema changes again. +1. Finally we have `Version N+1` on `Schema C`. + +With all those details in mind, let's imagine we need to replace a query, and this query has an index to support it. + +1. **expand**: this is the from `Schema A` to `Schema B` deployment. We add the new index, but the application will ignore it for now +1. **migrate**: this is the `Version N` to `Version N+1` application deployment. The new code is deployed, at this point in time only the new query will run. +1. **contract**: from `Schema B` to `Schema C` (post-deployment migration). Nothing uses the old index anymore, we can safely remove it. + +This is only an example. More complex migrations, especially when background migrations are needed will +still require more than one milestone. For details please refer to our [migration style guide](migration_style_guide.md). + ## Examples of previous incidents ### Some links to issues and MRs were broken @@ -60,3 +175,52 @@ We added a `NOT NULL` constraint to a column and marked it as a `NOT VALID` cons But even with that, this was still a problem because the old servers were still inserting new rows with null values. For more information, see [the relevant issue](https://gitlab.com/gitlab-com/gl-infra/production/-/issues/1944). + +### Downtime on release features between canary and production deployment + +To address the issue, we added a new column to an existing table with a `NOT NULL` constraint without +specifying a default value. In other words, this requires the application to set a value to the column. + +The older version of the application didn't set the `NOT NULL` constraint since the entity/concept didn't +exist before. + +The problem starts right after the canary deployment is complete. At that moment, +the database migration (to add the column) has successfully run and canary instance starts using +the new application code, hence QA was successful. Unfortunately, the production +instance still uses the older code, so it started failing to insert a new release entry. + +For more information, see [this issue related to the Releases API](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/64151). + +### Builds failing due to varying deployment times across node types + +In [one production issue](https://gitlab.com/gitlab-com/gl-infra/production/-/issues/2442), +CI builds that used the `parallel` keyword and depending on the +variable `CI_NODE_TOTAL` being an integer failed. This was caused because after a user pushed a commit: + +1. New code: Sidekiq created a new pipeline and new build. `build.options[:parallel]` is a `Hash`. +1. Old code: Runners requested a job from an API node that is running the previous version. +1. As a result, the [new code](https://gitlab.com/gitlab-org/gitlab/blob/42b82a9a3ac5a96f9152aad6cbc583c42b9fb082/app/models/concerns/ci/contextable.rb#L104) +was not run on the API server. The runner's request failed because the +older API server tried return the `CI_NODE_TOTAL` CI variable, but +instead of sending an integer value (e.g. 9), it sent a serialized +`Hash` value (`{:number=>9, :total=>9}`). + +If you look at the [deployment pipeline](https://ops.gitlab.net/gitlab-com/gl-infra/deployer/-/pipelines/202212), +you see all nodes were updated in parallel: + + + +However, even though the updated started around the same time, the completion time varied significantly: + +|Node type|Duration (min)| +|---------|--------------| +|API |54 | +|Sidekiq |21 | +|K8S |8 | + +Builds that used the `parallel` keyword and depended on `CI_NODE_TOTAL` +and `CI_NODE_INDEX` would fail during the time after Sidekiq was +updated. Since Kubernetes (K8S) also runs Sidekiq pods, the window could +have been as long as 46 minutes or as short as 33 minutes. Either way, +having a feature flag to turn on after the deployment finished would +prevent this from happening. diff --git a/doc/development/new_fe_guide/development/performance.md b/doc/development/new_fe_guide/development/performance.md index 7b58a80576a..ad6fdc0b85c 100644 --- a/doc/development/new_fe_guide/development/performance.md +++ b/doc/development/new_fe_guide/development/performance.md @@ -10,7 +10,7 @@ Any frontend engineer can contribute to this dashboard. They can contribute by a There are 3 recommended high impact metrics to review on each page: - [First visual change](https://web.dev/first-meaningful-paint/) -- [Speed Index](https://sites.google.com/a/webpagetest.org/docs/using-webpagetest/metrics/speed-index) -- [Visual Complete 95%](https://sites.google.com/a/webpagetest.org/docs/using-webpagetest/metrics/speed-index) +- [Speed Index](https://github.com/WPO-Foundation/webpagetest-docs/blob/master/user/Metrics/SpeedIndex.md) +- [Visual Complete 95%](https://github.com/WPO-Foundation/webpagetest-docs/blob/master/user/Metrics/SpeedIndex.md) For these metrics, lower numbers are better as it means that the website is more performant. diff --git a/doc/development/new_fe_guide/tips.md b/doc/development/new_fe_guide/tips.md index 13222e0cc9b..c65266a3f25 100644 --- a/doc/development/new_fe_guide/tips.md +++ b/doc/development/new_fe_guide/tips.md @@ -10,12 +10,12 @@ yarn clean ## Creating feature flags in development -The process for creating a feature flag is the same as [enabling a feature flag in development](../feature_flags/development.md#enabling-a-feature-flag-in-development). +The process for creating a feature flag is the same as [enabling a feature flag in development](../feature_flags/development.md#enabling-a-feature-flag-locally-in-development). Your feature flag can now be: - [Made available to the frontend](../feature_flags/development.md#frontend) via the `gon` -- Queried in [tests](../feature_flags/development.md#specs) +- Queried in [tests](../feature_flags/development.md#feature-flags-in-tests) - Queried in HAML templates and Ruby files via the `Feature.enabled?(:my_shiny_new_feature_flag)` method ### More on feature flags diff --git a/doc/development/packages.md b/doc/development/packages.md index ae67af10d88..e21eff543b4 100644 --- a/doc/development/packages.md +++ b/doc/development/packages.md @@ -1,4 +1,10 @@ -# Packages **(PREMIUM)** +--- +stage: Package +group: Package +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Packages This document will guide you through adding another [package management system](../administration/packages/index.md) support to GitLab. @@ -25,9 +31,9 @@ The existing database model requires the following: ### API endpoints -Package systems work with GitLab via API. For example `ee/lib/api/npm_packages.rb` +Package systems work with GitLab via API. For example `lib/api/npm_packages.rb` implements API endpoints to work with NPM clients. So, the first thing to do is to -add a new `ee/lib/api/your_name_packages.rb` file with API endpoints that are +add a new `lib/api/your_name_packages.rb` file with API endpoints that are necessary to make the package system client to work. Usually that means having endpoints like: @@ -73,7 +79,8 @@ NPM is currently a hybrid of the instance level and group level. It is using the top-level group or namespace as the defining portion of the name (for example, `@my-group-name/my-package-name`). -**Note:** Composer package naming scope is Instance Level. +NOTE: **Note:** +Composer package naming scope is Instance Level. ### Naming conventions @@ -174,15 +181,15 @@ The implementation of the different Merge Requests will vary between different p The MVC must support [Personal Access Tokens](../user/profile/personal_access_tokens.md) right from the start. We currently support two options for these tokens: OAuth and Basic Access. -OAuth authentication is already supported. You can see an example in the [npm API](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/api/npm_packages.rb). +OAuth authentication is already supported. You can see an example in the [npm API](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/npm_packages.rb). [Basic Access authentication](https://developer.mozilla.org/en-US/docs/Web/HTTP/Authentication) support is done by overriding a specific function in the API helpers, like -[this example in the Conan API](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/api/conan_packages.rb). +[this example in the Conan API](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/conan_packages.rb). For this authentication mechanism, keep in mind that some clients can send an unauthenticated request first, wait for the 401 Unauthorized response with the [`WWW-Authenticate`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/WWW-Authenticate) field, then send an updated (authenticated) request. This case is more involved as -GitLab needs to handle the 401 Unauthorized response. The [Nuget API](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/api/nuget_packages.rb) +GitLab needs to handle the 401 Unauthorized response. The [Nuget API](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/nuget_packages.rb) supports this case. #### Authorization diff --git a/doc/development/performance.md b/doc/development/performance.md index 16ea1aa27ff..1dc7538c55b 100644 --- a/doc/development/performance.md +++ b/doc/development/performance.md @@ -281,6 +281,10 @@ This can be done via `pkill -USR2 puma:`. The `:` disambiguates between `puma 4.3.3.gitlab.2 ...` (the master process) from `puma: cluster worker 0: ...` (the worker processes), selecting the latter. +For Sidekiq, the signal can be sent to the `sidekiq-cluster` process via `pkill +-USR2 bin/sidekiq-cluster`, which will forward the signal to all Sidekiq +children. Alternatively, you can also select a specific pid of interest. + Production profiles can be especially noisy. It can be helpful to visualize them as a [flamegraph](https://github.com/brendangregg/FlameGraph). This can be done via: diff --git a/doc/development/pipelines.md b/doc/development/pipelines.md index d3623529cd4..aef14535a96 100644 --- a/doc/development/pipelines.md +++ b/doc/development/pipelines.md @@ -13,10 +13,38 @@ as much as possible. ## Overview -### Pipeline types +Pipelines for the GitLab project are created using the [`workflow:rules` keyword](../ci/yaml/README.md#workflowrules) +feature of the GitLab CI/CD. -Since we use the [`rules:`](../ci/yaml/README.md#rules) and [`needs:`](../ci/yaml/README.md#needs) keywords extensively, -we have four main pipeline types which are described below. Note that an MR that includes multiple types of changes would +Pipelines are always created for the following scenarios: + +- `master` branch, including on schedules, pushes, merges, and so on. +- Merge requests. +- Tags. +- Stable, `auto-deploy`, and security branches. + +Pipeline creation is also affected by the following CI variables: + +- If `$FORCE_GITLAB_CI` is set, pipelines are created. +- If `$GITLAB_INTERNAL` is not set, pipelines are not created. + +No pipeline is created in any other cases (for example, when pushing a branch with no +MR for it). + +The source of truth for these workflow rules is defined in <https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab-ci.yml>. + +### Pipelines for Merge Requests + +In general, pipelines for an MR fall into one or more of the following types, +depending on the changes made in the MR: + +- [Docs-only MR pipeline](#docs-only-mr-pipeline): This is typically created for an MR that only changes documentation. +- [Code-only MR pipeline](#code-only-mr-pipeline): This is typically created for an MR that only changes code, either backend or frontend. +- [Frontend-only MR pipeline](#frontend-only-mr-pipeline): This is typically created for an MR that only changes frontend code. +- [QA-only MR pipeline](#qa-only-mr-pipeline): This is typically created for an MR that only changes end to end tests related code. + +We use the [`rules:`](../ci/yaml/README.md#rules) and [`needs:`](../ci/yaml/README.md#needs) keywords extensively +to determine the jobs that need to be run in a pipeline. Note that an MR that includes multiple types of changes would have a pipelines that include jobs from multiple types (e.g. a combination of docs-only and code-only pipelines). #### Docs-only MR pipeline @@ -345,22 +373,6 @@ graph RL; end ``` -### `workflow:rules` - -We're using the [`workflow:rules` keyword](../ci/yaml/README.md#workflowrules) to -define default rules to determine whether or not a pipeline is created. - -These rules are defined in <https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab-ci.yml> -and are as follows: - -1. If `$FORCE_GITLAB_CI` is set, create a pipeline. -1. For merge requests, create a pipeline. -1. For `master` branch, create a pipeline (this includes on schedules, pushes, merges, etc.). -1. For tags, create a pipeline. -1. If `$GITLAB_INTERNAL` isn't set, don't create a pipeline. -1. For stable, auto-deploy, and security branches, create a pipeline. -1. For any other cases (e.g. when pushing a branch with no MR for it), no pipeline is created. - ### PostgreSQL versions testing #### Current versions testing @@ -535,7 +547,8 @@ The current stages are: - `pages`: This stage includes a job that deploys the various reports as GitLab Pages (e.g. [`coverage-ruby`](https://gitlab-org.gitlab.io/gitlab/coverage-ruby/), [`coverage-javascript`](https://gitlab-org.gitlab.io/gitlab/coverage-javascript/), - [`webpack-report`](https://gitlab-org.gitlab.io/gitlab/webpack-report/). + and `webpack-report` (found at `https://gitlab-org.gitlab.io/gitlab/webpack-report/`, but there is + [an issue with the deployment](https://gitlab.com/gitlab-org/gitlab/-/issues/233458)). ### Default image @@ -572,7 +585,7 @@ that are scoped to a single [configuration parameter](../ci/yaml/README.md#confi | `.static-analysis-cache` | Allows a job to use a default `cache` definition suitable for static analysis tasks. | | `.yarn-cache` | Allows a job to use a default `cache` definition suitable for frontend jobs that do a `yarn install`. | | `.assets-compile-cache` | Allows a job to use a default `cache` definition suitable for frontend jobs that compile assets. | -| `.use-pg11` | Allows a job to use the `postgres:11.6` and `redis:alpine` services. | +| `.use-pg11` | Allows a job to use the `postgres:11.6` and `redis:4.0-alpine` services. | | `.use-pg11-ee` | Same as `.use-pg11` but also use the `docker.elastic.co/elasticsearch/elasticsearch:6.4.2` services. | | `.use-kaniko` | Allows a job to use the `kaniko` tool to build Docker images. | | `.as-if-foss` | Simulate the FOSS project by setting the `FOSS_ONLY='1'` environment variable. | diff --git a/doc/development/prometheus.md b/doc/development/prometheus.md index f64d4a2eda1..902d4e6a1d0 100644 --- a/doc/development/prometheus.md +++ b/doc/development/prometheus.md @@ -1,3 +1,9 @@ +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Working with Prometheus For more information on working with [Prometheus metrics](prometheus_metrics.md), see diff --git a/doc/development/prometheus_metrics.md b/doc/development/prometheus_metrics.md index 024da5cc943..a39d19d8750 100644 --- a/doc/development/prometheus_metrics.md +++ b/doc/development/prometheus_metrics.md @@ -1,3 +1,9 @@ +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Working with Prometheus Metrics ## Adding to the library diff --git a/doc/development/query_recorder.md b/doc/development/query_recorder.md index f7d2c23e28d..ef9a3c657aa 100644 --- a/doc/development/query_recorder.md +++ b/doc/development/query_recorder.md @@ -20,7 +20,8 @@ end As an example you might create 5 issues in between counts, which would cause the query count to increase by 5 if an N+1 problem exists. -> **Note:** In some cases the query count might change slightly between runs for unrelated reasons. In this case you might need to test `exceed_query_limit(control_count + acceptable_change)`, but this should be avoided if possible. +NOTE: **Note:** +In some cases the query count might change slightly between runs for unrelated reasons. In this case you might need to test `exceed_query_limit(control_count + acceptable_change)`, but this should be avoided if possible. ## Cached queries diff --git a/doc/development/redis.md b/doc/development/redis.md index 693b9e1ad0d..d5d42a3869e 100644 --- a/doc/development/redis.md +++ b/doc/development/redis.md @@ -41,8 +41,16 @@ moment, but may wish to in the future: [#118820](https://gitlab.com/gitlab-org/g This imposes an additional constraint on naming: where GitLab is performing operations that require several keys to be held on the same Redis server - for instance, diffing two sets held in Redis - the keys should ensure that by -enclosing the changeable parts in curly braces, such as, `project:{1}:set_a` and -`project:{1}:set_b`. +enclosing the changeable parts in curly braces. +For example: + +```plaintext +project:{1}:set_a +project:{1}:set_b +project:{2}:set_c +``` + +`set_a` and `set_b` are guaranteed to be held on the same Redis server, while `set_c` is not. Currently, we validate this in the development and test environments with the [`RedisClusterValidator`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/instrumentation/redis_cluster_validator.rb), diff --git a/doc/development/reference_processing.md b/doc/development/reference_processing.md index 79377533966..527fb94f228 100644 --- a/doc/development/reference_processing.md +++ b/doc/development/reference_processing.md @@ -18,6 +18,16 @@ and link the same type of objects (as specified by the `data-reference-type` attribute), then we only need one reference parser for that type of domain object. +## Banzai pipeline + +`Banzai` pipeline returns the `result` Hash after being filtered by the Pipeline. + +The `result` Hash is passed to each filter for modification. This is where Filters store extracted information from the content. +It contains: + +- An `:output` key with the DocumentFragment or String HTML markup based on the output of the last filter in the pipeline. +- A `:reference_filter_nodes` key with the list of DocumentFragment `nodes` that are ready for processing, updated by each filter in the pipeline. + ## Reference filters The first way that references are handled is by reference filters. These are @@ -69,6 +79,8 @@ a minimum implementation of `AbstractReferenceFilter` should define: ### Performance +#### Find object optimization + This default implementation is not very efficient, because we need to call `#find_object` for each reference, which may require issuing a DB query every time. For this reason, most reference filter implementations will instead use an @@ -96,6 +108,22 @@ This makes the number of queries linear in the number of projects. We only need to implement `parent_records` method when we call `records_per_parent` in our reference filter. +#### Filtering nodes optimization + +Each `ReferenceFilter` would iterate over all `<a>` and `text()` nodes in a document. + +Not all nodes are processed, document is filtered only for nodes that we want to process. +We are skipping: + +- Link tags already processed by some previous filter (if they have a `gfm` class). +- Nodes with the ancestor node that we want to ignore (`ignore_ancestor_query`). +- Empty line. +- Link tags with the empty `href` attribute. + +To avoid filtering such nodes for each `ReferenceFilter`, we do it only once and store the result in the result Hash of the pipeline as `result[:reference_filter_nodes]`. + +Pipeline `result` is passed to each filter for modification, so every time when `ReferenceFilter` replaces text or link tag, filtered list (`reference_filter_nodes`) will be updated for the next filter to use. + ## Reference parsers In a number of cases, as a performance optimization, we render Markdown to HTML diff --git a/doc/development/service_measurement.md b/doc/development/service_measurement.md index e53864c8640..24b16aac95b 100644 --- a/doc/development/service_measurement.md +++ b/doc/development/service_measurement.md @@ -71,7 +71,7 @@ In order to actually use it, you need to enable measuring for the desired servic ### Enabling measurement using feature flags In the following example, the `:gitlab_service_measuring_projects_import_service` -[feature flag](feature_flags/development.md#enabling-a-feature-flag-in-development) is used to enable the measuring feature +[feature flag](feature_flags/development.md#enabling-a-feature-flag-locally-in-development) is used to enable the measuring feature for `Projects::ImportService`. From ChatOps: diff --git a/doc/development/sidekiq_style_guide.md b/doc/development/sidekiq_style_guide.md index 2793756ff64..c5dfc5731e6 100644 --- a/doc/development/sidekiq_style_guide.md +++ b/doc/development/sidekiq_style_guide.md @@ -64,6 +64,36 @@ the extra jobs will take resources away from jobs from workers that were already there, if the resources available to the Sidekiq process handling the namespace are not adjusted appropriately. +## Versioning + +Version can be specified on each Sidekiq worker class. +This is then sent along when the job is created. + +```ruby +class FooWorker + include ApplicationWorker + + version 2 + + def perform(*args) + if job_version == 2 + foo = args.first['foo'] + else + foo = args.first + end + end +end +``` + +Under this schema, any worker is expected to be able to handle any job that was +enqueued by an older version of that worker. This means that when changing the +arguments a worker takes, you must increment the `version` (or set `version 1` +if this is the first time a worker's arguments are changing), but also make sure +that the worker is still able to handle jobs that were queued with any earlier +version of the arguments. From the worker's `perform` method, you can read +`self.job_version` if you want to specifically branch on job version, or you +can read the number or type of provided arguments. + ## Idempotent Jobs It's known that a job can fail for multiple reasons. For example, network outages or bugs. @@ -145,10 +175,27 @@ authorizations for both projects. GitLab doesn't skip jobs scheduled in the future, as we assume that the state will have changed by the time the job is scheduled to -execute. +execute. If you do want to deduplicate jobs scheduled in the future +this can be specified on the worker as follows: + +```ruby +module AuthorizedProjectUpdate + class UserRefreshOverUserRangeWorker + include ApplicationWorker + + deduplicate :until_executing, including_scheduled: true + idempotent! + + # ... + end +end +``` -More [deduplication strategies have been suggested](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/195). If you are implementing a worker that -could benefit from a different strategy, please comment in the issue. +This strategy is called `until_executing`. More [deduplication +strategies have been +suggested](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/195). If +you are implementing a worker that could benefit from a different +strategy, please comment in the issue. If the automatic deduplication were to cause issues in certain queues. This can be temporarily disabled by enabling a feature flag diff --git a/doc/development/sql.md b/doc/development/sql.md index d584a26e455..3b969c7d27a 100644 --- a/doc/development/sql.md +++ b/doc/development/sql.md @@ -218,9 +218,9 @@ Project.select(:id, :user_id).joins(:merge_requests) ## Plucking IDs -This can't be stressed enough: **never** use ActiveRecord's `pluck` to pluck a -set of values into memory only to use them as an argument for another query. For -example, this will make the database **very** sad: +Never use ActiveRecord's `pluck` to pluck a set of values into memory only to +use them as an argument for another query. For example, this will execute an +extra unecessary database query and load a lot of unecessary data into memory: ```ruby projects = Project.all.pluck(:id) diff --git a/doc/development/telemetry/event_dictionary.md b/doc/development/telemetry/event_dictionary.md new file mode 100644 index 00000000000..d8cc32ea8d0 --- /dev/null +++ b/doc/development/telemetry/event_dictionary.md @@ -0,0 +1,32 @@ +--- +stage: Growth +group: Telemetry +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Event Dictionary + +**Note: We've temporarily moved the Event Dictionary to a [Google Sheet](https://docs.google.com/spreadsheets/d/1VzE8R72Px_Y_LlE3Z05LxUlG_dumWe3vl-HeUo70TPw/edit?usp=sharing)**. The previous Markdown table exceeded 600 rows making it difficult to manage. In the future, our intention is to move this back into our docs using a [YAML file](https://gitlab.com/gitlab-org/gitlab-docs/-/issues/823). + +The event dictionary is a single source of truth for the metrics and events we collect for product usage data. The Event Dictionary lists all the metrics and events we track, why we're tracking them, and where they are tracked. + +This is a living document that is updated any time a new event is planned or implemented. It includes the following information. + +- Section, stage, or group +- Description +- Implementation status +- Availability by plan type +- Code path + +We're currently focusing our Event Dictionary on [Usage Ping](usage_ping.md). In the future, we will also include [Snowplow](snowplow.md). We currently have an initiative across the entire product organization to complete the [Event Dictionary for Usage Ping](https://gitlab.com/groups/gitlab-org/-/epics/4174). + +## Instructions + +1. Open the Event Dictionary and fill in all the **PM to edit** columns highlighted in yellow. +1. Check that all the metrics and events are assigned to the correct section, stage, or group. If a metric is used across many groups, assign it to the stage. If a metric is used across many stages, assign it to the section. If a metric is incorrectly assigned to another section, stage, or group, let the PM know you have reassigned it. If your group has no assigned metrics and events, check that your metrics and events are not incorrectly assigned to another PM. +1. Add descriptions of what your metrics and events are tracking. Work with your Engineering team or the Telemetry team if you need help understanding this. +1. Add what plans this metric is available on. Work with your Engineering team or the Telemetry team if you need help understanding this. + +## Planned metrics and events + +For future metrics and events you plan to track, please add them to the Event Dictionary and note the status as `Planned`, `In Progress`, or `Implemented`. Once you have confirmed the metric has been implemented and have confirmed the metric data is in our data warehouse, change the status to **Data Available**. diff --git a/doc/development/telemetry/index.md b/doc/development/telemetry/index.md index 0000e7e9e4f..b5032ce3730 100644 --- a/doc/development/telemetry/index.md +++ b/doc/development/telemetry/index.md @@ -6,60 +6,35 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Telemetry Guide -At GitLab, we collect telemetry for the purpose of helping us build a better GitLab. Data about how GitLab is used is collected to better understand what parts of GitLab needs improvement and what features to build next. Telemetry also helps our team better understand the reasons why people use GitLab and with this knowledge we are able to make better product decisions. +At GitLab, we collect product usage data for the purpose of helping us build a better product. Data helps GitLab understand which parts of the product need improvement and which features we should build next. Product usage data also helps our team better understand the reasons why people use GitLab. With this knowledge we are able to make better product decisions. -We also encourage users to enable tracking, and we embrace full transparency with our tracking approach so it can be easily understood and trusted. By enabling tracking, users can: +We encourage users to enable tracking, and we embrace full transparency with our tracking approach so it can be easily understood and trusted. + +By enabling tracking, users can: - Contribute back to the wider community. - Help GitLab improve on the product. -This documentation consists of three guides providing an overview of Telemetry at GitLab. - -Telemetry Guide: - - 1. [Our tracking tools](#our-tracking-tools) - 1. [What data can be tracked](#what-data-can-be-tracked) - 1. [Telemetry systems overview](#telemetry-systems-overview) - 1. [Snowflake data warehouse](#snowflake-data-warehouse) +## Our tracking tools -[Usage Ping Guide](usage_ping.md) +We use three methods to gather product usage data: - 1. [What is Usage Ping](usage_ping.md#what-is-usage-ping) - 1. [Usage Ping payload](usage_ping.md#usage-ping-payload) - 1. [Disable Usage Ping](usage_ping.md#disable-usage-ping) - 1. [Usage Ping request flow](usage_ping.md#usage-ping-request-flow) - 1. [How Usage Ping works](usage_ping.md#how-usage-ping-works) - 1. [Implementing Usage Ping](usage_ping.md#implementing-usage-ping) - 1. [Developing and testing Usage Ping](usage_ping.md#developing-and-testing-usage-ping) +- [Snowplow](#snowplow) +- [Usage Ping](#usage-ping) +- [Database import](#database-import) -[Snowplow Guide](snowplow.md) +### Snowplow -1. [What is Snowplow](snowplow.md#what-is-snowplow) -1. [Snowplow schema](snowplow.md#snowplow-schema) -1. [Enabling Snowplow](snowplow.md#enabling-snowplow) -1. [Snowplow request flow](snowplow.md#snowplow-request-flow) -1. [Implementing Snowplow JS (Frontend) tracking](snowplow.md#implementing-snowplow-js-frontend-tracking) -1. [Implementing Snowplow Ruby (Backend) tracking](snowplow.md#implementing-snowplow-ruby-backend-tracking) -1. [Developing and testing Snowplow](snowplow.md#developing-and-testing-snowplow) +Snowplow is an enterprise-grade marketing and product analytics platform which helps track the way +users engage with our website and application. -More useful links: +Snowplow consists of two components: -- [Telemetry Direction](https://about.gitlab.com/direction/telemetry/) -- [Data Analysis Process](https://about.gitlab.com/handbook/business-ops/data-team/#data-analysis-process/) -- [Data for Product Managers](https://about.gitlab.com/handbook/business-ops/data-team/programs/data-for-product-managers/) -- [Data Infrastructure](https://about.gitlab.com/handbook/business-ops/data-team/platform/infrastructure/) - -## Our tracking tools +- [Snowplow JS](https://github.com/snowplow/snowplow/wiki/javascript-tracker) tracks client-side + events. +- [Snowplow Ruby](https://github.com/snowplow/snowplow/wiki/ruby-tracker) tracks server-side events. -We use several different technologies to gather product usage data. - -### Snowplow JS (Frontend) - -Snowplow is an enterprise-grade marketing and product analytics platform which helps track the way users engage with our website and application. [Snowplow JS](https://github.com/snowplow/snowplow/wiki/javascript-tracker) is a frontend tracker for client-side events. - -### Snowplow Ruby (Backend) - -Snowplow is an enterprise-grade marketing and product analytics platform which helps track the way users engage with our website and application. [Snowplow Ruby](https://github.com/snowplow/snowplow/wiki/ruby-tracker) is a backend tracker for server-side events. +For more details, read the [Snowplow](snowplow.md) guide. ### Usage Ping @@ -71,37 +46,39 @@ For more details, read the [Usage Ping](usage_ping.md) guide. Database imports are full imports of data into GitLab's data warehouse. For GitLab.com, the PostgreSQL database is loaded into Snowflake data warehouse every 6 hours. For more details, see the [data team handbook](https://about.gitlab.com/handbook/business-ops/data-team/platform/#extract-and-load). -### Log system - -System logs are the application logs generated from running the GitLab Rails application. For more details, see the [log system](../../administration/logs.md) and [logging infrastructure](https://gitlab.com/gitlab-com/runbooks/tree/master/logging/doc#logging-infrastructure-overview). - ## What data can be tracked Our different tracking tools allows us to track different types of events. The event types and examples of what data can be tracked are outlined below. -| Event Type | Snowplow JS (Frontend) | Snowplow Ruby (Backend) | Usage Ping | Database import | Log system | -|---------------------|------------------------|-------------------------|---------------------|---------------------|---------------------| -| Database counts | **{dotted-circle}** | **{dotted-circle}** | **{check-circle}** | **{check-circle}** | **{dotted-circle}** | -| Pageview events | **{check-circle}** | **{check-circle}** | **{dotted-circle}** | **{dotted-circle}** | **{dotted-circle}** | -| UI events | **{check-circle}** | **{dotted-circle}** | **{dotted-circle}** | **{dotted-circle}** | **{dotted-circle}** | -| CRUD and API events | **{dotted-circle}** | **{check-circle}** | **{dotted-circle}** | **{dotted-circle}** | **{dotted-circle}** | -| Event funnels | **{check-circle}** | **{check-circle}** | **{dotted-circle}** | **{dotted-circle}** | **{dotted-circle}** | -| PostgreSQL Data | **{dotted-circle}** | **{dotted-circle}** | **{dotted-circle}** | **{check-circle}** | **{dotted-circle}** | -| Logs | **{dotted-circle}** | **{dotted-circle}** | **{dotted-circle}** | **{dotted-circle}** | **{check-circle}** | -| External services | **{dotted-circle}** | **{dotted-circle}** | **{dotted-circle}** | **{dotted-circle}** | **{dotted-circle}** | +The availability of event types and their tracking tools varies by segment. For example, on Self-Managed Users, we only have reporting using Database records via Usage Ping. + +| Event Types | SaaS Instance | SaaS Plan | SaaS Group | SaaS Session | SaaS User | SM Instance | SM Plan | SM Group | SM Session | SM User | +|----------------------------------------|---------------|-----------|------------|--------------|-----------|-------------|---------|----------|------------|---------| +| Snowplow (JS Pageview events) | ✅ | 📅 | 📅 | ✅ | 📅 | 📅 | 📅 | 📅 | 📅 | 📅 | +| Snowplow (JS UI events) | ✅ | 📅 | 📅 | ✅ | 📅 | 📅 | 📅 | 📅 | 📅 | 📅 | +| Snowplow (Ruby Pageview events) | ✅ | 📅 | 📅 | ✅ | 📅 | 📅 | 📅 | 📅 | 📅 | 📅 | +| Snowplow (Ruby CRUD / API events) | ✅ | 📅 | 📅 | ✅ | 📅 | 📅 | 📅 | 📅 | 📅 | 📅 | +| Usage Ping (Redis UI counters) | 🔄 | 🔄 | 🔄 | ✖️ | 🔄 | 🔄 | 🔄 | 🔄 | ✖️ | 🔄 | +| Usage Ping (Redis Pageview counters) | 🔄 | 🔄 | 🔄 | ✖️ | 🔄 | 🔄 | 🔄 | 🔄 | ✖️ | 🔄 | +| Usage Ping (Redis CRUD / API counters) | 🔄 | 🔄 | 🔄 | ✖️ | 🔄 | 🔄 | 🔄 | 🔄 | ✖️ | 🔄 | +| Usage Ping (Database counters) | ✅ | 🔄 | 📅 | ✖️ | ✅ | ✅ | ✅ | ✅ | ✖️ | ✅ | +| Usage Ping (Instance settings) | ✅ | 🔄 | 📅 | ✖️ | ✅ | ✅ | ✅ | ✅ | ✖️ | ✅ | +| Usage Ping (Integration settings) | ✅ | 🔄 | 📅 | ✖️ | ✅ | ✅ | ✅ | ✅ | ✖️ | ✅ | +| Database import (Database records) | ✅ | ✅ | ✅ | ✖️ | ✅ | ✖️ | ✖️ | ✖️ | ✖️ | ✖️ | + +[Source file](https://docs.google.com/spreadsheets/d/1e8Afo41Ar8x3JxAXJF3nL83UxVZ3hPIyXdt243VnNuE/edit?usp=sharing) -### Database counts +**Legend** -- Number of Projects created by unique users -- Number of users logged in the past 28 day +✅ Available, 🔄 In Progress, 📅 Planned, ✖️ Not Possible -Database counts are row counts for different tables in an instance’s database. These are SQL count queries which have been filtered, grouped, or aggregated which provide high level usage data. The full list of available tables can be found in [structure.sql](https://gitlab.com/gitlab-org/gitlab/-/blob/master/db/structure.sql). +SaaS = GitLab.com. SM = Self-Managed instance ### Pageview events - Number of sessions that visited the /dashboard/groups page -### UI Events +### UI events - Number of sessions that clicked on a button or link - Number of sessions that closed a modal @@ -116,24 +93,55 @@ UI events are any interface-driven actions from the browser including click data These are backend events that include the creation, read, update, deletion of records, and other events that might be triggered from layers other than those available in the interface. -### Event funnels +### Database records -- Number of sessions that performed action A, B, then C -- Conversion rate from step A to B +These are raw database records which can be explored using business intelligence tools like Sisense. The full list of available tables can be found in [structure.sql](https://gitlab.com/gitlab-org/gitlab/-/blob/master/db/structure.sql). -### PostgreSQL data +### Instance settings -These are raw database records which can be explored using business intelligence tools like Sisense. The full list of available tables can be found in [structure.sql](https://gitlab.com/gitlab-org/gitlab/-/blob/master/db/structure.sql). +These are settings of your instance such as the instance's Git version and if certain features are enabled such as `container_registry_enabled`. -### Logs +### Integration settings -These are raw logs such as the [Production logs](../../administration/logs.md#production_jsonlog), [API logs](../../administration/logs.md#api_jsonlog), or [Sidekiq logs](../../administration/logs.md#sidekiqlog). See the [overview of Logging Infrastructure](https://gitlab.com/gitlab-com/runbooks/tree/master/logging/doc#logging-infrastructure-overview) for more details. +These are integrations your GitLab instance interacts with such as an [external storage provider](../../administration/static_objects_external_storage.md) or an [external container registry](../../administration/packages/container_registry.md#use-an-external-container-registry-with-gitlab-as-an-auth-endpoint). These services must be able to send data back into a GitLab instance for data to be tracked. -### External services +## Reporting level -These are external services a GitLab instance interacts with such as an [external storage provider](../../administration/static_objects_external_storage.md) or an [external container registry](../../administration/packages/container_registry.md#use-an-external-container-registry-with-gitlab-as-an-auth-endpoint). These services must be able to send data back into a GitLab instance for data to be tracked. +Our reporting levels of aggregate or individual reporting varies by segment. For example, on Self-Managed Users, we can report at an aggregate user level using Usage Ping but not on an Individual user level. -## Telemetry systems overview +| Aggregated Reporting | SaaS Instance | SaaS Plan | SaaS Group | SaaS Session | SaaS User | SM Instance | SM Plan | SM Group | SM Session | SM User | +|----------------------|---------------|-----------|------------|--------------|-----------|-------------|---------|----------|------------|---------| +| Snowplow | ✅ | 📅 | 📅 | ✅ | 📅 | ✅ | 📅 | 📅 | ✅ | 📅 | +| Usage Ping | ✅ | 🔄 | 📅 | 📅 | ✅ | ✅ | ✅ | ✅ | 📅 | ✅ | +| Database import | ✅ | ✅ | ✅ | ✖️ | ✅ | ✖️ | ✖️ | ✖️ | ✖️ | ✖️ | + +| Identifiable Reporting | SaaS Instance | SaaS Plan | SaaS Group | SaaS Session | SaaS User | SM Instance | SM Plan | SM Group | SM Session | SM User | +|------------------------|---------------|-----------|------------|--------------|-----------|-------------|---------|----------|------------|---------| +| Snowplow | ✅ | 📅 | 📅 | ✅ | 📅 | ✖️ | ✖️ | ✖️ | ✖️ | ✖️ | +| Usage Ping | ✅ | 🔄 | 📅 | ✖️ | ✖️ | ✅ | ✅ | ✖️ | ✖️ | ✖️ | +| Database import | ✅ | ✅ | ✅ | ✖️ | ✅ | ✖️ | ✖️ | ✖️ | ✖️ | ✖️ | + +**Legend** + +✅ Available, 🔄 In Progress, 📅 Planned, ✖️ Not Possible + +SaaS = GitLab.com. SM = Self-Managed instance + +## Reporting time period + +Our reporting time periods varies by segment. For example, on Self-Managed Users, we can report all time counts and 28 day counts in Usage Ping. + +| Reporting Time Period | All Time | 28 Days | 7 Days | Daily | +|-----------------------|----------|---------|--------|-------| +| Snowplow | ✅ | ✅ | ✅ | ✅ | +| Usage Ping | ✅ | ✅ | 📅 | ✖️ | +| Database import | ✅ | ✅ | ✅ | ✅ | + +**Legend** + +✅ Available, 🔄 In Progress, 📅 Planned, ✖️ Not Possible + +## Systems overview The systems overview is a simplified diagram showing the interactions between GitLab Inc and self-managed instances. @@ -145,7 +153,7 @@ The systems overview is a simplified diagram showing the interactions between Gi For Telemetry purposes, GitLab Inc has three major components: -1. [Data Infrastructure](https://about.gitlab.com/handbook/business-ops/data-team/platform/infrastructure/): This contains everything managed by our data team including Sisense Dashboards for visualization, Snowflake for Data Warehousing, incoming data sources such as PostgreSQL Pipeline and S3 Bucket, and lastly our data collectors [GitLab.com's Snowplow Collector](https://about.gitlab.com/handbook/engineering/infrastructure/library/snowplow/) and GitLab's Versions Application. +1. [Data Infrastructure](https://about.gitlab.com/handbook/business-ops/data-team/platform/infrastructure/): This contains everything managed by our data team including Sisense Dashboards for visualization, Snowflake for Data Warehousing, incoming data sources such as PostgreSQL Pipeline and S3 Bucket, and lastly our data collectors [GitLab.com's Snowplow Collector](https://gitlab.com/gitlab-com/gl-infra/readiness/-/tree/master/library/snowplow/) and GitLab's Versions Application. 1. GitLab.com: This is the production GitLab application which is made up of a Client and Server. On the Client or browser side, a Snowplow JS Tracker (Frontend) is used to track client-side events. On the Server or application side, a Snowplow Ruby Tracker (Backend) is used to track server-side events. The server also contains Usage Ping which leverages a PostgreSQL database and a Redis in-memory data store to report on usage data. Lastly, the server also contains System Logs which are generated from running the GitLab application. 1. [Monitoring infrastructure](https://about.gitlab.com/handbook/engineering/monitoring/): This is the infrastructure used to ensure GitLab.com is operating smoothly. System Logs are sent from GitLab.com to our monitoring infrastructure and collected by a FluentD collector. From FluentD, logs are either sent to long term Google Cloud Services cold storage via Stackdriver, or, they are sent to our Elastic Cluster via Cloud Pub/Sub which can be explored in real-time using Kibana. @@ -162,27 +170,13 @@ As shown by the orange lines, on GitLab.com Snowplow JS, Snowplow Ruby, Usage Pi As shown by the green lines, on GitLab.com system logs flow into GitLab Inc's monitoring infrastructure. On self-managed, there are no logs sent to GitLab Inc's monitoring infrastructure. -The differences between GitLab.com and self-managed are summarized below: - -| Environment | Snowplow JS (Frontend) | Snowplow Ruby (Backend) | Usage Ping | Database import | Logs system | -|--------------|------------------------|-------------------------|--------------------|---------------------|---------------------| -| GitLab.com | **{check-circle}** | **{check-circle}** | **{check-circle}** | **{check-circle}** | **{check-circle}** | -| Self-Managed | **{dotted-circle}**(1) | **{dotted-circle}**(1) | **{check-circle}** | **{dotted-circle}** | **{dotted-circle}** | - Note (1): Snowplow JS and Snowplow Ruby are available on self-managed, however, the Snowplow Collector endpoint is set to a self-managed Snowplow Collector which GitLab Inc does not have access to. -## Snowflake data warehouse - -The Snowflake data warehouse is where we keep all of GitLab Inc's data. - -### Data sources - -There are several data sources available in Snowflake and Sisense each representing a different view of the data along the transformation pipeline. +## Additional information -| Source | Description | Access | -| ------ | ------ | ------ | -| raw | These tables are the raw data source | Access via Snowflake | -| analytics_staging | These tables have undergone little to no data transformation, meaning they're basically clones of the raw data source | Access via Snowflake or Sisense | -| analytics | These tables have typically undergone more data transformation. They will typically end in `_xf` to represent the fact that they are transformed | Access via Snowflake or Sisense | +More useful links: -If you are a Product Manager interested in the raw data, you will likely focus on the `analytics` and `analytics_staging` sources. The raw source is limited to the data and infrastructure teams. For more information, please see [Data For Product Managers: What's the difference between analytics_staging and analytics?](https://about.gitlab.com/handbook/business-ops/data-team/programs/data-for-product-managers/#whats-the-difference-between-analytics_staging-and-analytics) +- [Telemetry Direction](https://about.gitlab.com/direction/telemetry/) +- [Data Analysis Process](https://about.gitlab.com/handbook/business-ops/data-team/#data-analysis-process/) +- [Data for Product Managers](https://about.gitlab.com/handbook/business-ops/data-team/programs/data-for-product-managers/) +- [Data Infrastructure](https://about.gitlab.com/handbook/business-ops/data-team/platform/infrastructure/) diff --git a/doc/development/telemetry/snowplow.md b/doc/development/telemetry/snowplow.md index f03742afe2d..547ba36464b 100644 --- a/doc/development/telemetry/snowplow.md +++ b/doc/development/telemetry/snowplow.md @@ -278,7 +278,7 @@ Custom event tracking and instrumentation can be added by directly calling the ` |:-----------|:-------|:--------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `category` | string | 'application' | Area or aspect of the application. This could be `HealthCheckController` or `Lfs::FileTransformer` for instance. | | `action` | string | 'generic' | The action being taken, which can be anything from a controller action like `create` to something like an Active Record callback. | -| `data` | object | {} | Additional data such as `label`, `property`, `value`, and `context` as described [in our Feature Instrumentation taxonomy](https://about.gitlab.com/handbook/product/feature-instrumentation/#taxonomy). These are set as empty strings if you don't provide them. | +| `data` | object | {} | Additional data such as `label`, `property`, `value`, and `context` as described in [Instrumentation at GitLab](https://about.gitlab.com/handbook/product/product-processes/#taxonomy). These are set as empty strings if you don't provide them. | Tracking can be viewed as either tracking user behavior, or can be utilized for instrumentation to monitor and visualize performance over time in an area or aspect of code. @@ -305,12 +305,16 @@ We use the [AsyncEmitter](https://github.com/snowplow/snowplow/wiki/Ruby-Tracker There are several tools for developing and testing Snowplow Event -| Testing Tool | Frontend Tracking | Backend Tracking | Local Development Environment | Production Environment | -|----------------------------------------------|--------------------|---------------------|-------------------------------|------------------------| -| Snowplow Analytics Debugger Chrome Extension | **{check-circle}** | **{dotted-circle}** | **{check-circle}** | **{check-circle}** | -| Snowplow Inspector Chrome Extension | **{check-circle}** | **{dotted-circle}** | **{check-circle}** | **{check-circle}** | -| Snowplow Micro | **{check-circle}** | **{check-circle}** | **{check-circle}** | **{dotted-circle}** | -| Snowplow Mini | **{check-circle}** | **{check-circle}** | **{dotted-circle}** | **{check-circle}** | +| Testing Tool | Frontend Tracking | Backend Tracking | Local Development Environment | Production Environment | Production Environment | +|----------------------------------------------|--------------------|---------------------|-------------------------------|------------------------|------------------------| +| Snowplow Analytics Debugger Chrome Extension | **{check-circle}** | **{dotted-circle}** | **{check-circle}** | **{check-circle}** | **{check-circle}** | +| Snowplow Inspector Chrome Extension | **{check-circle}** | **{dotted-circle}** | **{check-circle}** | **{check-circle}** | **{check-circle}** | +| Snowplow Micro | **{check-circle}** | **{check-circle}** | **{check-circle}** | **{dotted-circle}** | **{dotted-circle}** | +| Snowplow Mini | **{check-circle}** | **{check-circle}** | **{dotted-circle}** | **{status_preparing}** | **{status_preparing}** | + +**Legend** + +**{check-circle}** Available, **{status_preparing}** In progress, **{dotted-circle}** Not Planned ### Snowplow Analytics Debugger Chrome Extension diff --git a/doc/development/telemetry/usage_ping.md b/doc/development/telemetry/usage_ping.md index 5e78e2c5f25..ea5eb6c389f 100644 --- a/doc/development/telemetry/usage_ping.md +++ b/doc/development/telemetry/usage_ping.md @@ -131,7 +131,7 @@ There are four types of counters which are all found in `usage_data.rb`: - **Ordinary Batch Counters:** Simple count of a given ActiveRecord_Relation - **Distinct Batch Counters:** Distinct count of a given ActiveRecord_Relation on given column - **Alternative Counters:** Used for settings and configurations -- **Redis Counters:** Used for in-memory counts. This method is being deprecated due to data inaccuracies and will be replaced with a persistent method. +- **Redis Counters:** Used for in-memory counts. NOTE: **Note:** Only use the provided counter methods. Each counter method contains a built in fail safe to isolate each counter to avoid breaking the entire Usage Ping. @@ -155,7 +155,7 @@ There are two batch counting methods provided, `Ordinary Batch Counters` and `Di Handles `ActiveRecord::StatementInvalid` error -Simple count of a given ActiveRecord_Relation +Simple count of a given ActiveRecord_Relation, does a non-distinct batch count, smartly reduces batch_size and handles errors. Method: `count(relation, column = nil, batch: true, start: nil, finish: nil)` @@ -179,15 +179,16 @@ count(::Clusters::Cluster.aws_installed.enabled, :cluster_id, start: ::Clusters: Handles `ActiveRecord::StatementInvalid` error -Distinct count of a given ActiveRecord_Relation on given column +Distinct count of a given ActiveRecord_Relation on given column, a distinct batch count, smartly reduces batch_size and handles errors. -Method: `distinct_count(relation, column = nil, batch: true, start: nil, finish: nil)` +Method: `distinct_count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil)` Arguments: - `relation` the ActiveRecord_Relation to perform the count - `column` the column to perform the distinct count, by default is the primary key - `batch`: default `true` in order to use batch counting +- `batch_size`: if none set it will use default value 10000 from `Gitlab::Database::BatchCounter` - `start`: custom start of the batch counting in order to avoid complex min calculations - `end`: custom end of the batch counting in order to avoid complex min calculations @@ -212,14 +213,48 @@ Arguments: - `counter`: a counter from `Gitlab::UsageDataCounters`, that has `fallback_totals` method implemented - or a `block`: which is evaluated +#### Ordinary Redis Counters + +Examples of implementation: + +- Using Redis methods [`INCR`](https://redis.io/commands/incr), [`GET`](https://redis.io/commands/get), and [`Gitlab::UsageDataCounters::WikiPageCounter`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/wiki_page_counter.rb) +- Using Redis methods [`HINCRBY`](https://redis.io/commands/hincrby), [`HGETALL`](https://redis.io/commands/hgetall), and [`Gitlab::UsageCounters::PodLogs`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_counters/pod_logs.rb) + +#### Redis HLL Counters + +With `Gitlab::Redis::HLL` we have available data structures used to count unique values. + +Implemented using Redis methods [PFADD](https://redis.io/commands/pfadd) and [PFCOUNT](https://redis.io/commands/pfcount). + +Recommendations: + +- Key should expire in 29 days. +- If possible, data granularity should be a week. For example a key could be composed from the metric's name and week of the year, `2020-33-{metric_name}`. +- Use a [feature flag](../../operations/feature_flags.md) in order to have a control over the impact when adding new metrics. +- If possible, data granularity should be week, for example a key could be composed from metric name and week of the year, 2020-33-{metric_name} +- Use a [feature flag](../../operations/feature_flags.md) in order to have a control over the impact when adding new metrics + +Examples of implementation: + +- [`Gitlab::UsageDataCounters::TrackUniqueActions`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/track_unique_actions.rb) +- [`Gitlab::Analytics::UniqueVisits`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/analytics/unique_visits.rb) + Example of usage: ```ruby +# Redis Counters redis_usage_data(Gitlab::UsageDataCounters::WikiPageCounter) redis_usage_data { ::Gitlab::UsageCounters::PodLogs.usage_totals[:total] } -``` -Note that Redis counters are in the [process of being deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/216330) and you should instead try to use Snowplow events instead. We're in the process of building [self-managed event tracking](https://gitlab.com/gitlab-org/telemetry/-/issues/373) and once this is available, we will convert all Redis counters into Snowplow events. +# Redis HLL counter +counter = Gitlab::UsageDataCounters::TrackUniqueActions +redis_usage_data do + counter.count_unique_events( + event_action: Gitlab::UsageDataCounters::TrackUniqueActions::PUSH_ACTION, + date_from: time_period[:created_at].first, + date_to: time_period[:created_at].last + ) +``` ### Alternative Counters @@ -313,14 +348,16 @@ In order to have an understanding of the query's execution we add in the MR desc We also use `#database-lab` and [explain.depesz.com](https://explain.depesz.com/). For more details, see the [database review guide](../database_review.md#preparation-when-adding-or-modifying-queries). -Examples of query optimization work: +#### Optimization recommendations and examples -- [Example 1](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/26445) -- [Example 2](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/26871) +- Use specialized indexes [example 1](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/26871), [example 2](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/26445). +- Use defined `start` and `finish`, and simple queries, because these values can be memoized and reused, [example](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/37155). +- Avoid joins and write the queries as simply as possible, [example](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/36316). +- Set a custom `batch_size` for `distinct_count`, [example](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/38000). ### 4. Add the metric definition -When adding, changing, or updating metrics, please update the [Usage Statistics definition table](#usage-statistics-definitions). +When adding, changing, or updating metrics, please update the [Event Dictionary's **Usage Ping** table](event_dictionary.md). ### 5. Add new metric to Versions Application @@ -340,6 +377,10 @@ Ensure you comply with the [Changelog entries guide](../changelog.md). On GitLab.com, we have DangerBot setup to monitor Telemetry related files and DangerBot will recommend a Telemetry review. Mention `@gitlab-org/growth/telemetry/engineers` in your MR for a review. +### 9. Verify your metric + +On GitLab.com, the Product Analytics team regularly monitors Usage Ping. They may alert you that your metrics need further optimization to run quicker and with greater success. You may also use the [Usage Ping QA dashboard](https://app.periscopedata.com/app/gitlab/632033/Usage-Ping-QA) to check how well your metric performs. The dashboard allows filtering by GitLab version, by "Self-managed" & "Saas" and shows you how many failures have occurred for each metric. Whenever you notice a high failure rate, you may re-optimize your metric. + ### Optional: Test Prometheus based Usage Ping If the data submitted includes metrics [queried from Prometheus](#prometheus-queries) that you would like to inspect and verify, @@ -386,358 +427,6 @@ with any of the other services that are running. That is not how node metrics ar always runs as a process alongside other GitLab components on any given node. From Usage Ping's perspective none of the node data would therefore appear to be associated to any of the services running, since they all appear to be running on different hosts. To alleviate this problem, the `node_exporter` in GCK was arbitrarily "assigned" to the `web` service, meaning only for this service `node_*` metrics will appear in Usage Ping. -## Usage Statistics definitions - -| Statistic | Section | Stage | Tier | Edition | Description | -| --------------------------------------------------------- | ------------------------------------ | ------------- | ---------------- | ------- | -------------------------------------------------------------------------- | -| `uuid` | | | | | | -| `hostname` | | | | | | -| `version` | | | | | | -| `installation_type` | | | | | | -| `active_user_count` | | | | | | -| `recorded_at` | | | | | | -| `recording_ce_finished_at` | | | | CE+EE | When the core features were computed | -| `recording_ee_finished_at` | | | | EE | When the EE-specific features were computed | -| `edition` | | | | | | -| `license_md5` | | | | | | -| `license_id` | | | | | | -| `historical_max_users` | | | | | | -| `Name` | `licensee` | | | | | -| `Email` | `licensee` | | | | | -| `Company` | `licensee` | | | | | -| `license_user_count` | | | | | | -| `license_starts_at` | | | | | | -| `license_expires_at` | | | | | | -| `license_plan` | | | | | | -| `license_trial` | | | | | | -| `assignee_lists` | `counts` | | | | | -| `boards` | `counts` | | | | | -| `ci_builds` | `counts` | `verify` | | | Unique builds in project | -| `ci_internal_pipelines` | `counts` | `verify` | | | Total pipelines in GitLab repositories | -| `ci_external_pipelines` | `counts` | `verify` | | | Total pipelines in external repositories | -| `ci_pipeline_config_auto_devops` | `counts` | `verify` | | | Total pipelines from an Auto DevOps template | -| `ci_pipeline_config_repository` | `counts` | `verify` | | | Total Pipelines from templates in repository | -| `ci_runners` | `counts` | `verify` | | | Total configured Runners in project | -| `ci_triggers` | `counts` | `verify` | | | Total configured Triggers in project | -| `ci_pipeline_schedules` | `counts` | `verify` | | | Pipeline schedules in GitLab | -| `auto_devops_enabled` | `counts` | `configure` | | | Projects with Auto DevOps template enabled | -| `auto_devops_disabled` | `counts` | `configure` | | | Projects with Auto DevOps template disabled | -| `deploy_keys` | `counts` | | | | | -| `deployments` | `counts` | `release` | | | Total deployments | -| `deployments` | `counts_monthly` | `release` | | | Total deployments last 28 days | -| `dast_jobs` | `counts` | | | | | -| `successful_deployments` | `counts` | `release` | | | Total successful deployments | -| `successful_deployments` | `counts_monthly` | `release` | | | Total successful deployments last 28 days | -| `failed_deployments` | `counts` | `release` | | | Total failed deployments | -| `failed_deployments` | `counts_monthly` | `release` | | | Total failed deployments last 28 days | -| `environments` | `counts` | `release` | | | Total available and stopped environments | -| `clusters` | `counts` | `configure` | | | Total GitLab Managed clusters both enabled and disabled | -| `clusters_enabled` | `counts` | `configure` | | | Total GitLab Managed clusters currently enabled | -| `project_clusters_enabled` | `counts` | `configure` | | | Total GitLab Managed clusters attached to projects | -| `group_clusters_enabled` | `counts` | `configure` | | | Total GitLab Managed clusters attached to groups | -| `instance_clusters_enabled` | `counts` | `configure` | | | Total GitLab Managed clusters attached to the instance | -| `clusters_disabled` | `counts` | `configure` | | | Total GitLab Managed disabled clusters | -| `project_clusters_disabled` | `counts` | `configure` | | | Total GitLab Managed disabled clusters previously attached to projects | -| `group_clusters_disabled` | `counts` | `configure` | | | Total GitLab Managed disabled clusters previously attached to groups | -| `instance_clusters_disabled` | `counts` | `configure` | | | Total GitLab Managed disabled clusters previously attached to the instance | -| `clusters_platforms_eks` | `counts` | `configure` | | | Total GitLab Managed clusters provisioned with GitLab on AWS EKS | -| `clusters_platforms_gke` | `counts` | `configure` | | | Total GitLab Managed clusters provisioned with GitLab on GCE GKE | -| `clusters_platforms_user` | `counts` | `configure` | | | Total GitLab Managed clusters that are user provisioned | -| `clusters_applications_helm` | `counts` | `configure` | | | Total GitLab Managed clusters with Helm enabled | -| `clusters_applications_ingress` | `counts` | `configure` | | | Total GitLab Managed clusters with Ingress enabled | -| `clusters_applications_cert_managers` | `counts` | `configure` | | | Total GitLab Managed clusters with Cert Manager enabled | -| `clusters_applications_crossplane` | `counts` | `configure` | | | Total GitLab Managed clusters with Crossplane enabled | -| `clusters_applications_prometheus` | `counts` | `configure` | | | Total GitLab Managed clusters with Prometheus enabled | -| `clusters_applications_runner` | `counts` | `configure` | | | Total GitLab Managed clusters with Runner enabled | -| `clusters_applications_knative` | `counts` | `configure` | | | Total GitLab Managed clusters with Knative enabled | -| `clusters_applications_elastic_stack` | `counts` | `configure` | | | Total GitLab Managed clusters with Elastic Stack enabled | -| `clusters_applications_cilium` | `counts` | `configure` | | | Total GitLab Managed clusters with Cilium enabled | -| `clusters_management_project` | `counts` | `configure` | | | Total GitLab Managed clusters with defined cluster management project | -| `in_review_folder` | `counts` | | | | | -| `grafana_integrated_projects` | `counts` | | | | | -| `groups` | `counts` | | | | | -| `issues` | `counts` | | | | | -| `issues_created_from_gitlab_error_tracking_ui` | `counts` | `monitor` | | | | -| `issues_with_associated_zoom_link` | `counts` | `monitor` | | | | -| `issues_using_zoom_quick_actions` | `counts` | `monitor` | | | | -| `issues_with_embedded_grafana_charts_approx` | `counts` | `monitor` | | | | -| `issues_with_health_status` | `counts` | | | | | -| `keys` | `counts` | | | | | -| `label_lists` | `counts` | | | | | -| `lfs_objects` | `counts` | | | | | -| `milestone_lists` | `counts` | | | | | -| `milestones` | `counts` | | | | | -| `pages_domains` | `counts` | `release` | | | Total GitLab Pages domains | -| `pool_repositories` | `counts` | | | | | -| `projects` | `counts` | | | | | -| `projects_imported_from_github` | `counts` | | | | | -| `projects_with_repositories_enabled` | `counts` | | | | | -| `projects_with_error_tracking_enabled` | `counts` | `monitor` | | | | -| `protected_branches` | `counts` | | | | | -| `releases` | `counts` | `release` | | | Unique release tags | -| `remote_mirrors` | `counts` | | | | | -| `requirements_created` | `counts` | | | | | -| `snippets` | `counts` | 'create' | | CE+EE | | -| `snippets` | `counts_monthly` | 'create' | | CE+EE | | -| `personal_snippets` | `counts` | 'create' | | CE+EE | | -| `personal_snippets` | `counts_monthly` | 'create' | | CE+EE | | -| `project_snippets` | `counts` | 'create' | | CE+EE | | -| `project_snippets` | `counts_monthly` | 'create' | | CE+EE | | -| `suggestions` | `counts` | | | | | -| `todos` | `counts` | | | | | -| `uploads` | `counts` | | | | | -| `web_hooks` | `counts` | | | | | -| `projects_alerts_active` | `counts` | | | | | -| `projects_asana_active` | `counts` | | | | | -| `projects_assembla_active` | `counts` | | | | | -| `projects_bamboo_active` | `counts` | | | | | -| `projects_bugzilla_active` | `counts` | | | | | -| `projects_buildkite_active` | `counts` | | | | | -| `projects_campfire_active` | `counts` | | | | | -| `projects_custom_issue_tracker_active` | `counts` | | | | | -| `projects_discord_active` | `counts` | | | | | -| `projects_drone_ci_active` | `counts` | | | | | -| `projects_emails_on_push_active` | `counts` | | | | | -| `projects_external_wiki_active` | `counts` | | | | | -| `projects_flowdock_active` | `counts` | | | | | -| `projects_github_active` | `counts` | | | | | -| `projects_hangouts_chat_active` | `counts` | | | | | -| `projects_hipchat_active` | `counts` | | | | | -| `projects_irker_active` | `counts` | | | | | -| `projects_jenkins_active` | `counts` | | | | | -| `projects_jira_active` | `counts` | | | | | -| `projects_mattermost_active` | `counts` | | | | | -| `projects_mattermost_slash_commands_active` | `counts` | | | | | -| `projects_microsoft_teams_active` | `counts` | | | | | -| `projects_packagist_active` | `counts` | | | | | -| `projects_pipelines_email_active` | `counts` | | | | | -| `projects_pivotaltracker_active` | `counts` | | | | | -| `projects_prometheus_active` | `counts` | | | | | -| `projects_pushover_active` | `counts` | | | | | -| `projects_redmine_active` | `counts` | | | | | -| `projects_slack_active` | `counts` | | | | | -| `projects_slack_slash_commands_active` | `counts` | | | | | -| `projects_teamcity_active` | `counts` | | | | | -| `projects_unify_circuit_active` | `counts` | | | | | -| `projects_webex_teams_active` | `counts` | | | | | -| `projects_youtrack_active` | `counts` | | | | | -| `projects_jira_server_active` | `counts` | | | | | -| `projects_jira_cloud_active` | `counts` | | | | | -| `projects_jira_dvcs_cloud_active` | `counts` | | | | | -| `projects_jira_dvcs_server_active` | `counts` | | | | | -| `projects_jira_issuelist_active` | `counts` | `create` | | EE | Total Jira Issue feature enabled | -| `labels` | `counts` | | | | | -| `merge_requests` | `counts` | | | | | -| `merge_requests_users` | `counts` | | | | | -| `notes` | `counts` | | | | | -| `wiki_pages_create` | `counts` | | | | | -| `wiki_pages_update` | `counts` | | | | | -| `wiki_pages_delete` | `counts` | | | | | -| `web_ide_commits` | `counts` | | | | | -| `web_ide_views` | `counts` | | | | | -| `web_ide_merge_requests` | `counts` | | | | | -| `web_ide_previews` | `counts` | | | | | -| `snippet_comment` | `counts` | | | | | -| `commit_comment` | `counts` | | | | | -| `merge_request_comment` | `counts` | | | | | -| `snippet_create` | `counts` | | | | | -| `snippet_update` | `counts` | | | | | -| `navbar_searches` | `counts` | | | | | -| `cycle_analytics_views` | `counts` | | | | | -| `productivity_analytics_views` | `counts` | | | | | -| `source_code_pushes` | `counts` | | | | | -| `merge_request_create` | `counts` | | | | | -| `design_management_designs_create` | `counts` | | | | | -| `design_management_designs_update` | `counts` | | | | | -| `design_management_designs_delete` | `counts` | | | | | -| `licenses_list_views` | `counts` | | | | | -| `user_preferences_group_overview_details` | `counts` | | | | | -| `user_preferences_group_overview_security_dashboard` | `counts` | | | | | -| `ingress_modsecurity_logging` | `counts` | | | | | -| `ingress_modsecurity_blocking` | `counts` | | | | | -| `ingress_modsecurity_disabled` | `counts` | | | | | -| `ingress_modsecurity_not_installed` | `counts` | | | | | -| `dependency_list_usages_total` | `counts` | | | | | -| `epics` | `counts` | | | | | -| `feature_flags` | `counts` | | | | | -| `geo_nodes` | `counts` | `geo` | | | Number of sites in a Geo deployment | -| `geo_event_log_max_id` | `counts` | `geo` | | | Number of replication events on a Geo primary | -| `incident_issues` | `counts` | `monitor` | | | Issues created by the alert bot | -| `alert_bot_incident_issues` | `counts` | `monitor` | | | Issues created by the alert bot | -| `incident_labeled_issues` | `counts` | `monitor` | | | Issues with the incident label | -| `issues_created_gitlab_alerts` | `counts` | `monitor` | | | Issues created from alerts by non-alert bot users | -| `issues_created_manually_from_alerts` | `counts` | `monitor` | | | Issues created from alerts by non-alert bot users | -| `issues_created_from_alerts` | `counts` | `monitor` | | | Issues created from Prometheus and alert management alerts | -| `ldap_group_links` | `counts` | | | | | -| `ldap_keys` | `counts` | | | | | -| `ldap_users` | `counts` | | | | | -| `pod_logs_usages_total` | `counts` | | | | | -| `projects_enforcing_code_owner_approval` | `counts` | | | | | -| `projects_mirrored_with_pipelines_enabled` | `counts` | `release` | | | Projects with repository mirroring enabled | -| `projects_reporting_ci_cd_back_to_github` | `counts` | `verify` | | | Projects with a GitHub service pipeline enabled | -| `projects_with_packages` | `counts` | `package` | | | Projects with package registry configured | -| `projects_with_prometheus_alerts` | `counts` | `monitor` | | | Projects with Prometheus alerting enabled | -| `projects_with_tracing_enabled` | `counts` | `monitor` | | | Projects with tracing enabled | -| `projects_with_alerts_service_enabled` | `counts` | `monitor` | | | Projects with alerting service enabled | -| `template_repositories` | `counts` | | | | | -| `container_scanning_jobs` | `counts` | | | | | -| `dependency_scanning_jobs` | `counts` | | | | | -| `license_management_jobs` | `counts` | | | | | -| `sast_jobs` | `counts` | | | | | -| `status_page_projects` | `counts` | `monitor` | | | Projects with status page enabled | -| `status_page_issues` | `counts` | `monitor` | | | Issues published to a Status Page | -| `status_page_incident_publishes` | `counts` | `monitor` | | | Cumulative count of usages of publish operation | -| `status_page_incident_unpublishes` | `counts` | `monitor` | | | Cumulative count of usages of unpublish operation | -| `epics_deepest_relationship_level` | `counts` | | | | | -| `operations_dashboard_default_dashboard` | `counts` | `monitor` | | | Active users with enabled operations dashboard | -| `operations_dashboard_users_with_projects_added` | `counts` | `monitor` | | | Active users with projects on operations dashboard | -| `container_registry_enabled` | | | | | | -| `dependency_proxy_enabled` | | | | | | -| `gitlab_shared_runners_enabled` | | | | | | -| `gravatar_enabled` | | | | | | -| `ldap_enabled` | | | | | | -| `mattermost_enabled` | | | | | | -| `omniauth_enabled` | | | | | | -| `prometheus_enabled` | | | | | Whether the bundled Prometheus is enabled | -| `prometheus_metrics_enabled` | | | | | | -| `reply_by_email_enabled` | | | | | | -| `average` | `avg_cycle_analytics - code` | | | | | -| `sd` | `avg_cycle_analytics - code` | | | | | -| `missing` | `avg_cycle_analytics - code` | | | | | -| `average` | `avg_cycle_analytics - test` | | | | | -| `sd` | `avg_cycle_analytics - test` | | | | | -| `missing` | `avg_cycle_analytics - test` | | | | | -| `average` | `avg_cycle_analytics - review` | | | | | -| `sd` | `avg_cycle_analytics - review` | | | | | -| `missing` | `avg_cycle_analytics - review` | | | | | -| `average` | `avg_cycle_analytics - staging` | | | | | -| `sd` | `avg_cycle_analytics - staging` | | | | | -| `missing` | `avg_cycle_analytics - staging` | | | | | -| `average` | `avg_cycle_analytics - production` | | | | | -| `sd` | `avg_cycle_analytics - production` | | | | | -| `missing` | `avg_cycle_analytics - production` | | | | | -| `total` | `avg_cycle_analytics` | | | | | -| `g_analytics_contribution` | `analytics_unique_visits` | `manage` | | | Visits to /groups/:group/-/contribution_analytics | -| `g_analytics_insights` | `analytics_unique_visits` | `manage` | | | Visits to /groups/:group/-/insights | -| `g_analytics_issues` | `analytics_unique_visits` | `manage` | | | Visits to /groups/:group/-/issues_analytics | -| `g_analytics_productivity` | `analytics_unique_visits` | `manage` | | | Visits to /groups/:group/-/analytics/productivity_analytics | -| `g_analytics_valuestream` | `analytics_unique_visits` | `manage` | | | Visits to /groups/:group/-/analytics/value_stream_analytics | -| `p_analytics_pipelines` | `analytics_unique_visits` | `manage` | | | Visits to /:group/:project/pipelines/charts | -| `p_analytics_code_reviews` | `analytics_unique_visits` | `manage` | | | Visits to /:group/:project/-/analytics/code_reviews | -| `p_analytics_valuestream` | `analytics_unique_visits` | `manage` | | | Visits to /:group/:project/-/value_stream_analytics | -| `p_analytics_insights` | `analytics_unique_visits` | `manage` | | | Visits to /:group/:project/insights | -| `p_analytics_issues` | `analytics_unique_visits` | `manage` | | | Visits to /:group/:project/-/analytics/issues_analytics | -| `p_analytics_repo` | `analytics_unique_visits` | `manage` | | | Visits to /:group/:project/-/graphs/master/charts | -| `u_analytics_todos` | `analytics_unique_visits` | `manage` | | | Visits to /dashboard/todos | -| `i_analytics_cohorts` | `analytics_unique_visits` | `manage` | | | Visits to /-/instance_statistics/cohorts | -| `i_analytics_dev_ops_score` | `analytics_unique_visits` | `manage` | | | Visits to /-/instance_statistics/dev_ops_score | -| `analytics_unique_visits_for_any_target` | `analytics_unique_visits` | `manage` | | | Visits to any of the pages listed above | -| `clusters_applications_cert_managers` | `usage_activity_by_stage` | `configure` | | CE+EE | Unique clusters with certificate managers enabled | -| `clusters_applications_helm` | `usage_activity_by_stage` | `configure` | | CE+EE | Unique clusters with Helm enabled | -| `clusters_applications_ingress` | `usage_activity_by_stage` | `configure` | | CE+EE | Unique clusters with Ingress enabled | -| `clusters_applications_knative` | `usage_activity_by_stage` | `configure` | | CE+EE | Unique clusters with Knative enabled | -| `clusters_management_project` | `usage_activity_by_stage` | `configure` | | CE+EE | Unique clusters with project management enabled | -| `clusters_disabled` | `usage_activity_by_stage` | `configure` | | CE+EE | Total non-"GitLab Managed clusters" | -| `clusters_enabled` | `usage_activity_by_stage` | `configure` | | CE+EE | Total GitLab Managed clusters | -| `clusters_platforms_gke` | `usage_activity_by_stage` | `configure` | | CE+EE | Unique clusters with Google Cloud installed | -| `clusters_platforms_eks` | `usage_activity_by_stage` | `configure` | | CE+EE | Unique clusters with AWS installed | -| `clusters_platforms_user` | `usage_activity_by_stage` | `configure` | | CE+EE | Unique clusters that are user provided | -| `instance_clusters_disabled` | `usage_activity_by_stage` | `configure` | | CE+EE | Unique clusters disabled on instance | -| `instance_clusters_enabled` | `usage_activity_by_stage` | `configure` | | CE+EE | Unique clusters enabled on instance | -| `group_clusters_disabled` | `usage_activity_by_stage` | `configure` | | CE+EE | Unique clusters disabled on group | -| `group_clusters_enabled` | `usage_activity_by_stage` | `configure` | | CE+EE | Unique clusters enabled on group | -| `project_clusters_disabled` | `usage_activity_by_stage` | `configure` | | CE+EE | Unique clusters disabled on project | -| `project_clusters_enabled` | `usage_activity_by_stage` | `configure` | | CE+EE | Unique clusters enabled on project | -| `projects_slack_notifications_active` | `usage_activity_by_stage` | `configure` | | EE | Unique projects with Slack service enabled | -| `projects_slack_slash_active` | `usage_activity_by_stage` | `configure` | | EE | Unique projects with Slack '/' commands enabled | -| `projects_with_prometheus_alerts` | `usage_activity_by_stage` | `configure` | | EE | Projects with Prometheus enabled and no alerts | -| `deploy_keys` | `usage_activity_by_stage` | `create` | | CE+EE | | -| `keys` | `usage_activity_by_stage` | `create` | | CE+EE | | -| `merge_requests` | `usage_activity_by_stage` | `create` | | CE+EE | | -| `projects_with_disable_overriding_approvers_per_merge_request` | `usage_activity_by_stage` | `create` | | CE+EE | | -| `projects_without_disable_overriding_approvers_per_merge_request` | `usage_activity_by_stage` | `create` | | CE+EE | | -| `remote_mirrors` | `usage_activity_by_stage` | `create` | | CE+EE | | -| `snippets` | `usage_activity_by_stage` | `create` | | CE+EE | | -| `merge_requests_users` | `usage_activity_by_stage_monthly` | `create` | | CE+EE | Unique count of users who used a merge request | -| `action_monthly_active_users_project_repo` | `usage_activity_by_stage_monthly` | `create` | | CE+EE | Unique count of users who pushed to a project repo | -| `action_monthly_active_users_design_management` | `usage_activity_by_stage_monthly` | `create` | | CE+EE | Unique count of users who interacted with the design system management | -| `action_monthly_active_users_wiki_repo` | `usage_activity_by_stage_monthly` | `create` | | CE+EE | Unique count of users who created or updated a wiki repo | -| `projects_enforcing_code_owner_approval` | `usage_activity_by_stage` | `create` | | EE | | -| `merge_requests_with_optional_codeowners` | `usage_activity_by_stage` | `create` | | EE | | -| `merge_requests_with_required_codeowners` | `usage_activity_by_stage` | `create` | | EE | | -| `projects_imported_from_github` | `usage_activity_by_stage` | `create` | | EE | | -| `projects_with_repositories_enabled` | `usage_activity_by_stage` | `create` | | EE | | -| `protected_branches` | `usage_activity_by_stage` | `create` | | EE | | -| `suggestions` | `usage_activity_by_stage` | `create` | | EE | | -| `approval_project_rules` | `usage_activity_by_stage` | `create` | | EE | Number of project approval rules | -| `approval_project_rules_with_target_branch` | `usage_activity_by_stage` | `create` | | EE | Number of project approval rules with not default target branch | -| `merge_requests_with_added_rules` | `usage_activity_by_stage` | `create` | | EE | Merge Requests with added rules | -| `clusters` | `usage_activity_by_stage` | `monitor` | | CE+EE | | -| `clusters_applications_prometheus` | `usage_activity_by_stage` | `monitor` | | CE+EE | | -| `operations_dashboard_default_dashboard` | `usage_activity_by_stage` | `monitor` | | CE+EE | | -| `operations_dashboard_users_with_projects_added` | `usage_activity_by_stage` | `monitor` | | EE | | -| `projects_prometheus_active` | `usage_activity_by_stage` | `monitor` | | EE | | -| `projects_with_error_tracking_enabled` | `usage_activity_by_stage` | `monitor` | | EE | | -| `projects_with_tracing_enabled` | `usage_activity_by_stage` | `monitor` | | EE | | -| `events` | `usage_activity_by_stage` | `manage` | | CE+EE | | -| `groups` | `usage_activity_by_stage` | `manage` | | CE+EE | | -| `users_created_at` | `usage_activity_by_stage` | `manage` | | CE+EE | | -| `omniauth_providers` | `usage_activity_by_stage` | `manage` | | CE+EE | | -| `ldap_keys` | `usage_activity_by_stage` | `manage` | | EE | | -| `ldap_users` | `usage_activity_by_stage` | `manage` | | EE | | -| `value_stream_management_customized_group_stages` | `usage_activity_by_stage` | `manage` | | EE | | -| `projects_with_compliance_framework` | `usage_activity_by_stage` | `manage` | | EE | | -| `ldap_servers` | `usage_activity_by_stage` | `manage` | | EE | | -| `ldap_group_sync_enabled` | `usage_activity_by_stage` | `manage` | | EE | | -| `ldap_admin_sync_enabled` | `usage_activity_by_stage` | `manage` | | EE | | -| `group_saml_enabled` | `usage_activity_by_stage` | `manage` | | EE | | -| `issues` | `usage_activity_by_stage` | `plan` | | CE+EE | | -| `notes` | `usage_activity_by_stage` | `plan` | | CE+EE | | -| `projects` | `usage_activity_by_stage` | `plan` | | CE+EE | | -| `todos` | `usage_activity_by_stage` | `plan` | | CE+EE | | -| `assignee_lists` | `usage_activity_by_stage` | `plan` | | EE | | -| `epics` | `usage_activity_by_stage` | `plan` | | EE | | -| `label_lists` | `usage_activity_by_stage` | `plan` | | EE | | -| `milestone_lists` | `usage_activity_by_stage` | `plan` | | EE | | -| `projects_jira_active` | `usage_activity_by_stage` | `plan` | | EE | | -| `projects_jira_dvcs_server_active` | `usage_activity_by_stage` | `plan` | | EE | | -| `projects_jira_dvcs_server_active` | `usage_activity_by_stage` | `plan` | | EE | | -| `service_desk_enabled_projects` | `usage_activity_by_stage` | `plan` | | CE+EE | | -| `service_desk_issues` | `usage_activity_by_stage` | `plan` | | CE+EE | | -| `deployments` | `usage_activity_by_stage` | `release` | | CE+EE | Total deployments | -| `failed_deployments` | `usage_activity_by_stage` | `release` | | CE+EE | Total failed deployments | -| `projects_mirrored_with_pipelines_enabled` | `usage_activity_by_stage` | `release` | | EE | Projects with repository mirroring enabled | -| `releases` | `usage_activity_by_stage` | `release` | | CE+EE | Unique release tags in project | -| `successful_deployments` | `usage_activity_by_stage` | `release` | | CE+EE | Total successful deployments | -| `user_preferences_group_overview_security_dashboard` | `usage_activity_by_stage` | `secure` | | | | -| `ci_builds` | `usage_activity_by_stage` | `verify` | | CE+EE | Unique builds in project | -| `ci_external_pipelines` | `usage_activity_by_stage` | `verify` | | CE+EE | Total pipelines in external repositories | -| `ci_internal_pipelines` | `usage_activity_by_stage` | `verify` | | CE+EE | Total pipelines in GitLab repositories | -| `ci_pipeline_config_auto_devops` | `usage_activity_by_stage` | `verify` | | CE+EE | Total pipelines from an Auto DevOps template | -| `ci_pipeline_config_repository` | `usage_activity_by_stage` | `verify` | | CE+EE | Pipelines from templates in repository | -| `ci_pipeline_schedules` | `usage_activity_by_stage` | `verify` | | CE+EE | Pipeline schedules in GitLab | -| `ci_pipelines` | `usage_activity_by_stage` | `verify` | | CE+EE | Total pipelines | -| `ci_triggers` | `usage_activity_by_stage` | `verify` | | CE+EE | Triggers enabled | -| `clusters_applications_runner` | `usage_activity_by_stage` | `verify` | | CE+EE | Unique clusters with Runner enabled | -| `projects_reporting_ci_cd_back_to_github` | `usage_activity_by_stage` | `verify` | | EE | Unique projects with a GitHub pipeline enabled | -| `merge_requests_users` | `usage_activity_by_stage_monthly` | `create` | | | Unique count of users who used a merge request | -| `duration_s` | `topology` | `enablement` | | | Time it took to collect topology data | -| `application_requests_per_hour` | `topology` | `enablement` | | | Number of requests to the web application per hour | -| `failures` | `topology` | `enablement` | | | Contains information about failed queries | -| `nodes` | `topology` | `enablement` | | | The list of server nodes on which GitLab components are running | -| `node_memory_total_bytes` | `topology > nodes` | `enablement` | | | The total available memory of this node | -| `node_cpus` | `topology > nodes` | `enablement` | | | The number of CPU cores of this node | -| `node_uname_info` | `topology > nodes` | `enablement` | | | The basic hardware architecture and OS release information on this node | -| `node_services` | `topology > nodes` | `enablement` | | | The list of GitLab services running on this node | -| `name` | `topology > nodes > node_services` | `enablement` | | | The name of the GitLab service running on this node | -| `process_count` | `topology > nodes > node_services` | `enablement` | | | The number of processes running for this service | -| `process_memory_rss` | `topology > nodes > node_services` | `enablement` | | | The average Resident Set Size of a service process | -| `process_memory_uss` | `topology > nodes > node_services` | `enablement` | | | The average Unique Set Size of a service process | -| `process_memory_pss` | `topology > nodes > node_services` | `enablement` | | | The average Proportional Set Size of a service process | -| `server` | `topology > nodes > node_services` | `enablement` | | | The type of web server used (Unicorn or Puma) | -| `network_policy_forwards` | `counts` | `defend` | | EE | Cumulative count of forwarded packets by Container Network | -| `network_policy_drops` | `counts` | `defend` | | EE | Cumulative count of dropped packets by Container Network | - ## Example Usage Ping payload The following is example content of the Usage Ping payload. @@ -811,13 +500,14 @@ The following is example content of the Usage Ping payload. "enabled": true, "version": "1.17.0" }, + "container_registry_server": { + "vendor": "gitlab", + "version": "2.9.1-gitlab" + }, "database": { "adapter": "postgresql", "version": "9.6.15" }, - "app_server": { - "type": "console" - }, "avg_cycle_analytics": { "issue": { "average": 999, @@ -941,7 +631,9 @@ The following is example content of the Usage Ping payload. "nodes": [ { "node_memory_total_bytes": 33269903360, + "node_memory_utilization": 0.35, "node_cpus": 16, + "node_cpu_utilization": 0.2, "node_uname_info": { "machine": "x86_64", "sysname": "Linux", diff --git a/doc/development/testing_guide/best_practices.md b/doc/development/testing_guide/best_practices.md index 4e46e691405..b60a26c29b5 100644 --- a/doc/development/testing_guide/best_practices.md +++ b/doc/development/testing_guide/best_practices.md @@ -57,7 +57,7 @@ bundle exec guard When using spring and guard together, use `SPRING=1 bundle exec guard` instead to make use of spring. -Use [Factory Doctor](https://test-prof.evilmartians.io/#/factory_doctor.md) to find cases on un-necessary database manipulation, which can cause slow tests. +Use [Factory Doctor](https://test-prof.evilmartians.io/#/profilers/factory_doctor) to find cases on un-necessary database manipulation, which can cause slow tests. ```shell # run test for path @@ -261,8 +261,8 @@ As much as possible, do not implement this using `before(:all)` or `before(:cont you would need to manually clean up the data as those hooks run outside a database transaction. Instead, this can be achieved by using -[`let_it_be`](https://test-prof.evilmartians.io/#/let_it_be) variables and the -[`before_all`](https://test-prof.evilmartians.io/#/before_all) hook +[`let_it_be`](https://test-prof.evilmartians.io/#/recipes/let_it_be) variables and the +[`before_all`](https://test-prof.evilmartians.io/#/recipes/before_all) hook from the [`test-prof` gem](https://rubygems.org/gems/test-prof). ```ruby @@ -315,109 +315,7 @@ end ### Feature flags in tests -All feature flags are stubbed to be enabled by default in our Ruby-based -tests. - -To disable a feature flag in a test, use the `stub_feature_flags` -helper. For example, to globally disable the `ci_live_trace` feature -flag in a test: - -```ruby -stub_feature_flags(ci_live_trace: false) - -Feature.enabled?(:ci_live_trace) # => false -``` - -If you wish to set up a test where a feature flag is enabled only -for some actors and not others, you can specify this in options -passed to the helper. For example, to enable the `ci_live_trace` -feature flag for a specific project: - -```ruby -project1, project2 = build_list(:project, 2) - -# Feature will only be enabled for project1 -stub_feature_flags(ci_live_trace: project1) - -Feature.enabled?(:ci_live_trace) # => false -Feature.enabled?(:ci_live_trace, project1) # => true -Feature.enabled?(:ci_live_trace, project2) # => false -``` - -This represents an actual behavior of FlipperGate: - -1. You can enable an override for a specified actor to be enabled -1. You can disable (remove) an override for a specified actor, - falling back to default state -1. There's no way to model that you explicitly disable a specified actor - -```ruby -Feature.enable(:my_feature) -Feature.disable(:my_feature, project1) -Feature.enabled?(:my_feature) # => true -Feature.enabled?(:my_feature, project1) # => true -``` - -```ruby -Feature.disable(:my_feature2) -Feature.enable(:my_feature2, project1) -Feature.enabled?(:my_feature2) # => false -Feature.enabled?(:my_feature2, project1) # => true -``` - -#### `stub_feature_flags` vs `Feature.enable*` - -It is preferred to use `stub_feature_flags` for enabling feature flags -in testing environment. This method provides a simple and well described -interface for a simple use-cases. - -However, in some cases a more complex behaviors needs to be tested, -like a feature flag percentage rollouts. This can be achieved using -the `.enable_percentage_of_time` and `.enable_percentage_of_actors` - -```ruby -# Good: feature needs to be explicitly disabled, as it is enabled by default if not defined -stub_feature_flags(my_feature: false) -stub_feature_flags(my_feature: true) -stub_feature_flags(my_feature: project) -stub_feature_flags(my_feature: [project, project2]) - -# Bad -Feature.enable(:my_feature_2) - -# Good: enable my_feature for 50% of time -Feature.enable_percentage_of_time(:my_feature_3, 50) - -# Good: enable my_feature for 50% of actors/gates/things -Feature.enable_percentage_of_actors(:my_feature_4, 50) -``` - -Each feature flag that has a defined state will be persisted -for test execution time: - -```ruby -Feature.persisted_names.include?('my_feature') => true -Feature.persisted_names.include?('my_feature_2') => true -Feature.persisted_names.include?('my_feature_3') => true -Feature.persisted_names.include?('my_feature_4') => true -``` - -#### Stubbing gate - -It is required that a gate that is passed as an argument to `Feature.enabled?` -and `Feature.disabled?` is an object that includes `FeatureGate`. - -In specs you can use a `stub_feature_flag_gate` method that allows you to have -quickly your custom gate: - -```ruby -gate = stub_feature_flag_gate('CustomActor') - -stub_feature_flags(ci_live_trace: gate) - -Feature.enabled?(:ci_live_trace) # => false -Feature.enabled?(:ci_live_trace, gate) # => true -``` +This section was moved to [developing with feature flags](../feature_flags/development.md). ### Pristine test environments diff --git a/doc/development/testing_guide/end_to_end/best_practices.md b/doc/development/testing_guide/end_to_end/best_practices.md index 7df3cd614c7..3b193721143 100644 --- a/doc/development/testing_guide/end_to_end/best_practices.md +++ b/doc/development/testing_guide/end_to_end/best_practices.md @@ -55,6 +55,33 @@ Project::Issues::Index.perform do |index| end ``` +## Prefer `aggregate_failures` when there are back-to-back expectations + +In cases where there must be multiple (back-to-back) expectations within a test case, it is preferable to use `aggregate_failures`. + +This allows you to group a set of expectations and see all the failures altogether, rather than having the test being aborted on the first failure. + +For example: + +```ruby +#=> Good +Page::Search::Results.perform do |search| + search.switch_to_code + + aggregate_failures 'testing search results' do + expect(search).to have_file_in_project(template[:file_name], project.name) + expect(search).to have_file_with_content(template[:file_name], content[0..33]) + end +end + +#=> Bad +Page::Search::Results.perform do |search| + search.switch_to_code + expect(search).to have_file_in_project(template[:file_name], project.name) + expect(search).to have_file_with_content(template[:file_name], content[0..33]) +end +``` + ## Prefer to split tests across multiple files Our framework includes a couple of parallelization mechanisms that work by executing spec files in parallel. diff --git a/doc/development/testing_guide/end_to_end/index.md b/doc/development/testing_guide/end_to_end/index.md index ac051b827d2..f61eab5c8f3 100644 --- a/doc/development/testing_guide/end_to_end/index.md +++ b/doc/development/testing_guide/end_to_end/index.md @@ -178,6 +178,13 @@ Once you decided where to put [test environment orchestration scenarios](https:/ the [GitLab QA orchestrator README](https://gitlab.com/gitlab-org/gitlab-qa/tree/master/README.md), and [the already existing instance-level scenarios](https://gitlab.com/gitlab-org/gitlab-foss/tree/master/qa/qa/specs/features). +### Consider **not** writing an end-to-end test + +We should follow these best practices for end-to-end tests: + +- Do not write an end-to-end test if a lower-level feature test exists. End-to-end tests require more work and resources. +- Troubleshooting for end-to-end tests can be more complex as connections to the application under test are not known. + Continued reading: - [Beginner's Guide](beginners_guide.md) diff --git a/doc/development/testing_guide/end_to_end/page_objects.md b/doc/development/testing_guide/end_to_end/page_objects.md index d43d88779c7..6ce44b2d359 100644 --- a/doc/development/testing_guide/end_to_end/page_objects.md +++ b/doc/development/testing_guide/end_to_end/page_objects.md @@ -4,22 +4,22 @@ In GitLab QA we are using a known pattern, called _Page Objects_. This means that we have built an abstraction for all pages in GitLab that we use to drive GitLab QA scenarios. Whenever we do something on a page, like filling -in a form, or clicking a button, we do that only through a page object +in a form or clicking a button, we do that only through a page object associated with this area of GitLab. For example, when GitLab QA test harness signs in into GitLab, it needs to fill -in a user login and user password. In order to do that, we have a class, called +in user login and user password. To do that, we have a class, called `Page::Main::Login` and `sign_in_using_credentials` methods, that is the only -piece of the code, that has knowledge about `user_login` and `user_password` +piece of the code, that reads the `user_login` and `user_password` fields. ## Why do we need that? -We need page objects, because we need to reduce duplication and avoid problems +We need page objects because we need to reduce duplication and avoid problems whenever someone changes some selectors in GitLab's source code. Imagine that we have a hundred specs in GitLab QA, and we need to sign into -GitLab each time, before we make assertions. Without a page object one would +GitLab each time, before we make assertions. Without a page object, one would need to rely on volatile helpers or invoke Capybara methods directly. Imagine invoking `fill_in :user_login` in every `*_spec.rb` file / test example. @@ -28,7 +28,7 @@ this page to `t.text_field :username` it will generate a different field identifier, what would effectively break all tests. Because we are using `Page::Main::Login.perform(&:sign_in_using_credentials)` -everywhere, when we want to sign into GitLab, the page object is the single +everywhere, when we want to sign in to GitLab, the page object is the single source of truth, and we will need to update `fill_in :user_login` to `fill_in :user_username` only in a one place. @@ -42,23 +42,23 @@ That is why when someone changes `t.text_field :login` to change until our GitLab QA nightly pipeline fails, or until someone triggers `package-and-qa` action in their merge request. -Obviously such a change would break all tests. We call this problem a _fragile +Such a change would break all tests. We call this problem a _fragile tests problem_. -In order to make GitLab QA more reliable and robust, we had to solve this +To make GitLab QA more reliable and robust, we had to solve this problem by introducing coupling between GitLab CE / EE views and GitLab QA. ## How did we solve fragile tests problem? Currently, when you add a new `Page::Base` derived class, you will also need to -define all selectors that your page objects depends on. +define all selectors that your page objects depend on. Whenever you push your code to CE / EE repository, `qa:selectors` sanity test job is going to be run as a part of a CI pipeline. This test is going to validate all page objects that we have implemented in `qa/page` directory. When it fails, you will be notified about missing -or invalid views / selectors definition. +or invalid views/selectors definition. ## How to properly implement a page object? @@ -89,7 +89,7 @@ end ### Defining Elements -The `view` DSL method will correspond to the rails View, partial, or Vue component that renders the elements. +The `view` DSL method will correspond to the Rails view, partial, or Vue component that renders the elements. The `element` DSL method in turn declares an element for which a corresponding `data-qa-selector=element_name_snaked` data attribute will need to be added to the view file. @@ -134,7 +134,7 @@ view 'app/views/my/view.html.haml' do end ``` -To add these elements to the view, you must change the rails View, partial, or Vue component by adding a `data-qa-selector` attribute +To add these elements to the view, you must change the Rails view, partial, or Vue component by adding a `data-qa-selector` attribute for each element defined. In our case, `data-qa-selector="login_field"`, `data-qa-selector="password_field"` and `data-qa-selector="sign_in_button"` @@ -228,7 +228,7 @@ expect(the_page).to have_element(:model, index: 1) #=> select on the first model ### Exceptions -In some cases it might not be possible or worthwhile to add a selector. +In some cases, it might not be possible or worthwhile to add a selector. Some UI components use external libraries, including some maintained by third parties. Even if a library is maintained by GitLab, the selector sanity test only runs diff --git a/doc/development/testing_guide/end_to_end/running_tests_that_require_special_setup.md b/doc/development/testing_guide/end_to_end/running_tests_that_require_special_setup.md index 77d820e1686..2cf2bb5b1d0 100644 --- a/doc/development/testing_guide/end_to_end/running_tests_that_require_special_setup.md +++ b/doc/development/testing_guide/end_to_end/running_tests_that_require_special_setup.md @@ -48,3 +48,89 @@ only to prevent it from running in the pipelines for live environments such as S If Jenkins Docker container exits without providing any information in the logs, try increasing the memory used by the Docker Engine. + +## Gitaly Cluster tests + +The tests tagged `:gitaly_ha` are orchestrated tests that can only be run against a set of Docker containers as configured and started by [the `Test::Integration::GitalyCluster` GitLab QA scenario](https://gitlab.com/gitlab-org/gitlab-qa/-/blob/master/docs/what_tests_can_be_run.md#testintegrationgitalycluster-ceeefull-image-address). + +As described in the documentation about the scenario noted above, the following command will run the tests: + +```shell +gitlab-qa Test::Integration::GitalyCluster EE +``` + +However, that will remove the containers after it finishes running the tests. If you would like to do further testing, for example, if you would like to run a single test via a debugger, you can use [the `--no-tests` option](https://gitlab.com/gitlab-org/gitlab-qa#command-line-options) to make `gitlab-qa` skip running the tests, and to leave the containers running so that you can continue to use them. + +```shell +gitlab-qa Test::Integration::GitalyCluster EE --no-tests +``` + +When all the containers are running you will see the output of the `docker ps` command, showing on which ports the GitLab container can be accessed. For example: + +```plaintext +CONTAINER ID ... PORTS NAMES +d15d3386a0a8 ... 22/tcp, 443/tcp, 0.0.0.0:32772->80/tcp gitlab-gitaly-ha +``` + +That shows that the GitLab instance running in the `gitlab-gitaly-ha` container can be reached via `http://localhost:32772`. However, Git operations like cloning and pushing are performed against the URL revealed via the UI as the clone URL. It uses the hostname configured for the GitLab instance, which in this case matches the Docker container name and network, `gitlab-gitaly-ha.test`. Before you can run the tests you need to configure your computer to access the container via that address. One option is to [use caddyserver as described for running tests against GDK](https://gitlab.com/gitlab-org/gitlab-qa/-/blob/master/docs/run_qa_against_gdk.md#workarounds). + +Another option is to use NGINX. + +In both cases you will need to configure your machine to translate `gitlab-gitlab-ha.test` into an appropriate IP address: + +```shell +echo '127.0.0.1 gitlab-gitaly-ha.test' | sudo tee -a /etc/hosts +``` + +Then install NGINX: + +```shell +# on macOS +brew install nginx + +# on Debian/Ubuntu +apt install nginx + +# on Fedora +yum install nginx +``` + +Finally, configure NGINX to pass requests for `gitlab-gitaly-ha.test` to the GitLab instance: + +```plaintext +# On Debian/Ubuntu, in /etc/nginx/sites-enabled/gitlab-cluster +# On macOS, in /usr/local/etc/nginx/nginx.conf + +server { + server_name gitlab-gitaly-ha.test; + client_max_body_size 500m; + + location / { + proxy_pass http://127.0.0.1:32772; + proxy_set_header Host gitlab-gitaly-ha.test; + } +} +``` + +Restart NGINX for the configuration to take effect. For example: + +```shell +# On Debian/Ubuntu +sudo systemctl restart nginx + +# on macOS +sudo nginx -s reload +``` + +You could then run the tests from the `/qa` directory: + +```shell +CHROME_HEADLESS=false bin/qa Test::Instance::All http://gitlab-gitaly-ha.test -- --tag gitaly_ha +``` + +Once you have finished testing you can stop and remove the Docker containers: + +```shell +docker stop gitlab-gitaly-ha praefect postgres gitaly3 gitaly2 gitaly1 +docker rm gitlab-gitaly-ha praefect postgres gitaly3 gitaly2 gitaly1 +``` diff --git a/doc/development/testing_guide/frontend_testing.md b/doc/development/testing_guide/frontend_testing.md index ef9fd748dbb..42ca65a74f2 100644 --- a/doc/development/testing_guide/frontend_testing.md +++ b/doc/development/testing_guide/frontend_testing.md @@ -24,9 +24,8 @@ We have started to migrate frontend tests to the [Jest](https://jestjs.io) testi Jest tests can be found in `/spec/frontend` and `/ee/spec/frontend` in EE. -> **Note:** -> -> Most examples have a Jest and Karma example. See the Karma examples only as explanation to what's going on in the code, should you stumble over some use cases during your discovery. The Jest examples are the one you should follow. +NOTE: **Note:** +Most examples have a Jest and Karma example. See the Karma examples only as explanation to what's going on in the code, should you stumble over some use cases during your discovery. The Jest examples are the one you should follow. ## Karma test suite @@ -170,22 +169,14 @@ Some more examples can be found in the [Frontend unit tests section](testing_lev Another common gotcha is that the specs end up verifying the mock is working. If you are using mocks, the mock should support the test, but not be the target of the test. -**Bad:** - ```javascript const spy = jest.spyOn(idGenerator, 'create') spy.mockImplementation = () = '1234' +// Bad expect(idGenerator.create()).toBe('1234') -``` - -**Good:** - -```javascript -const spy = jest.spyOn(idGenerator, 'create') -spy.mockImplementation = () = '1234' -// Actually focusing on the logic of your component and just leverage the controllable mocks output +// Good: actually focusing on the logic of your component and just leverage the controllable mocks output expect(wrapper.find('div').html()).toBe('<div id="1234">...</div>') ``` @@ -204,29 +195,67 @@ Following you'll find some general common practices you will find as part of our ### How to query DOM elements -When it comes to querying DOM elements in your tests, it is best to uniquely and semantically target the element. Sometimes this cannot be done feasibly. In these cases, adding test attributes to simplify the selectors might be the best option. +When it comes to querying DOM elements in your tests, it is best to uniquely and semantically target +the element. + +Preferentially, this is done by targeting text the user actually sees using [DOM Testing Library](https://testing-library.com/docs/dom-testing-library/intro). +When selecting by text it is best to use [`getByRole` or `findByRole`](https://testing-library.com/docs/dom-testing-library/api-queries#byrole) +as these enforce accessibility best practices as well. The examples below demonstrate the order of preference. -Preferentially, in component testing with `@vue/test-utils`, you should query for child components using the component itself. This helps enforce that specific behavior can be covered by that component's individual unit tests. Otherwise, try to use: +Sometimes this cannot be done feasibly. In these cases, adding test attributes to simplify the +selectors might be the best option. - A semantic attribute like `name` (also verifies that `name` was setup properly) - A `data-testid` attribute ([recommended by maintainers of `@vue/test-utils`](https://github.com/vuejs/vue-test-utils/issues/1498#issuecomment-610133465)) - a Vue `ref` (if using `@vue/test-utils`) -Examples: - ```javascript +import { mount, shallowMount } from '@vue/test-utils' +import { getByRole, getByText } from '@testing-library/dom' + +let wrapper +let el + +const createComponent = (mountFn = shallowMount) => { + wrapper = mountFn(Component) + el = wrapper.vm.$el // reference to the container element +} + +beforeEach(() => { + createComponent() +}) + + +it('exists', () => { + // Best + + // NOTE: both mount and shallowMount work as long as a DOM element is available + // Finds a properly formatted link with an accessable name of "Click Me" + getByRole(el, 'link', { name: /Click Me/i }) + getByRole(el, 'link', { name: 'Click Me' }) + // Finds any element with the text "Click Me" + getByText(el, 'Click Me') + // Regex is also available + getByText(el, /Click Me/i) + + // Good + wrapper.find('input[name=foo]'); + wrapper.find('[data-testid="foo"]'); + wrapper.find({ ref: 'foo'}); + + // Bad + wrapper.find('.js-foo'); + wrapper.find('.btn-primary'); + wrapper.find('.qa-foo-component'); + wrapper.find('[data-qa-selector="foo"]'); +}); + +// Good it('exists', () => { - // Good wrapper.find(FooComponent); wrapper.find('input[name=foo]'); wrapper.find('[data-testid="foo"]'); wrapper.find({ ref: 'foo'}); - - // Bad - wrapper.find('.js-foo'); - wrapper.find('.btn-primary'); - wrapper.find('.qa-foo-component'); - wrapper.find('[data-qa-selector="foo"]'); }); ``` @@ -234,28 +263,47 @@ It is not recommended that you add `.js-*` classes just for testing purposes. On Do not use a `.qa-*` class or `data-qa-selector` attribute for any tests other than QA end-to-end testing. +### Querying for child components + +When testing Vue components with `@vue/test-utils` another possible approach is querying for child +components instead of querying for DOM nodes. This assumes that implementation details of behavior +under test should be covered by that component's individual unit test. There is no strong preference +in writing DOM or component queries as long as your tests reliably cover expected behavior for the +component under test. + +Example: + +```javascript +it('exists', () => { + wrapper.find(FooComponent); +}); +``` + ### Naming unit tests When writing describe test blocks to test specific functions/methods, please use the method name as the describe block name. +**Bad**: + ```javascript -// Good -describe('methodName', () => { +describe('#methodName', () => { it('passes', () => { expect(true).toEqual(true); }); }); -// Bad -describe('#methodName', () => { +describe('.methodName', () => { it('passes', () => { expect(true).toEqual(true); }); }); +``` -// Bad -describe('.methodName', () => { +**Good**: + +```javascript +describe('methodName', () => { it('passes', () => { expect(true).toEqual(true); }); @@ -286,61 +334,67 @@ it('tests a promise rejection', async () => { You can also work with Promise chains. In this case, you can make use of the `done` callback and `done.fail` in case an error occurred. Following are some examples: +**Bad**: + ```javascript -// Good +// missing done callback +it('tests a promise', () => { + promise.then(data => { + expect(data).toBe(asExpected); + }); +}); + +// missing catch it('tests a promise', done => { promise .then(data => { expect(data).toBe(asExpected); }) - .then(done) - .catch(done.fail); + .then(done); }); -// Good -it('tests a promise rejection', done => { +// use done.fail in asynchronous tests +it('tests a promise', done => { promise - .then(done.fail) - .catch(error => { - expect(error).toBe(expectedError); + .then(data => { + expect(data).toBe(asExpected); }) .then(done) - .catch(done.fail); -}); - -// Bad (missing done callback) -it('tests a promise', () => { - promise.then(data => { - expect(data).toBe(asExpected); - }); + .catch(fail); }); -// Bad (missing catch) -it('tests a promise', done => { +// missing catch +it('tests a promise rejection', done => { promise - .then(data => { - expect(data).toBe(asExpected); + .catch(error => { + expect(error).toBe(expectedError); }) .then(done); }); +``` -// Bad (use done.fail in asynchronous tests) +**Good**: + +```javascript +// handling success it('tests a promise', done => { promise .then(data => { expect(data).toBe(asExpected); }) .then(done) - .catch(fail); + .catch(done.fail); }); -// Bad (missing catch) +// failure case it('tests a promise rejection', done => { promise + .then(done.fail) .catch(error => { expect(error).toBe(expectedError); }) - .then(done); + .then(done) + .catch(done.fail); }); ``` @@ -564,11 +618,11 @@ Examples: ```javascript const foo = 1; -// good -expect(foo).toBe(1); - -// bad +// Bad expect(foo).toEqual(1); + +// Good +expect(foo).toBe(1); ``` #### Prefer more befitting matchers @@ -621,12 +675,11 @@ Jest has the tricky `toBeDefined` matcher that can produce false positive test. the given value for `undefined` only. ```javascript -// good -expect(wrapper.find('foo').exists()).toBe(true); - -// bad -// if finder returns null, the test will pass +// Bad: if finder returns null, the test will pass expect(wrapper.find('foo')).toBeDefined(); + +// Good +expect(wrapper.find('foo').exists()).toBe(true); ``` #### Avoid using `setImmediate` @@ -771,13 +824,37 @@ yarn karma -f 'spec/javascripts/ide/**/file_spec.js' ## Frontend test fixtures -Code that is added to HAML templates (in `app/views/`) or makes Ajax requests to the backend has tests that require HTML or JSON from the backend. -Fixtures for these tests are located at: +Frontend fixtures are files containing responses from backend controllers. These responses can be either HTML +generated from haml templates or JSON payloads. Frontend tests that rely on these responses are +often using fixtures to validate correct integration with the backend code. + +### Generate fixtures + +You can find code to generate test fixtures in: - `spec/frontend/fixtures/`, for running tests in CE. - `ee/spec/frontend/fixtures/`, for running tests in EE. -Fixture files in: +You can generate fixtures by running: + +- `bin/rake frontend:fixtures` to generate all fixtures +- `bin/rspec spec/frontend/fixtures/merge_requests.rb` to generate specific fixtures (in this case for `merge_request.rb`) + +You can find generated fixtures are in `tmp/tests/frontend/fixtures-ee`. + +#### Creating new fixtures + +For each fixture, you can find the content of the `response` variable in the output file. +For example, test named `"merge_requests/diff_discussion.json"` in `spec/frontend/fixtures/merge_requests.rb` +will produce output file `tmp/tests/frontend/fixtures-ee/merge_requests/diff_discussion.json`. +The `response` variable gets automatically set if the test is marked as `type: :request` or `type: :controller`. + +When creating a new fixture, it often makes sense to take a look at the corresponding tests for the +endpoint in `(ee/)spec/controllers/` or `(ee/)spec/requests/`. + +### Use fixtures + +Jest and Karma test suites import fixtures in different ways: - The Karma test suite are served by [jasmine-jquery](https://github.com/velesin/jasmine-jquery). - Jest use `spec/frontend/helpers/fixtures.js`. @@ -803,14 +880,6 @@ it('uses some HTML element', () => { }); ``` -HTML and JSON fixtures are generated from backend views and controllers using RSpec (see `spec/frontend/fixtures/*.rb`). - -For each fixture, the content of the `response` variable is stored in the output file. -This variable gets automatically set if the test is marked as `type: :request` or `type: :controller`. -Fixtures are regenerated using the `bin/rake frontend:fixtures` command but you can also generate them individually, -for example `bin/rspec spec/frontend/fixtures/merge_requests.rb`. -When creating a new fixture, it often makes sense to take a look at the corresponding tests for the endpoint in `(ee/)spec/controllers/` or `(ee/)spec/requests/`. - ## Data-driven tests Similar to [RSpec's parameterized tests](best_practices.md#table-based--parameterized-tests), diff --git a/doc/development/testing_guide/review_apps.md b/doc/development/testing_guide/review_apps.md index 54f8ca0d98b..68816ccfe45 100644 --- a/doc/development/testing_guide/review_apps.md +++ b/doc/development/testing_guide/review_apps.md @@ -142,6 +142,9 @@ the following node pools: - `e2-highcpu-16` (16 vCPU, 16 GB memory) pre-emptible nodes with autoscaling +Node pool image type must be `Container-Optimized OS (cos)`, not `Container-Optimized OS with Containerd (cos_containerd)`, +due to this [known issue on GitLab Runner Kubernetes executor](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/4755) + ### Helm The Helm version used is defined in the @@ -153,13 +156,12 @@ used by the `review-deploy` and `review-stop` jobs. ### Get access to the GCP Review Apps cluster You need to [open an access request (internal link)](https://gitlab.com/gitlab-com/access-requests/-/issues/new) -for the `gcp-review-apps-sg` GCP group. In order to join a group, you must specify the desired GCP role in your access request. -The role is what will grant you specific permissions in order to engage with Review App containers. +for the `gcp-review-apps-dev` GCP group and role. -Here are some permissions you may want to have, and the roles that grant them: +This will grant you the following permissions for: -- `container.pods.getLogs` - Required to [retrieve pod logs](#dig-into-a-pods-logs). Granted by [Viewer (`roles/viewer`)](https://cloud.google.com/iam/docs/understanding-roles#kubernetes-engine-roles). -- `container.pods.exec` - Required to [run a Rails console](#run-a-rails-console). Granted by [Kubernetes Engine Developer (`roles/container.developer`)](https://cloud.google.com/iam/docs/understanding-roles#kubernetes-engine-roles). +- [Retrieving pod logs](#dig-into-a-pods-logs). Granted by [Viewer (`roles/viewer`)](https://cloud.google.com/iam/docs/understanding-roles#kubernetes-engine-roles). +- [Running a Rails console](#run-a-rails-console). Granted by [Kubernetes Engine Developer (`roles/container.pods.exec`)](https://cloud.google.com/iam/docs/understanding-roles#kubernetes-engine-roles). ### Log into my Review App diff --git a/doc/development/testing_guide/testing_migrations_guide.md b/doc/development/testing_guide/testing_migrations_guide.md index 8ee758177c3..a5bcb651d71 100644 --- a/doc/development/testing_guide/testing_migrations_guide.md +++ b/doc/development/testing_guide/testing_migrations_guide.md @@ -37,15 +37,37 @@ ensures proper isolation. To test an `ActiveRecord::Migration` class (i.e., a regular migration `db/migrate` or a post-migration `db/post_migrate`), you -will need to manually `require` the migration file because it is not -autoloaded with Rails. Example: +will need to load the migration file by using the `require_migration!` helper +method because it is not autoloaded by Rails. + +Example: ```ruby -require Rails.root.join('db', 'post_migrate', '20170526185842_migrate_pipeline_stages.rb') +require 'spec_helper' + +require_migration! + +RSpec.describe ... ``` ### Test helpers +#### `require_migration!` + +Since the migration files are not autoloaded by Rails, you will need to manually +load the migration file. To do so, you can use the `require_migration!` helper method +which can automatically load the correct migration file based on the spec file name. + +For example, if your spec file is named as `populate_foo_column_spec.rb` then the +helper method will try to load `${schema_version}_populate_foo_column.rb` migration file. + +In case there is no pattern between your spec file and the actual migration file, +you can provide the migration file name without the schema version, like so: + +```ruby +require_migration!('populate_foo_column') +``` + #### `table` Use the `table` helper to create a temporary `ActiveRecord::Base`-derived model @@ -110,7 +132,8 @@ migration. You can find the complete spec in ```ruby require 'spec_helper' -require Rails.root.join('db', 'post_migrate', '20170526185842_migrate_pipeline_stages.rb') + +require_migration! RSpec.describe MigratePipelineStages do # Create test data - pipeline and CI/CD jobs. diff --git a/doc/development/uploads.md b/doc/development/uploads.md index 4693c93e3d0..0c8b712a001 100644 --- a/doc/development/uploads.md +++ b/doc/development/uploads.md @@ -214,7 +214,8 @@ In this setup, an extra Rails route must be implemented in order to handle autho and [its routes](https://gitlab.com/gitlab-org/gitlab/blob/cc723071ad337573e0360a879cbf99bc4fb7adb9/config/routes/git_http.rb#L31-32). - [API endpoints for uploading packages](packages.md#file-uploads). -**note:** this will fallback to _disk buffered upload_ when `direct_upload` is disabled inside the [object storage setting](../administration/uploads.md#object-storage-settings). +Note: **Note:** +This will fallback to _disk buffered upload_ when `direct_upload` is disabled inside the [object storage setting](../administration/uploads.md#object-storage-settings). The answer to the `/authorize` call will only contain a file system path. ```mermaid diff --git a/doc/development/what_requires_downtime.md b/doc/development/what_requires_downtime.md index d0b8aa18f5c..e9d4ed8eaf6 100644 --- a/doc/development/what_requires_downtime.md +++ b/doc/development/what_requires_downtime.md @@ -103,7 +103,8 @@ end This will take care of renaming the column, ensuring data stays in sync, copying over indexes and foreign keys, etc. -**NOTE:** if a column contains 1 or more indexes that do not contain the name of +NOTE: **Note:** +If a column contains 1 or more indexes that do not contain the name of the original column, the above procedure will fail. In this case you will first need to rename these indexes. @@ -132,7 +133,7 @@ end NOTE: **Note:** If you're renaming a [large table](https://gitlab.com/gitlab-org/gitlab/-/blob/master/rubocop/rubocop-migrations.yml#L3), please carefully consider the state when the first migration has run but the second cleanup migration hasn't been run yet. -With [Canary](https://about.gitlab.com/handbook/engineering/infrastructure/library/canary/) it is possible that the system runs in this state for a significant amount of time. +With [Canary](https://gitlab.com/gitlab-com/gl-infra/readiness/-/tree/master/library/canary/) it is possible that the system runs in this state for a significant amount of time. ## Changing Column Constraints diff --git a/doc/gitlab-basics/README.md b/doc/gitlab-basics/README.md index 0f95c9e0cb6..a7f18e13f74 100644 --- a/doc/gitlab-basics/README.md +++ b/doc/gitlab-basics/README.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" comments: false type: index --- diff --git a/doc/gitlab-basics/add-file.md b/doc/gitlab-basics/add-file.md index 13a1591fb76..c5b57d4623d 100644 --- a/doc/gitlab-basics/add-file.md +++ b/doc/gitlab-basics/add-file.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" type: howto --- diff --git a/doc/gitlab-basics/command-line-commands.md b/doc/gitlab-basics/command-line-commands.md index b9426ec0849..e259aeef37b 100644 --- a/doc/gitlab-basics/command-line-commands.md +++ b/doc/gitlab-basics/command-line-commands.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" type: howto, reference --- diff --git a/doc/gitlab-basics/create-branch.md b/doc/gitlab-basics/create-branch.md index e2a2fb52af8..0ad5cb53e97 100644 --- a/doc/gitlab-basics/create-branch.md +++ b/doc/gitlab-basics/create-branch.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" type: howto --- diff --git a/doc/gitlab-basics/create-project.md b/doc/gitlab-basics/create-project.md index def14cfce6d..929062102bb 100644 --- a/doc/gitlab-basics/create-project.md +++ b/doc/gitlab-basics/create-project.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" type: howto --- diff --git a/doc/gitlab-basics/create-your-ssh-keys.md b/doc/gitlab-basics/create-your-ssh-keys.md index 248f39b842c..fba0408e26b 100644 --- a/doc/gitlab-basics/create-your-ssh-keys.md +++ b/doc/gitlab-basics/create-your-ssh-keys.md @@ -1,6 +1,10 @@ --- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" type: howto --- + # Create and add your SSH key pair It is best practice to use [Git over SSH instead of Git over HTTP](https://git-scm.com/book/en/v2/Git-on-the-Server-The-Protocols). diff --git a/doc/gitlab-basics/feature_branch_workflow.md b/doc/gitlab-basics/feature_branch_workflow.md index dd7eed89ea1..0fe04e62f22 100644 --- a/doc/gitlab-basics/feature_branch_workflow.md +++ b/doc/gitlab-basics/feature_branch_workflow.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" disqus_identifier: 'https://docs.gitlab.com/ee/workflow/workflow.html' --- diff --git a/doc/gitlab-basics/fork-project.md b/doc/gitlab-basics/fork-project.md index e92491a0821..76685db8d7d 100644 --- a/doc/gitlab-basics/fork-project.md +++ b/doc/gitlab-basics/fork-project.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" type: howto --- diff --git a/doc/gitlab-basics/start-using-git.md b/doc/gitlab-basics/start-using-git.md index d767bee4cf1..95b3a59ef6e 100644 --- a/doc/gitlab-basics/start-using-git.md +++ b/doc/gitlab-basics/start-using-git.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" type: howto, tutorial description: "Introduction to using Git through the command line." last_updated: 2020-06-30 diff --git a/doc/img/devops-stages-13_3.png b/doc/img/devops-stages-13_3.png Binary files differnew file mode 100644 index 00000000000..609533e12e7 --- /dev/null +++ b/doc/img/devops-stages-13_3.png diff --git a/doc/img/devops-stages.png b/doc/img/devops-stages.png Binary files differdeleted file mode 100644 index 2fa5357062c..00000000000 --- a/doc/img/devops-stages.png +++ /dev/null diff --git a/doc/install/aws/img/aws_ha_architecture_diagram.png b/doc/install/aws/img/aws_ha_architecture_diagram.png Binary files differindex dc63d36e0b3..b8b60d13ac5 100644 --- a/doc/install/aws/img/aws_ha_architecture_diagram.png +++ b/doc/install/aws/img/aws_ha_architecture_diagram.png diff --git a/doc/install/aws/index.md b/doc/install/aws/index.md index 35c046423b0..92a4ce860c3 100644 --- a/doc/install/aws/index.md +++ b/doc/install/aws/index.md @@ -246,7 +246,7 @@ On the EC2 dashboard, look for Load Balancer in the left navigation bar: 1. For **Ping Protocol**, select HTTP. 1. For **Ping Port**, enter 80. 1. For **Ping Path**, enter `/users/sign_in`. (We use `/users/sign_in` as it's a public endpoint that does - not require authorization.) + not require authentication.) 1. Keep the default **Advanced Details** or adjust them according to your needs. 1. Click **Add EC2 Instances** - don't add anything as we will create an Auto Scaling Group later to manage instances for us. 1. Click **Add Tags** and add any tags you need. @@ -473,9 +473,9 @@ Since we're adding our SSL certificate at the load balancer, we do not need GitL sudo gitlab-ctl reconfigure ``` -#### Install the `pg_trgm` extension for PostgreSQL +#### Install the required extensions for PostgreSQL -From your GitLab instance, connect to the RDS instance to verify access and to install the required `pg_trgm` extension. +From your GitLab instance, connect to the RDS instance to verify access and to install the required `pg_trgm` and `btree_gist` extensions. To find the host or endpoint, navigate to **Amazon RDS > Databases** and click on the database you created earlier. Look for the endpoint under the **Connectivity & security** tab. @@ -492,6 +492,7 @@ psql (10.9) Type "help" for help. gitlab=# CREATE EXTENSION pg_trgm; +gitlab=# CREATE EXTENSION btree_gist; gitlab=# \q ``` @@ -574,7 +575,7 @@ Let's create an EC2 instance where we'll install Gitaly: 1. Finally, acknowledge that you have access to the selected private key file or create a new one. Click **Launch Instances**. NOTE: **Note:** -Instead of storing configuration _and_ repository data on the root volume, you can also choose to add an additional EBS volume for repository storage. Follow the same guidance as above. See the [Amazon EBS pricing](https://aws.amazon.com/ebs/pricing/). We do not recommend using EFS as it may negatively impact GitLab’s performance. You can review the [relevant documentation](../../administration/high_availability/nfs.md#avoid-using-awss-elastic-file-system-efs) for more details. +Instead of storing configuration _and_ repository data on the root volume, you can also choose to add an additional EBS volume for repository storage. Follow the same guidance as above. See the [Amazon EBS pricing](https://aws.amazon.com/ebs/pricing/). We do not recommend using EFS as it may negatively impact GitLab’s performance. You can review the [relevant documentation](../../administration/nfs.md#avoid-using-awss-elastic-file-system-efs) for more details. Now that we have our EC2 instance ready, follow the [documentation to install GitLab and set up Gitaly on its own server](../../administration/gitaly/index.md#run-gitaly-on-its-own-server). Perform the client setup steps from that document on the [GitLab instance we created](#install-gitlab) above. @@ -642,6 +643,13 @@ to eliminate the need for NFS to support GitLab Pages. That concludes the configuration changes for our GitLab instance. Next, we'll create a custom AMI based on this instance to use for our launch configuration and auto scaling group. +### Log in for the first time + +Using the domain name you used when setting up [DNS for the load balancer](#configure-dns-for-load-balancer), you should now be able to visit GitLab in your browser. You will be asked to set up a password +for the `root` user which has admin privileges on the GitLab instance. This password will be stored in the database. + +When our [auto scaling group](#create-an-auto-scaling-group) spins up new instances, we'll be able to log in with username `root` and the newly created password. + ### Create custom AMI On the EC2 dashboard: @@ -696,13 +704,6 @@ As the auto scaling group is created, you'll see your new instances spinning up Since our instances are created by the auto scaling group, go back to your instances and terminate the [instance we created manually above](#install-gitlab). We only needed this instance to create our custom AMI. -### Log in for the first time - -Using the domain name you used when setting up [DNS for the load balancer](#configure-dns-for-load-balancer), you should now be able to visit GitLab in your browser. The very first time you will be asked to set up a password -for the `root` user which has admin privileges on the GitLab instance. - -After you set it up, login with username `root` and the newly created password. - ## Health check and monitoring with Prometheus Apart from Amazon's Cloudwatch which you can enable on various services, diff --git a/doc/install/azure/index.md b/doc/install/azure/index.md index 3cf963bdf57..548c7d8c92e 100644 --- a/doc/install/azure/index.md +++ b/doc/install/azure/index.md @@ -41,8 +41,7 @@ create SQL Databases, author websites, and perform lots of other cloud tasks. The [Azure Marketplace](https://azuremarketplace.microsoft.com/en-us/marketplace/) is an online store for pre-configured applications and services which have been optimized for the cloud by software vendors like GitLab, available on the Azure Marketplace as pre-configured solutions. In this tutorial -we will install GitLab Community Edition, but for GitLab Enterprise Edition you -can follow the same process. +we will install GitLab Community Edition. To begin creating a new GitLab VM, click on the **+ New** icon, type "GitLab" into the search box, and then click the **"GitLab Community Edition"** search result: @@ -67,7 +66,8 @@ The first items we need to configure are the basic settings of the underlying vi 1. Enter a `User name` - e.g. `gitlab-admin` 1. Select an `Authentication type`, either **SSH public key** or **Password**: - > **Note:** if you're unsure which authentication type to use, select **Password** + NOTE: **Note:** + If you're unsure which authentication type to use, select **Password** 1. If you chose **SSH public key** - enter your `SSH public key` into the field provided _(read the [SSH documentation](../../ssh/README.md) to learn more about how to set up SSH @@ -78,8 +78,9 @@ The first items we need to configure are the basic settings of the underlying vi 1. Choose the appropriate `Subscription` tier for your Azure account 1. Choose an existing `Resource Group` or create a new one - e.g. **"GitLab-CE-Azure"** - > **Note:** a "Resource group" is a way to group related resources together for easier administration. - > We chose "GitLab-CE-Azure", but your resource group can have the same name as your VM. + NOTE **Note:** + A "Resource group" is a way to group related resources together for easier administration. + We chose "GitLab-CE-Azure", but your resource group can have the same name as your VM. 1. Choose a `Location` - if you're unsure, select the default location @@ -94,7 +95,8 @@ Check the settings you have entered, and then click **"OK"** when you're ready t Next, you need to choose the size of your VM - selecting features such as the number of CPU cores, the amount of RAM, the size of storage (and its speed), etc. -> **Note:** in common with other cloud vendors, Azure operates a resource/usage pricing model, i.e. +NOTE: **Note:** +In common with other cloud vendors, Azure operates a resource/usage pricing model, i.e. the more resources your VM consumes the more it will cost you to run, so make your selection carefully. You'll see that Azure provides an _estimated_ monthly cost beneath each VM Size to help guide your selection. @@ -105,7 +107,8 @@ ahead and select this one, but please choose the size which best meets your own  -> **Note:** be aware that while your VM is active (known as "allocated"), it will incur +NOTE: **Note:** +Be aware that while your VM is active (known as "allocated"), it will incur "compute charges" which, ultimately, you will be billed for. So, even if you're using the free trial credits, you'll likely want to learn [how to properly shutdown an Azure VM to save money](https://build5nines.com/properly-shutdown-azure-vm-to-save-money/). @@ -131,7 +134,8 @@ new VM. You'll be billed only for the VM itself (e.g. "Standard DS1 v2") because  -> **Note:** at this stage, you can review and modify the any of the settings you have made during all +NOTE: **Note:** +At this stage, you can review and modify the any of the settings you have made during all previous steps, just click on any of the four steps to re-open them. When you have read and agreed to the terms of use and are ready to proceed, click **"Purchase"**. @@ -173,7 +177,8 @@ _(the full domain name of your own VM will be different, of course)_. Click **"Save"** for the changes to take effect. -> **Note:** if you want to use your own domain name, you will need to add a DNS `A` record at your +NOTE **Note:** +If you want to use your own domain name, you will need to add a DNS `A` record at your domain registrar which points to the public IP address of your Azure VM. If you do this, you'll need to make sure your VM is configured to use a _static_ public IP address (i.e. not a _dynamic_ one) or you will have to reconfigure the DNS `A` record each time Azure reassigns your VM a new public IP @@ -189,7 +194,8 @@ Ports are opened by adding _security rules_ to the **"Network security group"** has been assigned to. If you followed the process above, then Azure will have automatically created an NSG named `GitLab-CE-nsg` and assigned the `GitLab-CE` VM to it. -> **Note:** if you gave your VM a different name then the NSG automatically created by Azure will +NOTE: **Note:** +If you gave your VM a different name then the NSG automatically created by Azure will also have a different name - the name you have your VM, with `-nsg` appended to it. You can navigate to the NSG settings via many different routes in the Azure Portal, but one of the @@ -320,7 +326,8 @@ Under the **"Components"** section, we can see that our VM is currently running GitLab. This is the version of GitLab which was contained in the Azure Marketplace **"GitLab Community Edition"** offering we used to build the VM when we wrote this tutorial. -> **Note:** The version of GitLab in your own VM instance may well be different, but the update +NOTE **Note:** +The version of GitLab in your own VM instance may well be different, but the update process will still be the same. ### Connect via SSH @@ -332,12 +339,11 @@ connect to it using SSH ([Secure Shell](https://en.wikipedia.org/wiki/Secure_She If you're running Windows, you'll need to connect using [PuTTY](https://www.putty.org) or an equivalent Windows SSH client. If you're running Linux or macOS, then you already have an SSH client installed. -> **Note:** -> -> - Remember that you will need to login with the username and password you specified -> [when you created](#basics) your Azure VM -> - If you need to reset your VM password, read -> [how to reset SSH credentials for a user on an Azure VM](https://docs.microsoft.com/en-us/azure/virtual-machines/troubleshooting/troubleshoot-ssh-connection). +Remember that you will need to login with the username and password you specified +[when you created](#basics) your Azure VM. + +If you need to reset your VM password, read +[how to reset SSH credentials for a user on an Azure VM](https://docs.microsoft.com/en-us/azure/virtual-machines/troubleshooting/troubleshoot-ssh-connection). #### SSH from the command-line diff --git a/doc/install/installation.md b/doc/install/installation.md index 8b285e0c9f1..7216f750624 100644 --- a/doc/install/installation.md +++ b/doc/install/installation.md @@ -28,12 +28,12 @@ following the Since an installation from source is a lot of work and error prone we strongly recommend the fast and reliable [Omnibus package installation](https://about.gitlab.com/install/) (deb/rpm). One reason the Omnibus package is more reliable is its use of runit to restart any of the GitLab processes in case one crashes. -On heavily used GitLab instances the memory usage of the Sidekiq background worker will grow over time. +On heavily used GitLab instances the memory usage of the Sidekiq background worker grows over time. Omnibus packages solve this by [letting the Sidekiq terminate gracefully](../administration/operations/sidekiq_memory_killer.md) if it uses too much memory. -After this termination runit will detect Sidekiq is not running and will start it. +After this termination runit detects Sidekiq is not running and starts it. Since installations from source don't use runit for process supervision, Sidekiq -can't be terminated and its memory usage will grow over time. +can't be terminated and its memory usage grows over time. ## Select a version to install @@ -44,7 +44,7 @@ If the highest number stable branch is unclear, check the [GitLab blog](https:// ## GitLab directory structure -This is the main directory structure you will end up with following the instructions +This is the main directory structure you end up with following the instructions of this page: ```plaintext @@ -99,7 +99,7 @@ apt-get install sudo -y ``` NOTE: **Note:** -During this installation, some files will need to be edited manually. If you are familiar with vim, set it as default editor with the commands below. If you are not familiar with vim, skip this and keep using the default editor. +During this installation, some files need to be edited manually. If you are familiar with vim, set it as default editor with the commands below. If you are not familiar with vim, skip this and keep using the default editor. ```shell # Install vim and set as default editor @@ -134,7 +134,7 @@ Make sure you have the right version of Git installed: # Install Git sudo apt-get install -y git-core -# Make sure Git is version 2.27.0 or higher (minimal supported version is 2.25.0) +# Make sure Git is version 2.24.0 or higher (recommended version is 2.28.0) git --version ``` @@ -181,9 +181,9 @@ sudo make install # Download and compile from source cd /tmp -curl --remote-name --location --progress https://www.kernel.org/pub/software/scm/git/git-2.27.0.tar.gz -echo '77ded85cbe42b1ffdc2578b460a1ef5d23bcbc6683eabcafbb0d394dffe2e787 git-2.27.0.tar.gz' | shasum -a256 -c - && tar -xzf git-2.27.0.tar.gz -cd git-2.27.0/ +curl --remote-name --location --progress https://www.kernel.org/pub/software/scm/git/git-2.28.0.tar.gz +echo 'f914c60a874d466c1e18467c864a910dd4ea22281ba6d4d58077cb0c3f115170 git-2.28.0.tar.gz' | shasum -a256 -c - && tar -xzf git-2.28.0.tar.gz +cd git-2.28.0/ ./configure --with-libpcre make prefix=/usr/local all @@ -200,7 +200,8 @@ needs to be installed. sudo apt-get install -y graphicsmagick ``` -**Note:** In order to receive mail notifications, make sure to install a mail server. By default, Debian is shipped with exim4 but this [has problems](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/12754) while Ubuntu does not ship with one. The recommended mail server is postfix and you can install it with: +NOTE: **Note:** +In order to receive mail notifications, make sure to install a mail server. By default, Debian is shipped with exim4 but this [has problems](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/12754) while Ubuntu does not ship with one. The recommended mail server is postfix and you can install it with: ```shell sudo apt-get install -y postfix @@ -219,8 +220,9 @@ sudo apt-get install -y libimage-exiftool-perl The Ruby interpreter is required to run GitLab. -**Note:** The current supported Ruby (MRI) version is 2.6.x. GitLab 12.2 - dropped support for Ruby 2.5.x. +NOTE: **Note:** +The current supported Ruby (MRI) version is 2.6.x. GitLab 12.2 +dropped support for Ruby 2.5.x. The use of Ruby version managers such as [RVM](https://rvm.io/), [rbenv](https://github.com/rbenv/rbenv) or [chruby](https://github.com/postmodern/chruby) with GitLab in production, frequently leads to hard to diagnose problems. Version managers @@ -284,7 +286,7 @@ requirements for these are: In many distros, the versions provided by the official package repositories are out of date, so -we'll need to install through the following commands: +we need to install through the following commands: ```shell # install node v12.x @@ -331,12 +333,18 @@ Starting from GitLab 12.1, only PostgreSQL is supported. Since GitLab 13.0, we r sudo -u postgres psql -d template1 -c "CREATE USER git CREATEDB;" ``` -1. Create the `pg_trgm` extension (required for GitLab 8.6+): +1. Create the `pg_trgm` extension: ```shell sudo -u postgres psql -d template1 -c "CREATE EXTENSION IF NOT EXISTS pg_trgm;" ``` +1. Create the `btree_gist` extension (required for GitLab 13.1+): + + ```shell + sudo -u postgres psql -d template1 -c "CREATE EXTENSION IF NOT EXISTS btree_gist;" + ``` + 1. Create the GitLab production database and grant all privileges on the database: ```shell @@ -358,7 +366,25 @@ Starting from GitLab 12.1, only PostgreSQL is supported. Since GitLab 13.0, we r AND installed_version IS NOT NULL; ``` - If the extension is enabled this will produce the following output: + If the extension is enabled this produces the following output: + + ```plaintext + enabled + --------- + t + (1 row) + ``` + +1. Check if the `btree_gist` extension is enabled: + + ```sql + SELECT true AS enabled + FROM pg_available_extensions + WHERE name = 'btree_gist' + AND installed_version IS NOT NULL; + ``` + + If the extension is enabled this produces the following output: ```plaintext enabled @@ -436,7 +462,7 @@ Clone Enterprise Edition: ```shell # Clone GitLab repository -sudo -u git -H git clone https://gitlab.com/gitlab-org/gitlab-ee.git -b X-Y-stable gitlab +sudo -u git -H git clone https://gitlab.com/gitlab-org/gitlab.git -b X-Y-stable gitlab ``` Make sure to replace `X-Y-stable` with the stable branch that matches the @@ -496,9 +522,6 @@ sudo -u git -H cp config/puma.rb.example config/puma.rb # cores you have available. You can get that number via the `nproc` command. sudo -u git -H editor config/puma.rb -# Copy the example Rack attack config -sudo -u git -H cp config/initializers/rack_attack.rb.example config/initializers/rack_attack.rb - # Configure Git global settings for git user # 'autocrlf' is needed for the web editor sudo -u git -H git config --global core.autocrlf input @@ -541,7 +564,6 @@ sudo -u git cp config/database.yml.postgresql config/database.yml # adapter: postgresql # encoding: unicode # database: gitlabhq_production -# pool: 10 # sudo -u git -H editor config/database.yml @@ -591,12 +613,12 @@ NOTE: **Note:** If you want to use HTTPS, see [Using HTTPS](#using-https) for the additional steps. NOTE: **Note:** -Make sure your hostname can be resolved on the machine itself by either a proper DNS record or an additional line in `/etc/hosts` ("127.0.0.1 hostname"). This might be necessary, for example, if you set up GitLab behind a reverse proxy. If the hostname cannot be resolved, the final installation check will fail with `Check GitLab API access: FAILED. code: 401` and pushing commits will be rejected with `[remote rejected] master -> master (hook declined)`. +Make sure your hostname can be resolved on the machine itself by either a proper DNS record or an additional line in `/etc/hosts` ("127.0.0.1 hostname"). This might be necessary, for example, if you set up GitLab behind a reverse proxy. If the hostname cannot be resolved, the final installation check fails with `Check GitLab API access: FAILED. code: 401` and pushing commits are rejected with `[remote rejected] master -> master (hook declined)`. ### Install GitLab Workhorse GitLab-Workhorse uses [GNU Make](https://www.gnu.org/software/make/). The -following command-line will install GitLab-Workhorse in `/home/git/gitlab-workhorse` +following command-line installs GitLab-Workhorse in `/home/git/gitlab-workhorse` which is the recommended location. ```shell @@ -612,7 +634,7 @@ sudo -u git -H bundle exec rake "gitlab:workhorse:install[/home/git/gitlab-workh ### Install GitLab-Elasticsearch-indexer on Enterprise Edition GitLab-Elasticsearch-Indexer uses [GNU Make](https://www.gnu.org/software/make/). The -following command-line will install GitLab-Elasticsearch-Indexer in `/home/git/gitlab-elasticsearch-indexer` +following command-line installs GitLab-Elasticsearch-Indexer in `/home/git/gitlab-elasticsearch-indexer` which is the recommended location. ```shell @@ -625,15 +647,15 @@ You can specify a different Git repository by providing it as an extra parameter sudo -u git -H bundle exec rake "gitlab:indexer:install[/home/git/gitlab-elasticsearch-indexer,https://example.com/gitlab-elasticsearch-indexer.git]" RAILS_ENV=production ``` -The source code will first be fetched to the path specified by the first parameter. Then a binary will be built under its `bin` directory. -You will then need to update `gitlab.yml`'s `production -> elasticsearch -> indexer_path` setting to point to that binary. +The source code first is fetched to the path specified by the first parameter. Then a binary is built under its `bin` directory. +You then need to update `gitlab.yml`'s `production -> elasticsearch -> indexer_path` setting to point to that binary. NOTE: **Note:** Elasticsearch is a feature of GitLab Enterprise Edition and isn't included in GitLab Community Edition. ### Install GitLab Pages -GitLab Pages uses [GNU Make](https://www.gnu.org/software/make/). This step is optional and only needed if you wish to host static sites from within GitLab. The following commands will install GitLab Pages in `/home/git/gitlab-pages`. For additional setup steps, consult the [administration guide](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/administration/pages/source.md) for your version of GitLab as the GitLab Pages daemon can be run several different ways. +GitLab Pages uses [GNU Make](https://www.gnu.org/software/make/). This step is optional and only needed if you wish to host static sites from within GitLab. The following commands install GitLab Pages in `/home/git/gitlab-pages`. For additional setup steps, consult the [administration guide](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/administration/pages/source.md) for your version of GitLab as the GitLab Pages daemon can be run several different ways. ```shell cd /home/git @@ -712,7 +734,7 @@ Otherwise, your secrets are exposed if one of your backups is compromised. ### Install Init Script -Download the init script (will be `/etc/init.d/gitlab`): +Download the init script (is `/etc/init.d/gitlab`): ```shell sudo cp lib/support/init.d/gitlab /etc/init.d/gitlab @@ -724,7 +746,7 @@ And if you are installing with a non-default folder or user copy and edit the de sudo cp lib/support/init.d/gitlab.default.example /etc/default/gitlab ``` -If you installed GitLab in another directory or as a user other than the default, you should change these settings in `/etc/default/gitlab`. Do not edit `/etc/init.d/gitlab` as it will be changed on upgrade. +If you installed GitLab in another directory or as a user other than the default, you should change these settings in `/etc/default/gitlab`. Do not edit `/etc/init.d/gitlab` as it is changed on upgrade. Make GitLab start on boot: @@ -812,7 +834,8 @@ If you intend to enable GitLab Pages, there is a separate NGINX config you need to use. Read all about the needed configuration at the [GitLab Pages administration guide](../administration/pages/index.md). -**Note:** If you want to use HTTPS, replace the `gitlab` NGINX config with `gitlab-ssl`. See [Using HTTPS](#using-https) for HTTPS configuration details. +NOTE: **Note:** +If you want to use HTTPS, replace the `gitlab` NGINX config with `gitlab-ssl`. See [Using HTTPS](#using-https) for HTTPS configuration details. ### Test Configuration @@ -920,7 +943,7 @@ See the [GitLab Runner section](https://about.gitlab.com/stages-devops-lifecycle ### Adding your Trusted Proxies If you are using a reverse proxy on a separate machine, you may want to add the -proxy to the trusted proxies list. Otherwise users will appear signed in from the +proxy to the trusted proxies list. Otherwise users appear signed in from the proxy's IP address. You can add trusted proxies in `config/gitlab.yml` by customizing the `trusted_proxies` @@ -992,7 +1015,7 @@ If you want to switch back to Unicorn, follow these steps: ### Using Sidekiq instead of Sidekiq Cluster As of GitLab 12.10, Source installations are using `bin/sidekiq-cluster` for managing Sidekiq processes. -Using Sidekiq directly will still be supported until 14.0. So if you're experiencing issues, please: +Using Sidekiq directly is still supported until 14.0. So if you're experiencing issues, please: 1. Edit the system `init.d` script to remove the `SIDEKIQ_WORKERS` flag. If you have `/etc/default/gitlab`, then you should edit it instead. 1. Restart GitLab. diff --git a/doc/install/openshift_and_gitlab/index.md b/doc/install/openshift_and_gitlab/index.md index 519a0a5b7aa..022e2095c68 100644 --- a/doc/install/openshift_and_gitlab/index.md +++ b/doc/install/openshift_and_gitlab/index.md @@ -46,7 +46,7 @@ latest Origin release is used: - **OpenShift** `v1.3.0` (is pre-installed in the [VM image](https://app.vagrantup.com/openshift/boxes/origin-all-in-one)) - **Kubernetes** `v1.3.0` (is pre-installed in the [VM image](https://app.vagrantup.com/openshift/boxes/origin-all-in-one)) ->**Note:** +NOTE: **Note:** If you intend to deploy GitLab on a production OpenShift cluster, there are some limitations to bare in mind. Read on the [limitations](#current-limitations) section for more information and follow the linked links for the relevant @@ -266,7 +266,7 @@ And then let's import it in OpenShift: oc create -f openshift-template.json -n openshift ``` ->**Note:** +NOTE: **Note:** The `-n openshift` namespace flag is a trick to make the template available to all projects. If you recall from when we created the `gitlab` project, `oc` switched to it automatically, and that can be verified by the `oc status` command. If @@ -313,7 +313,7 @@ If you are deploying to production you will want to change the **GitLab instance hostname** and use greater values for the volume sizes. If you don't provide a password for PostgreSQL, it will be created automatically. ->**Note:** +NOTE: **Note:** The `gitlab.apps.10.2.2.2.nip.io` hostname that is used by default will resolve to the host with IP `10.2.2.2` which is the IP our VM uses. It is a trick to have distinct FQDNs pointing to services that are on our local network. diff --git a/doc/install/relative_url.md b/doc/install/relative_url.md index f8ff8e75c4d..09069a2d1fd 100644 --- a/doc/install/relative_url.md +++ b/doc/install/relative_url.md @@ -108,7 +108,7 @@ Make sure to follow all steps below: -authBackend http://127.0.0.1:8080/gitlab ``` - **Note:** + NOTE: **Note:** If you are using a custom init script, make sure to edit the above GitLab Workhorse setting as needed. diff --git a/doc/install/requirements.md b/doc/install/requirements.md index 5bc587627f5..54fcfeb2ee4 100644 --- a/doc/install/requirements.md +++ b/doc/install/requirements.md @@ -56,8 +56,8 @@ The minimum required Go version is 1.13. From GitLab 13.1: -- Git 2.25.x and later is required. -- Git 2.27.x and later [is recommended](https://gitlab.com/gitlab-org/gitaly/-/issues/2829). +- Git 2.24.x and later is required. +- Git 2.28.x and later [is recommended](https://gitlab.com/gitlab-org/gitaly/-/issues/2959). ### Node.js versions @@ -91,7 +91,7 @@ Apart from a local hard drive you can also mount a volume that supports the netw If you have enough RAM and a recent CPU the speed of GitLab is mainly limited by hard drive seek times. Having a fast drive (7200 RPM and up) or a solid state drive (SSD) will improve the responsiveness of GitLab. NOTE: **Note:** -Since file system performance may affect GitLab's overall performance, [we don't recommend using AWS EFS for storage](../administration/high_availability/nfs.md#avoid-using-awss-elastic-file-system-efs). +Since file system performance may affect GitLab's overall performance, [we don't recommend using AWS EFS for storage](../administration/nfs.md#avoid-using-awss-elastic-file-system-efs). ### CPU @@ -140,8 +140,8 @@ GitLab version | Minimum PostgreSQL version 12.10 | 11 13.0 | 11 -You must also ensure the `pg_trgm` extension is loaded into every -GitLab database. This extension [can be enabled](https://www.postgresql.org/docs/11/sql-createextension.html) using a PostgreSQL super user. +You must also ensure the `pg_trgm` and `btree_gist` extensions are loaded into every +GitLab database. These extensions [can be enabled](https://www.postgresql.org/docs/11/sql-createextension.html) using a PostgreSQL super user. On some systems you may need to install an additional package (for example, `postgresql-contrib`) for this extension to become available. @@ -156,10 +156,6 @@ If you're using [GitLab Geo](../administration/geo/replication/index.md): - We strongly recommend running Omnibus-managed instances as they are actively developed and tested. We aim to be compatible with most external (not managed by Omnibus) databases (for example, [AWS Relational Database Service (RDS)](https://aws.amazon.com/rds/)) but we don't guarantee compatibility. -- You must also ensure the `postgres_fdw` extension is loaded into every - GitLab database. This extension - [can be enabled](https://www.postgresql.org/docs/11/sql-createextension.html) - using a PostgreSQL super user. ## Puma settings diff --git a/doc/integration/bitbucket.md b/doc/integration/bitbucket.md index dc5c7a138f8..a151fbf50e7 100644 --- a/doc/integration/bitbucket.md +++ b/doc/integration/bitbucket.md @@ -21,7 +21,7 @@ Bitbucket.org. ## Bitbucket OmniAuth provider -> **Note:** +NOTE: **Note:** GitLab 8.15 significantly simplified the way to integrate Bitbucket.org with GitLab. You are encouraged to upgrade your GitLab instance if you haven't done so already. If you're using GitLab 8.14 or below, [use the previous integration diff --git a/doc/integration/elasticsearch.md b/doc/integration/elasticsearch.md index 8ca45547f11..67b256cc944 100644 --- a/doc/integration/elasticsearch.md +++ b/doc/integration/elasticsearch.md @@ -23,9 +23,8 @@ special searches: ## Installing Elasticsearch -Elasticsearch is _not_ included in the Omnibus packages. You will have to -[install it yourself](https://www.elastic.co/guide/en/elasticsearch/reference/6.8/install-elasticsearch.html "Elasticsearch 6.8 installation documentation") -whether you are using the Omnibus package or installed GitLab from source. +Elasticsearch is _not_ included in the Omnibus packages or when you install from source. You must +[install it separately](https://www.elastic.co/guide/en/elasticsearch/reference/7.x/install-elasticsearch.html "Elasticsearch 7.x installation documentation"). Be sure to select your version. Providing detailed information on installing Elasticsearch is out of the scope of this document. @@ -118,14 +117,21 @@ Once installed, enable it under your instance's Elasticsearch settings explained ## System Requirements Elasticsearch requires additional resources in excess of those documented in the -[GitLab system requirements](../install/requirements.md). These will vary by -installation size, but you should ensure **at least** an additional **8 GiB of RAM** -for each Elasticsearch node, per the [official guidelines](https://www.elastic.co/guide/en/elasticsearch/guide/current/hardware.html). +[GitLab system requirements](../install/requirements.md). -Keep in mind, this is the **minimum requirements** as per Elasticsearch. For -production instances, they recommend considerably more resources. +The amount of resources (memory, CPU, storage) will vary greatly, based on the amount of data being indexed into the Elasticsearch cluster. According to [Elasticsearch official guidelines](https://www.elastic.co/guide/en/elasticsearch/guide/current/hardware.html#_memory), each node should have: -The necessary storage space largely depends on the size of the repositories you want to store in GitLab but as a rule of thumb you should have at least 50% of free space as all your repositories combined take up. +- [RAM](https://www.elastic.co/guide/en/elasticsearch/guide/current/hardware.html#_disks): **8 GiB as the bare minimum** +- [CPU](https://www.elastic.co/guide/en/elasticsearch/guide/current/hardware.html#_cpus): Modern processor with multiple cores +- [Storage](https://www.elastic.co/guide/en/elasticsearch/guide/current/hardware.html#_disks): Use SSD storage. As a guide you will need enough storage for 50% of the total size of your Git repositories. + +A few notes on CPU and storage: + +- CPU requirements for Elasticsearch tend to be light. There are specific scenarios where this isn't true, but GitLab.com isn't using Elasticsearch in an exceptionally CPU-heavy way. More cores will be more performant than faster CPUs. Extra concurrency from multiple cores will far outweigh a slightly faster clock speed in Elasticsearch. + +- Storage requirements for Elasticsearch are important, especially for indexing-heavy clusters. When possible, use SSDs, Their speed is far superior to any spinning media for Elasticsearch. In testing, nodes that use SSD storage see boosts in both query and indexing performance. + +Keep in mind, these are **minimum requirements** for Elasticsearch. Heavily-utilized Elasticsearch clusters will likely require considerably more resources. ## Enabling Elasticsearch @@ -142,13 +148,14 @@ The following Elasticsearch settings are available: | `Elasticsearch pause indexing` | Enables/disables temporary indexing pause. This is useful for cluster migration/reindexing. All changes are still tracked, but they are not committed to the Elasticsearch index until unpaused. | | `Search with Elasticsearch enabled` | Enables/disables using Elasticsearch in search. | | `URL` | The URL to use for connecting to Elasticsearch. Use a comma-separated list to support clustering (e.g., `http://host1, https://host2:9200`). If your Elasticsearch instance is password protected, pass the `username:password` in the URL (e.g., `http://<username>:<password>@<elastic_host>:9200/`). | -| `Number of Elasticsearch shards` | Elasticsearch indexes are split into multiple shards for performance reasons. In general, larger indexes need to have more shards. Changes to this value do not take effect until the index is recreated. You can read more about tradeoffs in the [Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-create-index.html#create-index-settings) | +| `Number of Elasticsearch shards` | Elasticsearch indexes are split into multiple shards for performance reasons. In general, larger indexes need to have more shards. Changes to this value do not take effect until the index is recreated. You can read more about tradeoffs in the [Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/scalability.html). | | `Number of Elasticsearch replicas` | Each Elasticsearch shard can have a number of replicas. These are a complete copy of the shard, and can provide increased query performance or resilience against hardware failure. Increasing this value will greatly increase total disk space required by the index. | | `Limit namespaces and projects that can be indexed` | Enabling this will allow you to select namespaces and projects to index. All other namespaces and projects will use database search instead. Please note that if you enable this option but do not select any namespaces or projects, none will be indexed. [Read more below](#limiting-namespaces-and-projects). -| `Using AWS hosted Elasticsearch with IAM credentials` | Sign your Elasticsearch requests using [AWS IAM authorization](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html) or [AWS EC2 Instance Profile Credentials](https://docs.aws.amazon.com/codedeploy/latest/userguide/getting-started-create-iam-instance-profile.html#getting-started-create-iam-instance-profile-cli). The policies must be configured to allow `es:*` actions. | +| `Using AWS hosted Elasticsearch with IAM credentials` | Sign your Elasticsearch requests using [AWS IAM authorization](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html), [AWS EC2 Instance Profile Credentials](https://docs.aws.amazon.com/codedeploy/latest/userguide/getting-started-create-iam-instance-profile.html#getting-started-create-iam-instance-profile-cli), or [AWS ECS Tasks Credentials](https://docs.aws.amazon.com/AmazonECS/latest/userguide/task-iam-roles.html). The policies must be configured to allow `es:*` actions. | | `AWS Region` | The AWS region your Elasticsearch service is located in. | | `AWS Access Key` | The AWS access key. | | `AWS Secret Access Key` | The AWS secret access key. | +| `Maximum file size indexed` | See [the explanation in instance limits.](../administration/instance_limits.md#maximum-file-size-indexed). | | `Maximum field length` | See [the explanation in instance limits.](../administration/instance_limits.md#maximum-field-length). | | `Maximum bulk request size (MiB)` | The Maximum Bulk Request size is used by GitLab's Golang-based indexer processes and indicates how much data it ought to collect (and store in memory) in a given indexing process before submitting the payload to Elasticsearch’s Bulk API. This setting should be used with the Bulk request concurrency setting (see below) and needs to accommodate the resource constraints of both the Elasticsearch host(s) and the host(s) running GitLab's Golang-based indexer either from the `gitlab-rake` command or the Sidekiq tasks. | | `Bulk request concurrency` | The Bulk request concurrency indicates how many of GitLab's Golang-based indexer processes (or threads) can run in parallel to collect data to subsequently submit to Elasticsearch’s Bulk API. This increases indexing performance, but fills the Elasticsearch bulk requests queue faster. This setting should be used together with the Maximum bulk request size setting (see above) and needs to accommodate the resource constraints of both the Elasticsearch host(s) and the host(s) running GitLab's Golang-based indexer either from the `gitlab-rake` command or the Sidekiq tasks. | @@ -284,9 +291,16 @@ or creating [extra Sidekiq processes](../administration/operations/extra_sidekiq ``` 1. [Enable **Elasticsearch indexing**](#enabling-elasticsearch). -1. Indexing large Git repositories can take a while. To speed up the process, you - can temporarily disable auto-refreshing and replicating. In our experience, you can expect a 20% - decrease in indexing time. We'll enable them when indexing is done. This step is optional! +1. Indexing large Git repositories can take a while. To speed up the process, you can [tune for indexing speed](https://www.elastic.co/guide/en/elasticsearch/reference/current/tune-for-indexing-speed.html#tune-for-indexing-speed): + + - You can temporarily disable [`refresh`](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-refresh.html), the operation responsible for making changes to an index available to search. + + - You can set the number of replicas to 0. This setting controls the number of copies each primary shard of an index will have. Thus, having 0 replicas effectively disables the replication of shards across nodes, which should increase the indexing performance. This is an important trade-off in terms of reliability and query performance. It is important to remember to set the replicas to a considered value after the initial indexing is complete. + + In our experience, you can expect a 20% decrease in indexing time. After completing indexing in a later step, you can return `refresh` and `number_of_replicas` to their desired settings. + + NOTE: **Note:** + This step is optional but may help significantly speed up large indexing operations. ```shell curl --request PUT localhost:9200/gitlab-production/_settings --header 'Content-Type: application/json' --data '{ @@ -409,6 +423,144 @@ or creating [extra Sidekiq processes](../administration/operations/extra_sidekiq For repository and snippet files, GitLab will only index up to 1 MiB of content, in order to avoid indexing timeouts. +## Zero downtime reindexing + +The idea behind this reindexing method is to leverage Elasticsearch index alias feature to atomically swap between two indices. +We will refer to each index as `primary` (online and used by GitLab for read/writes) and `secondary` (offline, for reindexing purpose). + +Instead of connecting directly to the `primary` index, we'll setup an index alias such as we can change the underlying index at will. + +NOTE: **Note:** +Any index attached to the production alias is deemed a `primary` and will end up being used by the GitLab Elasticsearch integration. + +### Pause the indexing + +Under **Admin Area > Integration > Elasticsearch**, check the **Pause Elasticsearch Indexing** setting and save. + +With this, all updates that should happen on your Elasticsearch index will be buffered and caught up once unpaused. + +### Setup + +TIP: **Tip:** +If your index has been created with GitLab v13.0+ you can skip directly to [trigger the reindex](#trigger-the-reindex-via-the-elasticsearch-administration). + +This process involves multiple shell commands and curl invocations, so a good initial setup will help down the road: + +```shell +# You can find this value under Admin Area > Integration > Elasticsearch > URL +export CLUSTER_URL="http://localhost:9200" +export PRIMARY_INDEX="gitlab-production" +export SECONDARY_INDEX="gitlab-production-$(date +%s)" +``` + +### Reclaiming the `gitlab-production` index name + +CAUTION: **Caution:** +It is highly recommended that you take a snapshot of your cluster to make sure there is a recovery path if anything goes wrong. + +NOTE: **Note:** +Due to a technical limitation, there will be a slight downtime because of the fact that we need to reclaim the current `primary` index to be used as the alias. + +To reclaim the `gitlab-production` index name, you need to first create a `secondary` index and then trigger the re-index from `primary`. + +#### Creating a secondary index + +To create a secondary index, run the following Rake task. The `SKIP_ALIAS` +environment variable will disable the automatic creation of the Elasticsearch +alias, which would conflict with the existing index under `$PRIMARY_INDEX`: + +```shell +# Omnibus installation +sudo SKIP_ALIAS=1 gitlab-rake "gitlab:elastic:create_empty_index[$SECONDARY_INDEX]" + +# Source installation +SKIP_ALIAS=1 bundle exec rake "gitlab:elastic:create_empty_index[$SECONDARY_INDEX]" +``` + +The index should be created successfully, with the latest index options and mappings. + +#### Trigger the re-index from `primary` + +To trigger the re-index from `primary` index: + +1. Use the Elasticsearch [Reindex API](https://www.elastic.co/guide/en/elasticsearch/reference/7.6/docs-reindex.html): + + ```shell + curl --request POST \ + --header 'Content-Type: application/json' \ + --data "{ \"source\": { \"index\": \"$PRIMARY_INDEX\" }, \"dest\": { \"index\": \"$SECONDARY_INDEX\" } }" \ + "$CLUSTER_URL/_reindex?slices=auto&wait_for_completion=false" + ``` + + There will be an output like: + + ```plaintext + {"task":"3qw_Tr0YQLq7PF16Xek8YA:1012"} + ``` + + Note the `task` value here as it will be useful to follow the reindex progress. + +1. Wait for the reindex process to complete, by checking the `completed` value. + Using the `task` value form the previous step: + + ```shell + export TASK_ID=3qw_Tr0YQLq7PF16Xek8YA:1012 + curl "$CLUSTER_URL/_tasks/$TASK_ID?pretty" + ``` + + The output will be like: + + ```plaintext + {"completed":false, …} + ``` + + Once the returned value is `true`, you may continue to the next step. + +1. Make sure that the secondary index has data in it. You can use the Elasticsearch + API to look for the index size and compare our two indices: + + ```shell + curl $CLUSTER_URL/$PRIMARY_INDEX/_count => 123123 + curl $CLUSTER_URL/$SECONDARY_INDEX/_count => 123123 + ``` + + TIP: **Tip:** + Comparing the document count is more accurate than using the index size, as improvements to the storage might cause the new index to be smaller than the original one. + +1. Once you are confident your `secondary` index is valid, you can process to the creation of the alias. + + ```shell + # Delete the original index + curl --request DELETE $CLUSTER_URL/$PRIMARY_INDEX + + # Create the alias and add the `secondary` index to it + curl --request POST \ + --header 'Content-Type: application/json' \ + --data "{\"actions\":[{\"add\":{\"index\":\"$SECONDARY_INDEX\",\"alias\":\"$PRIMARY_INDEX\"}}]}}" \ + $CLUSTER_URL/_aliases + ``` + + The reindexing is now completed. Your GitLab instance is now ready to use the [automated in-cluster reindexing](#trigger-the-reindex-via-the-elasticsearch-administration) feature for future reindexing. + +1. Unpause the indexing + + Under **Admin Area > Integration > Elasticsearch**, uncheck the **Pause Elasticsearch Indexing** setting and save. + +### Trigger the reindex via the Elasticsearch administration + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/34069) in [GitLab Starter](https://about.gitlab.com/pricing/) 13.2. +> - A scheduled index deletion and the ability to cancel it was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/38914) in GitLab Starter 13.3. + +Under **Admin Area > Integration > Elasticsearch zero-downtime reindexing**, click on **Trigger cluster reindexing**. + +NOTE: **Note:** +Reindexing can be a lengthy process depending on the size of your Elasticsearch cluster. + +CAUTION: **Caution:** +After the reindexing is completed, the original index will be scheduled to be deleted after 14 days. You can cancel this action by pressing the cancel button. + +While the reindexing is running, you will be able to follow its progress under that same section. + ## GitLab Elasticsearch Rake tasks Rake tasks are available to: @@ -572,7 +724,7 @@ Here are some common pitfalls and how to overcome them: - **I indexed all the repositories but then switched Elasticsearch servers and now I can't find anything** - You will need to re-run all the Rake tasks to re-index the database, repositories, and wikis. + You will need to re-run all the Rake tasks to reindex the database, repositories, and wikis. - **The indexing process is taking a very long time** @@ -602,7 +754,7 @@ Here are some common pitfalls and how to overcome them: ``` This is because we changed the index mapping in GitLab 8.12 and the old indexes should be removed and built from scratch again, - see details in the [8-11-to-8-12 update guide](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/update/8.11-to-8.12.md#11-elasticsearch-index-update-if-you-currently-use-elasticsearch). + see details in the [update guide](../update/upgrading_from_source.md). - Exception `Elasticsearch::Transport::Transport::Errors::BadRequest` @@ -657,11 +809,11 @@ There is a [more structured, lower-level troubleshooting document](../administra ### Known Issues -- **[Elasticsearch `code_analyzer` doesn't account for all code cases](https://gitlab.com/gitlab-org/gitlab/-/issues/10693)** +- **[Elasticsearch `code_analyzer` doesn't account for all code cases](https://gitlab.com/groups/gitlab-org/-/epics/3621)** - The `code_analyzer` pattern and filter configuration is being evaluated for improvement. We have noticed [several edge cases](https://gitlab.com/gitlab-org/gitlab/-/issues/10693#note_158382332) that are not returning expected search results due to our pattern and filter configuration. + The `code_analyzer` pattern and filter configuration is being evaluated for improvement. We have fixed [most edge cases](https://gitlab.com/groups/gitlab-org/-/epics/3621#note_363429094) that were not returning expected search results due to our pattern and filter configuration. - An improved strategy for the `code_analyzer` pattern and filters are being discussed in [issue 29443](https://gitlab.com/gitlab-org/gitlab/-/issues/29443). + Improvements to the `code_analyzer` pattern and filters is being discussed in [epic 3621](https://gitlab.com/groups/gitlab-org/-/epics/3621). ### Reverting to basic search diff --git a/doc/integration/jenkins.md b/doc/integration/jenkins.md index 485ca67c9fc..8fc638db95a 100644 --- a/doc/integration/jenkins.md +++ b/doc/integration/jenkins.md @@ -3,10 +3,10 @@ NOTE: **Note:** This documentation focuses only on how to **configure** a Jenkins *integration* with GitLab. Learn how to **migrate** from Jenkins to GitLab CI/CD in our -[Migrating from Jenkins](../ci/jenkins/index.md) documentation. +[Migrating from Jenkins](../ci/migration/jenkins.md) documentation. From GitLab, you can trigger a Jenkins build when you push code to a repository, or when a merge -request is created. In return, Jenkins shows the pipeline status on merge requests widgets and +request is created. In return, the Jenkins pipeline status is shown on merge requests widgets and on the GitLab project's home page. To better understand GitLab's Jenkins integration, watch the following video: @@ -62,7 +62,7 @@ Grant a GitLab user access to the select GitLab projects. Create a personal access token to authorize Jenkins' access to GitLab. 1. Log in to GitLab as the user to be used with Jenkins. -1. Click your avatar, then **Settings. +1. Click your avatar, then **Settings**. 1. Click **Access Tokens** in the sidebar. 1. Create a personal access token with the **API** scope checkbox checked. For more details, see [Personal access tokens](../user/profile/personal_access_tokens.md). @@ -147,24 +147,6 @@ Configure the GitLab integration with Jenkins. authentication. 1. Click **Test settings and save changes**. GitLab tests the connection to Jenkins. -## Plugin functional overview - -GitLab does not contain a database table listing commits. Commits are always -read from the repository directly. Therefore, it's not possible to retain the -build status of a commit in GitLab. This is overcome by requesting build -information from the integrated CI tool. The CI tool is responsible for creating -and storing build status for Commits and Merge Requests. - -### Steps required to implement a similar integration - -**Note:** -All steps are implemented using AJAX requests on the merge request page. - -1. In order to display the build status in a merge request you must create a project service in GitLab. -1. Your project service will do a (JSON) query to a URL of the CI tool with the SHA1 of the commit. -1. The project service builds this URL and payload based on project service settings and knowledge of the CI tool. -1. The response is parsed to give a response in GitLab (success/failed/pending). - ## Troubleshooting ### Error in merge requests - "Could not connect to the CI server" diff --git a/doc/integration/jenkins_deprecated.md b/doc/integration/jenkins_deprecated.md index 31bb271782a..5fc30bf3305 100644 --- a/doc/integration/jenkins_deprecated.md +++ b/doc/integration/jenkins_deprecated.md @@ -1,6 +1,7 @@ # Jenkins CI (deprecated) service ->**Note:** In GitLab 8.3, Jenkins integration using the +NOTE: **Note:** +In GitLab 8.3, Jenkins integration using the [GitLab Hook Plugin](https://wiki.jenkins.io/display/JENKINS/GitLab+Hook+Plugin) was deprecated in favor of the [GitLab Plugin](https://wiki.jenkins.io/display/JENKINS/GitLab+Plugin). diff --git a/doc/integration/jira_development_panel.md b/doc/integration/jira_development_panel.md index 85404ff3164..dc19d42ee2e 100644 --- a/doc/integration/jira_development_panel.md +++ b/doc/integration/jira_development_panel.md @@ -1,132 +1,148 @@ -# GitLab Jira development panel integration **(PREMIUM)** +--- +stage: Create +group: Ecosystem +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# GitLab Jira Development Panel integration **(PREMIUM)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/2381) in [GitLab Premium](https://about.gitlab.com/pricing/) 10.0. -Complementary to our [existing Jira](../user/project/integrations/jira.md) project integration, you're now able to integrate -GitLab projects with [Jira Development Panel](https://confluence.atlassian.com/adminjiraserver070/). Both can be used -simultaneously. This works with self-managed GitLab or GitLab.com integrated with: +The Jira Development Panel integration allows you to reference Jira issues within GitLab, displaying activity in the [Development panel](https://support.atlassian.com/jira-software-cloud/docs/view-development-information-for-an-issue/) in the issue. It complements the [GitLab Jira integration](../user/project/integrations/jira.md). You may choose to configure both integrations to take advantage of both sets of features. (See a [feature comparison](../user/project/integrations/jira_integrations.md#feature-comparison)). + +Depending on your environment, you can enable this integration by configuring the Jira DVCS connector or by using the GitLab for Jira app in the Atlassian Marketplace. See the [Configuration](#configuration) section for details. + +## Features -- Jira hosted by you. -- Cloud Jira. +| Your mention of Jira issue ID in GitLab context | Automated effect in Jira issue | +|---------------------------------------------------|--------------------------------------------------------------------------------------------------------| +| In a merge request | Link to the MR is displayed in Development panel. | +| In a branch name | Link to the branch is displayed in Development panel. | +| In a commit message | Link to the commit is displayed in Development panel. | +| In a commit message with Jira Smart Commit format | Displays your custom comment or logged time spent and/or performs specified issue transition on merge. | -By doing this you can easily access related GitLab merge requests, branches, and commits directly from a Jira issue. +With this integration, you can access related GitLab merge requests, branches, and commits directly from a Jira issue, reflecting your work in GitLab. From the Development panel, you can open a detailed view and take actions including creating a new merge request from a branch. For more information, see [Usage](#usage). This integration connects all GitLab projects within a top-level group or a personal namespace to projects in the Jira instance. A top-level GitLab group is one that does not have any parent group itself. All the projects of that top-level group, as well as projects of the top-level group's subgroups nesting down, are connected. Alternatively, you can specify a GitLab personal namespace in the Jira configuration, which will then connect the projects in that personal namespace to Jira. -NOTE: **Note:** -Note this is different from the [existing Jira](../user/project/integrations/jira.md) project integration, where the mapping -is one GitLab project to the entire Jira instance. +This differs from the [Jira integration](../user/project/integrations/jira.md), where the mapping is between one GitLab project and the entire Jira instance. + +## Configuration + +<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> +For an overview, see [Agile Management - GitLab-Jira Development Panel Integration](https://www.youtube.com/watch?v=VjVTOmMl85M&feature=youtu.be). + +- If you're using GitLab.com and Jira Cloud, the recommended method to enable this integration is to install the [GitLab for Jira app](#gitlab-for-jira-app) from the Atlassian Marketplace, which offers a real-time sync between GitLab and Jira. +- If you're using self-managed GitLab, self-managed Jira, or both, configure the integration using [Jira's DVCS Connector](#jira-dvcs-configuration), which syncs data hourly. We recommend that a GitLab group admin or instance admin (in the case of self-managed GitLab) set up the integration, in order to simplify administration. -TIP: **Tip:** -Create and use a single-purpose `jira` user in GitLab, so that removing -regular users won't impact your integration. - -## Requirements - -### Self-managed GitLab +### Jira DVCS configuration -If you are using self-managed GitLab, make sure your GitLab instance is accessible by Jira. +NOTE: **Note:** +If you're using GitLab.com and Jira Cloud, we recommend you use the [GitLab for Jira app](#gitlab-for-jira-app), unless you have a specific need for the DVCS Connector. -- If you are connecting to Jira Cloud, make sure your instance is accessible via the internet. +- If you are using self-managed GitLab, make sure your GitLab instance is accessible by Jira. +- If you're connecting to Jira Cloud, ensure your instance is accessible through the internet. - If you are using Jira Server, make sure your instance is accessible however your network is set up. -### GitLab.com - -There are no special requirements if you are using GitLab.com. +#### GitLab account configuration for DVCS -## GitLab Configuration +TIP: **Tip:** +To ensure that regular user account maintenance doesn't impact your integration, +create and use a single-purpose `jira` user in GitLab. -1. In GitLab, create a new application in order to allow Jira to connect with your GitLab account +1. In GitLab, create a new application to allow Jira to connect with your GitLab account. - While logged-in, go to `Settings -> Applications`. (Click your profile avatar at - the top right, choose `Settings`, and then navigate to `Applications` from the left - navigation menu.) Use the form to create a new application. + While signed in to the GitLab account that you want Jira to use to connect to GitLab, + click your profile avatar at the top right, and then click **Settings > Applications**. + Use the form to create a new application. - Enter a useful name for the `Name` field. + In the **Name** field, enter a descriptive name for the integration, such as `Jira`. - For the `Redirect URI` field, enter `https://<your-gitlab-instance-domain>/login/oauth/callback`, - replacing `<your-gitlab-instance-domain>` appropriately. So for example, if you are using GitLab.com, + For the **Redirect URI** field, enter `https://<gitlab.example.com>/login/oauth/callback`, + replacing `<gitlab.example.com>` with your GitLab instance domain. For example, if you are using GitLab.com, this would be `https://gitlab.com/login/oauth/callback`. NOTE: **Note:** If using a GitLab version earlier than 11.3, the `Redirect URI` must be - `https://<your-gitlab-instance-domain>/-/jira/login/oauth/callback`. If you want Jira - to have access to all projects, GitLab recommends an administrator creates the - Application. + `https://<gitlab.example.com>/-/jira/login/oauth/callback`. If you want Jira + to have access to all projects, GitLab recommends that an administrator create the + application. -  +  - - Check `api` in the Scopes section. + - Check **API** in the Scopes section and uncheck any other checkboxes. -1. Click `Save application`. You will see the generated 'Application ID' and 'Secret' values. - Copy these values that you will use on the Jira configuration side. +1. Click **Save application**. GitLab displays the generated **Application ID** + and **Secret** values. Copy these values, which you will use in Jira. -## Jira Configuration +#### Jira DVCS Connector setup -### GitLab self-managed +NOTE: **Note:** +If you're using GitLab.com and Jira Cloud, we recommend you use the [GitLab for Jira app](#gitlab-for-jira-app), unless you have a specific need for the DVCS Connector. -1. In Jira, go to **Jira Settings > Applications > DVCS accounts**, then click **Link GitHub Enterprise account** to start creating a new integration. - (We are pretending to be GitHub in this integration until there is further platform support from Jira.) +1. Ensure you have completed the [GitLab configuration](#gitlab-account-configuration-for-dvcs). +1. If you're using Jira Server, go to **Settings (gear) > Applications > DVCS accounts**. + If you're using Jira Cloud, go to **Settings (gear) > Products > DVCS accounts**. +1. Click **Link GitHub Enterprise account** to start creating a new integration. + (We're pretending to be GitHub in this integration, until there's additional platform support in Jira.)  -1. Complete the form +1. Complete the form: - Select GitHub Enterprise for the `Host` field. + Select **GitHub Enterprise** for the **Host** field. - For the `Team or User Account` field, enter the relative path of a top-level GitLab group that you have access to, + In the **Team or User Account** field, enter the relative path of a top-level GitLab group that you have access to, or the relative path of your personal namespace.  - For the `Host URL` field, enter `https://<your-gitlab-instance-domain>/`, - replacing `<your-gitlab-instance-domain>` appropriately. So for example, if you are using GitLab.com, + In the **Host URL** field, enter `https://<gitlab.example.com>/`, + replacing `<gitlab.example.com>` with your GitLab instance domain. For example, if you are using GitLab.com, this would be `https://gitlab.com/`. NOTE: **Note:** - If using a GitLab version earlier than 11.3 the `Host URL` value should be `https://<your-gitlab-instance-domain>/-/jira` + If using a GitLab version earlier than 11.3 the **Host URL** value should be `https://<gitlab.example.com>/-/jira` - For the `Client ID` field, use the `Application ID` value from the previous section. + For the **Client ID** field, use the **Application ID** value from the previous section. - For the `Client Secret` field, use the `Secret` value from the previous section. + For the **Client Secret** field, use the **Secret** value from the previous section. Ensure that the rest of the checkboxes are checked. -1. Click `Add` to complete and create the integration. +1. Click **Add** to complete and create the integration. Jira takes up to a few minutes to know about (import behind the scenes) all the commits and branches for all the projects in the GitLab group you specified in the previous step. These are refreshed every 60 minutes. - > **Note:** - > In the future, we plan on implementing real-time integration. If you need - > to refresh the data manually, you can do this from the `Applications -> DVCS - > accounts` screen where you initially set up the integration: - > - >  + In the future, we plan on implementing real-time integration. If you need + to refresh the data manually, you can do this from the `Applications -> DVCS + accounts` screen where you initially set up the integration: + +  -To connect additional GitLab projects from other GitLab top-level groups (or personal namespaces), repeat the above +To connect additional GitLab projects from other GitLab top-level groups (or personal namespaces), repeat the previous steps with additional Jira DVCS accounts. -### GitLab.com +Now that the integration is configured, read more about how to test and use it in [Usage](#usage). -You can integrate GitLab.com and Jira Cloud using the **GitLab for Jira** App in the [Atlassian Marketplace](https://marketplace.atlassian.com/apps/1221011/gitlab-for-jira). +### GitLab for Jira app -GitLab and Jira can also be integrated using the DVCS connector as described in the [GitLab self-managed section](#gitlab-self-managed). The [GitLab for Jira App](https://marketplace.atlassian.com/apps/1221011/gitlab-for-jira) is recommended when using GitLab.com and Jira Cloud because data is synchronized in real time, while the DVCS connector updates data only once per hour. +You can integrate GitLab.com and Jira Cloud using the [GitLab for Jira](https://marketplace.atlassian.com/apps/1221011/gitlab-for-jira) app in the Atlassian Marketplace. + +This method is recommended when using GitLab.com and Jira Cloud because data is synchronized in realtime, while the DVCS connector updates data only once per hour. If you are not using both of these environments, use the [Jira DVCS Connector](#jira-dvcs-configuration) method. <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> For a walkthrough of the integration with GitLab for Jira, watch [Configure GitLab Jira Integration using Marketplace App](https://youtu.be/SwR-g1s1zTo) on YouTube. -NOTE: **Note:** -The **GitLab for Jira** App is only compatible with GitLab.com **and** Jira Cloud. - 1. Go to **Jira Settings > Apps > Find new apps**, then search for GitLab. 1. Click **GitLab for Jira**, then click **Get it now**. Or go the [App in the marketplace directly](https://marketplace.atlassian.com/apps/1221011/gitlab-for-jira) @@ -143,6 +159,8 @@ The GitLab user only needs access when adding a new namespace. For syncing with After a namespace is added, all future commits, branches and merge requests of all projects under that namespace will be synced to Jira. Past data cannot be synced at the moment. +For more information, see [Usage](#usage). + #### Troubleshooting GitLab for Jira The GitLab for Jira App uses an iframe to add namespaces on the settings page. Some browsers block cross-site cookies which can lead to a message saying that the user needs to log in on GitLab.com even though the user is already logged in. @@ -165,6 +183,8 @@ Click the links to see your GitLab repository data.  +For more information on using Jira Smart Commits to track time against an issue, specify an issue transition, or add a custom comment, see the Atlassian page [Using Smart Commits](https://confluence.atlassian.com/fisheye/using-smart-commits-960155400.html). + ## Limitations - This integration is currently not supported on GitLab instances under a [relative URL](https://docs.gitlab.com/omnibus/settings/configuration.html#configuring-a-relative-url-for-gitlab) (for example, `http://example.com/gitlab`). diff --git a/doc/integration/kerberos.md b/doc/integration/kerberos.md index 56f7b50496c..09bc795f7ef 100644 --- a/doc/integration/kerberos.md +++ b/doc/integration/kerberos.md @@ -42,7 +42,7 @@ sudo chmod 0600 /etc/http.keytab **Installations from source** ->**Note:** +NOTE: **Note:** For source installations, make sure the `kerberos` gem group [has been installed](../install/installation.md#install-gems). diff --git a/doc/integration/omniauth.md b/doc/integration/omniauth.md index 2ace05a8320..9dd7f2cd9e1 100644 --- a/doc/integration/omniauth.md +++ b/doc/integration/omniauth.md @@ -1,7 +1,8 @@ # OmniAuth GitLab leverages OmniAuth to allow users to sign in using Twitter, GitHub, and -other popular services. +other popular services. [OmniAuth](https://rubygems.org/gems/omniauth/) is +"a generalized Rack framework for multiple-provider authentication, built on Ruby. Configuring OmniAuth does not prevent standard GitLab authentication or LDAP (if configured) from continuing to work. Users can choose to sign in using any @@ -42,9 +43,9 @@ contains some settings that are common for all providers. Before configuring individual OmniAuth providers there are a few global settings that are in common for all providers that we need to consider. -> **NOTE:** -> Starting from GitLab 11.4, OmniAuth is enabled by default. If you're using an -> earlier version, you'll need to explicitly enable it. +NOTE: **Note:** +Starting from GitLab 11.4, OmniAuth is enabled by default. If you're using an +earlier version, you'll need to explicitly enable it. - `allow_single_sign_on` allows you to specify the providers you want to allow to automatically create an account. It defaults to `false`. If `false` users must @@ -56,16 +57,16 @@ that are in common for all providers that we need to consider. be blocked by default and will have to be unblocked by an administrator before they are able to sign in. -> **Note:** -> If you set `block_auto_created_users` to `false`, make sure to only -> define providers under `allow_single_sign_on` that you are able to control, like -> SAML, Shibboleth, Crowd or Google, or set it to `false` otherwise any user on -> the Internet will be able to successfully sign in to your GitLab without -> administrative approval. -> -> **Note:** -> `auto_link_ldap_user` requires the `uid` of the user to be the same in both LDAP -> and the OmniAuth provider. +NOTE: **Note:** +If you set `block_auto_created_users` to `false`, make sure to only +define providers under `allow_single_sign_on` that you are able to control, like +SAML, Shibboleth, Crowd or Google, or set it to `false` otherwise any user on +the Internet will be able to successfully sign in to your GitLab without +administrative approval. + +NOTE: **Note:** +`auto_link_ldap_user` requires the `uid` of the user to be the same in both LDAP +and the OmniAuth provider. To change these settings: @@ -139,10 +140,26 @@ OmniAuth provider for an existing user. The chosen OmniAuth provider is now active and can be used to sign in to GitLab from then on. +## Automatically Link Existing Users to OmniAuth Users + +You can automatically link OmniAuth users with existing GitLab users if their email addresses match by adding the following setting: + +**For Omnibus installations** + +```ruby +gitlab_rails['omniauth_auto_link_user'] = true +``` + +**For installations from source** + +```yaml +omniauth: + auto_link_user: true +``` + ## Configure OmniAuth Providers as External ->**Note:** -This setting was introduced with version 8.7 of GitLab +> Introduced in GitLab 8.7. You can define which OmniAuth providers you want to be `external` so that all users **creating accounts, or logging in via these providers** will not be able to have @@ -150,7 +167,7 @@ access to internal projects. You will need to use the full name of the provider, like `google_oauth2` for Google. Refer to the examples for the full names of the supported providers. ->**Note:** +NOTE: **Note:** If you decide to remove an OmniAuth provider from the external providers list you will need to manually update the users that use this method to login, if you want their accounts to be upgraded to full internal accounts. @@ -170,7 +187,7 @@ omniauth: ## Using Custom OmniAuth Providers ->**Note:** +NOTE: **Note:** The following information only applies for installations from source. GitLab uses [OmniAuth](https://github.com/omniauth/omniauth) for authentication and already ships @@ -223,12 +240,11 @@ we'd like to at least help those with specific needs. ## Enable or disable Sign In with an OmniAuth provider without disabling import sources ->**Note:** -This setting was introduced with version 8.8 of GitLab. +> Introduced in GitLab 8.8. Administrators are able to enable or disable Sign In via some OmniAuth providers. ->**Note:** +NOTE: **Note:** By default Sign In is enabled via all the OAuth Providers that have been configured in `config/gitlab.yml`. In order to enable/disable an OmniAuth provider, go to Admin Area -> Settings -> Sign-in Restrictions section -> Enabled OAuth Sign-In sources and select the providers you want to enable or disable. @@ -325,3 +341,7 @@ of the OmniAuth users has admin permissions. You may also bypass the auto signin feature by browsing to `https://gitlab.example.com/users/sign_in?auto_sign_in=false`. + +## Passwords for users created via OmniAuth + +The [Generated passwords for users created through integrated authentication](../security/passwords_for_integrated_authentication_methods.md) guide provides an overview of how GitLab generates and sets passwords for users created via OmniAuth. diff --git a/doc/integration/recaptcha.md b/doc/integration/recaptcha.md index 48c83f1393b..1868711ca9c 100644 --- a/doc/integration/recaptcha.md +++ b/doc/integration/recaptcha.md @@ -1,6 +1,6 @@ # reCAPTCHA -GitLab leverages [Google's reCAPTCHA](https://www.google.com/recaptcha/intro/index.html) +GitLab leverages [Google's reCAPTCHA](https://www.google.com/recaptcha/about/) to protect against spam and abuse. GitLab displays the CAPTCHA form on the sign-up page to confirm that a real user, not a bot, is attempting to create an account. @@ -36,5 +36,5 @@ proxy_set_header X-GitLab-Show-Login-Captcha 1; In Omnibus GitLab, this can be configured via `/etc/gitlab/gitlab.rb`: ```ruby -nginx['proxy_set_headers'] = { 'X-GitLab-Show-Login-Captcha' => 1 } +nginx['proxy_set_headers'] = { 'X-GitLab-Show-Login-Captcha' => '1' } ``` diff --git a/doc/integration/saml.md b/doc/integration/saml.md index 9dce7269c86..e7e94b21683 100644 --- a/doc/integration/saml.md +++ b/doc/integration/saml.md @@ -253,8 +253,7 @@ considered admin users. ### Auditor Groups **(STARTER ONLY)** ->**Note:** -This setting is only available on GitLab 11.4 EE and above. +> Introduced in [GitLab Starter](https://about.gitlab.com/pricing/) 11.4. The requirements are the same as the previous settings, your IdP needs to pass Group information to GitLab, you need to tell GitLab where to look for the groups in the SAML response, and which group(s) should be @@ -379,7 +378,7 @@ You may also bypass the auto signin feature by browsing to ### `attribute_statements` ->**Note:** +NOTE: **Note:** This setting should only be used to map attributes that are part of the OmniAuth `info` hash schema. @@ -585,6 +584,10 @@ These attributes define the SAML user. If users can change these attributes, the Refer to the documentation for your SAML Identity Provider for information on how to fix these attributes. +## Passwords for users created via SAML + +The [Generated passwords for users created through integrated authentication](../security/passwords_for_integrated_authentication_methods.md) guide provides an overview of how GitLab generates and sets passwords for users created via SAML. + ## Troubleshooting You can find the base64-encoded SAML Response in the [`production_json.log`](../administration/logs.md#production_jsonlog). diff --git a/doc/integration/shibboleth.md b/doc/integration/shibboleth.md index a4a71c655a2..1b645541cec 100644 --- a/doc/integration/shibboleth.md +++ b/doc/integration/shibboleth.md @@ -46,7 +46,7 @@ The following changes are needed to enable Shibboleth: RequestHeader set X_FORWARDED_PROTO 'https' ``` - **NOTE:** + NOTE: **Note:** Starting from GitLab 11.4, OmniAuth is enabled by default. If you're using an earlier version, you'll need to explicitly enable it in `/etc/gitlab/gitlab.rb`. diff --git a/doc/integration/sourcegraph.md b/doc/integration/sourcegraph.md index 90e9c7b9534..c366dab49b1 100644 --- a/doc/integration/sourcegraph.md +++ b/doc/integration/sourcegraph.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" type: reference, how-to --- diff --git a/doc/intro/README.md b/doc/intro/README.md index 879d406d144..4bf9c766fc4 100644 --- a/doc/intro/README.md +++ b/doc/intro/README.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" comments: false --- diff --git a/doc/migrate_ci_to_ce/README.md b/doc/migrate_ci_to_ce/README.md index 6710356d760..3233cfdbaba 100644 --- a/doc/migrate_ci_to_ce/README.md +++ b/doc/migrate_ci_to_ce/README.md @@ -205,7 +205,7 @@ If you were running GitLab and GitLab CI on the same server you can skip this step. Copy your CI data archive to your GitLab server. There are many ways to do -this, below we use SSH agent forwarding and 'scp', which will be easy and fast +this, below we use SSH agent forwarding and `scp`, which will be easy and fast for most setups. You can also copy the data archive first from the CI server to your laptop and then from your laptop to the GitLab server. diff --git a/doc/operations/error_tracking.md b/doc/operations/error_tracking.md new file mode 100644 index 00000000000..c5699ee3d22 --- /dev/null +++ b/doc/operations/error_tracking.md @@ -0,0 +1,97 @@ +--- +stage: Monitor +group: Health +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Error Tracking + +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/169) in GitLab 11.8. + +Error Tracking allows developers to easily discover and view the errors that their application may be generating. By surfacing error information where the code is being developed, efficiency and awareness can be increased. + +## Sentry error tracking + +[Sentry](https://sentry.io/) is an open source error tracking system. GitLab allows administrators to connect Sentry to GitLab, to allow users to view a list of Sentry errors in GitLab. + +### Deploying Sentry + +You can sign up to the cloud hosted <https://sentry.io>, deploy your own [on-premise instance](https://github.com/getsentry/onpremise/) or use GitLab to [install Sentry to a Kubernetes cluster](../user/clusters/applications.md#install-sentry-using-gitlab-cicd). + +### Enabling Sentry + +NOTE: **Note:** +You will need at least Maintainer [permissions](../user/permissions.md) to enable the Sentry integration. + +GitLab provides an easy way to connect Sentry to your project: + +1. Sign up to Sentry.io or [deploy your own](#deploying-sentry) Sentry instance. +1. [Create](https://docs.sentry.io/product/sentry-basics/guides/integrate-frontend/create-new-project/) a new Sentry project. For each GitLab project that you want to integrate, we recommend that you create a new Sentry project. +1. [Find or generate](https://docs.sentry.io/api/auth/) a Sentry auth token for your Sentry project. + Make sure to give the token at least the following scopes: `event:read` and `project:read`. +1. In GitLab, navigate to your project’s **Operations > Error Tracking** page, and + click **Enable Error Tracking**. +1. Navigate to your project’s **Settings > Operations**. In the **Error Tracking** section, + ensure the **Active** checkbox is set. +1. In the **Sentry API URL** field, enter your Sentry hostname. For example, enter `https://sentry.example.com` if this is the address at which your Sentry instance is available. For the SaaS version of Sentry, the hostname will be `https://sentry.io`. +1. In the **Auth Token** field, enter the token you previously generated. +1. Click the **Connect** button to test the connection to Sentry and populate the **Project** dropdown. +1. From the **Project** dropdown, choose a Sentry project to link to your GitLab project. +1. Click **Save changes** for the changes to take effect. +1. You can now visit **Operations > Error Tracking** in your project's sidebar to [view a list](#error-tracking-list) of Sentry errors. + +### Enabling GitLab issues links + +You may also want to enable Sentry's GitLab integration by following the steps in the [Sentry documentation](https://docs.sentry.io/product/integrations/gitlab/) + +## Error Tracking List + +NOTE: **Note:** +You will need at least Reporter [permissions](../user/permissions.md) to view the Error Tracking list. + +You can find the Error Tracking list at **Operations > Error Tracking** in your project's sidebar. +Here, you can filter errors by title or by status (one of Ignored , Resolved, or Unresolved) and sort in descending order by Frequency, First Seen, or Last Seen. By default, the error list is ordered by Last Seen and filtered to Unresolved errors. + + + +## Error Details + +From error list, users can navigate to the error details page by clicking the title of any error. + +This page has: + +- A link to the Sentry issue. +- A link to the GitLab commit if the Sentry [release ID/version](https://docs.sentry.io/product/releases/?platform=javascript#configure-sdk) on the Sentry Issue's first release matches a commit SHA in your GitLab hosted project. +- Other details about the issue, including a full stack trace. +- In [GitLab 12.7 and newer](https://gitlab.com/gitlab-org/gitlab/-/issues/36246), language and urgency are displayed. + +By default, a **Create issue** button is displayed: + + + +If you create a GitLab issue from the error, the **Create issue** button will change to a **View issue** button and a link to the GitLab issue will surface within the error detail section: + + + +## Taking Action on errors + +You can take action on Sentry Errors from within the GitLab UI. + +### Ignoring errors + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/39665) in GitLab 12.7. + +From within the [Error Details](#error-details) page you can ignore a Sentry error by simply clicking the **Ignore** button near the top of the page. + +Ignoring an error will prevent it from appearing in the [Error Tracking List](#error-tracking-list), and will silence notifications that were set up within Sentry. + +### Resolving errors + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/39825) in GitLab 12.7. + +From within the [Error Details](#error-details) page you can resolve a Sentry error by +clicking the **Resolve** button near the top of the page. + +Marking an error as resolved indicates that the error has stopped firing events. If a GitLab issue is linked to the error, then the issue will be closed. + +If another event occurs, the error reverts to unresolved. diff --git a/doc/operations/feature_flags.md b/doc/operations/feature_flags.md index 7614d70e132..46a57e72484 100644 --- a/doc/operations/feature_flags.md +++ b/doc/operations/feature_flags.md @@ -69,9 +69,11 @@ It can be set to: ## Feature flag strategies > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/35555) in GitLab 13.0. -> - It's deployed behind a feature flag, disabled by default. +> - It was deployed behind a feature flag, disabled by default. +> - It became [enabled by default](https://gitlab.com/gitlab-org/gitlab/-/issues/214684) in GitLab 13.2. +> - It's recommended for production use. > - It's enabled on GitLab.com. -> - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enable-or-disable-feature-flag-strategies). **(CORE ONLY)** +> - For GitLab self-managed instances, a GitLab administrator can choose to [disable it](#enable-or-disable-feature-flag-strategies). **(CORE ONLY)** You can apply a feature flag strategy across multiple environments, without defining the strategy multiple times. @@ -134,27 +136,72 @@ target users. See the [Ruby example](#ruby-application-example) below. > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/35930) in GitLab 13.1. -Enables the feature for lists of users created with the [Feature Flag User List API](../api/feature_flag_user_lists.md). +Enables the feature for lists of users created [in the Feature Flags UI](#create-a-user-list), or with the [Feature Flag User List API](../api/feature_flag_user_lists.md). Similar to [User IDs](#user-ids), it uses the Unleash [`userWithId`](https://unleash.github.io/docs/activation_strategy#userwithid) activation strategy. +It's not possible to *disable* a feature for members of a user list, but you can achieve the same +effect by enabling a feature for a user list that doesn't contain the excluded users. + +For example: + +- `Full-user-list` = `User1A, User1B, User2A, User2B, User3A, User3B, ...` +- `Full-user-list-excluding-B-users` = `User1A, User2A, User3A, ...` + +#### Create a user list + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/13308) in GitLab 13.3. + +To create a user list: + +1. In your project, navigate to **Operations > Feature Flags**. +1. Click on **New list**. +1. Enter a name for the list. +1. Click **Create**. + +You can view a list's User IDs by clicking the **{pencil}** (edit) button next to it. +When viewing a list, you can rename it by clicking the **Edit** button. + +#### Add users to a user list + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/13308) in GitLab 13.3. + +To add users to a user list: + +1. In your project, navigate to **Operations > Feature Flags**. +1. Click on the **{pencil}** (edit) button next to the list you want to add users to. +1. Click on **Add Users**. +1. Enter the user IDs as a comma-separated list of values. For example, + `user@example.com, user2@example.com`, or `username1,username2,username3`, and so on. +1. Click on **Add**. + +#### Remove users from a user list + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/13308) in GitLab 13.3. + +To remove users from a user list: + +1. In your project, navigate to **Operations > Feature Flags**. +1. Click on the **{pencil}** (edit) button next to the list you want to change. +1. Click on the **{remove}** (remove) button next to the ID you want to remove. + ### Enable or disable feature flag strategies -This feature is under development, but is ready for testing. It's -deployed behind a feature flag that is **disabled by default**. +This feature is under development, but is ready for production use. It's +deployed behind a feature flag that is **enabled by default**. [GitLab administrators with access to the GitLab Rails console](../administration/feature_flags.md) -can enable it for your instance. +can disable it for your instance. -To enable it: +To disable it: ```ruby -Feature.enable(:feature_flags_new_version) +Feature.disable(:feature_flags_new_version) ``` -To disable it: +To enable it: ```ruby -Feature.disable(:feature_flags_new_version) +Feature.enable(:feature_flags_new_version) ``` ## Disable a feature flag for a specific environment @@ -194,9 +241,11 @@ To get the access credentials that your application needs to communicate with Gi 1. Click the **Configure** button to view the following: - **API URL**: URL where the client (application) connects to get a list of feature flags. - **Instance ID**: Unique token that authorizes the retrieval of the feature flags. - - **Application name**: The name of the running environment. For instance, - if the application runs for a production server, the application name would be - `production` or similar. This value is used for the environment spec evaluation. + - **Application name**: The name of the *environment* the application runs in + (not the name of the application itself). + + For example, if the application runs for a production server, the **Application name** + could be `production` or similar. This value is used for the environment spec evaluation. NOTE: **Note:** The meaning of these fields might change over time. For example, we are not sure @@ -245,7 +294,7 @@ func init() { unleash.Initialize( unleash.WithUrl("https://gitlab.com/api/v4/feature_flags/unleash/42"), unleash.WithInstanceId("29QmjsW6KngPR5JNPMWx"), - unleash.WithAppName("production"), + unleash.WithAppName("production"), // Set to the running environment of your application unleash.WithListener(&metricsInterface{}), ) } @@ -278,7 +327,7 @@ require 'unleash/context' unleash = Unleash::Client.new({ url: 'http://gitlab.com/api/v4/feature_flags/unleash/42', - app_name: 'production', + app_name: 'production', # Set to the running environment of your application instance_id: '29QmjsW6KngPR5JNPMWx' }) diff --git a/doc/user/project/operations/img/error_details_v12_7.png b/doc/operations/img/error_details_v12_7.png Binary files differindex 1c7ace35e2a..1c7ace35e2a 100644 --- a/doc/user/project/operations/img/error_details_v12_7.png +++ b/doc/operations/img/error_details_v12_7.png diff --git a/doc/user/project/operations/img/error_details_with_issue_v12_8.png b/doc/operations/img/error_details_with_issue_v12_8.png Binary files differindex 0536861b070..0536861b070 100644 --- a/doc/user/project/operations/img/error_details_with_issue_v12_8.png +++ b/doc/operations/img/error_details_with_issue_v12_8.png diff --git a/doc/user/project/operations/img/error_tracking_list_v12_6.png b/doc/operations/img/error_tracking_list_v12_6.png Binary files differindex b99c83c14d3..b99c83c14d3 100644 --- a/doc/user/project/operations/img/error_tracking_list_v12_6.png +++ b/doc/operations/img/error_tracking_list_v12_6.png diff --git a/doc/operations/incident_management/alertdetails.md b/doc/operations/incident_management/alertdetails.md new file mode 100644 index 00000000000..774eaee286f --- /dev/null +++ b/doc/operations/incident_management/alertdetails.md @@ -0,0 +1,194 @@ +--- +stage: Monitor +group: Health +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Alert details page + +Navigate to the Alert details view by visiting the +[Alert list](alerts.md) and selecting an alert from the +list. You need least Developer [permissions](../../user/permissions.md) to access +alerts. + +Alerts provide **Overview** and **Alert details** tabs to give you the right +amount of information you need. + +## Alert overview tab + +The **Overview** tab provides basic information about the alert: + + + +## Alert details tab + + + +### Update an Alert's status + +The Alert detail view enables you to update the Alert Status. +See [Create and manage alerts in GitLab](alerts.md) for more details. + +### Create an Issue from an Alert + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/217745) in GitLab 13.1. + +The Alert detail view enables you to create an issue with a +description automatically populated from an alert. To create the issue, +click the **Create Issue** button. You can then view the issue from the +alert by clicking the **View Issue** button. + +Closing a GitLab issue associated with an alert changes the alert's status to Resolved. +See [Create and manage alerts in GitLab](alerts.md) for more details about alert statuses. + +### Update an Alert's assignee + +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3066) in GitLab 13.1. + +The Alert detail view allows users to update the Alert assignee. + +In large teams, where there is shared ownership of an alert, it can be difficult +to track who is investigating and working on it. The Alert detail view +enables you to update the Alert assignee: + +NOTE: **Note:** +GitLab currently only supports a single assignee per alert. + +1. To display the list of current alerts, click + **{cloud-gear}** **Operations > Alerts**: + +  + +1. Select your desired alert to display its **Alert Details View**: + +  + +1. If the right sidebar is not expanded, click + **{angle-double-right}** **Expand sidebar** to expand it. +1. In the right sidebar, locate the **Assignee** and click **Edit**. From the + dropdown menu, select each user you want to assign to the alert. GitLab creates + a [To-Do list item](../../user/todos.md) for each user. + +  + +To remove an assignee, click **Edit** next to the **Assignee** dropdown menu and +deselect the user from the list of assignees, or click **Unassigned**. + +### Alert system notes + +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3066) in GitLab 13.1. + +When you take action on an alert, this is logged as a system note, +which is visible in the Alert Details view. This gives you a linear +timeline of the alert's investigation and assignment history. + +The following actions will result in a system note: + +- [Updating the status of an alert](#update-an-alerts-status) +- [Creating an issue based on an alert](#create-an-issue-from-an-alert) +- [Assignment of an alert to a user](#update-an-alerts-assignee) + + + +### Create a To-Do from an Alert + +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3066) in GitLab 13.1. + +You can manually create [To-Do list items](../../user/todos.md) for yourself from the +Alert details screen, and view them later on your **To-Do List**. To add a To-Do: + +1. To display the list of current alerts, click + **{cloud-gear}** **Operations > Alerts**. +1. Select your desired alert to display its **Alert Management Details View**. +1. Click the **Add a To-Do** button in the right sidebar: + +  + +Click the **To-Do** **{todo-done}** in the navigation bar to view your current To-Do list. + + + +### View an Alert's metrics data + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/217768) in GitLab 13.2. + +To view the metrics for an alert: + + 1. Sign in as a user with Developer or higher [permissions](../../user/permissions.md). + 1. Navigate to **{cloud-gear}** **Operations > Alerts**. + 1. Click the alert you want to view. + 1. Below the title of the alert, click the **Metrics** tab. + + + +For GitLab-managed Prometheus instances, metrics data is automatically available +for the alert, making it easy to see surrounding behavior. See +[Managed Prometheus instances](../metrics/alerts.md#managed-prometheus-instances) +for information on setting up alerts. + +For externally-managed Prometheus instances, you can configure your alerting rules to +display a chart in the alert. See +[Embedding metrics based on alerts in incident issues](../metrics/embed.md#embedding-metrics-based-on-alerts-in-incident-issues) +for information on how to appropriately configure your alerting rules. See +[External Prometheus instances](../metrics/alerts.md#external-prometheus-instances) +for information on setting up alerts for your self-managed Prometheus instance. + +## Use cases for assigning alerts + +Consider a team formed by different sections of monitoring, collaborating on a +single application. After an alert surfaces, it's extremely important to +route the alert to the team members who can address and resolve the alert. + +Assigning Alerts eases collaboration and delegation. All +assignees are shown in your team's work-flows, and all assignees receive +notifications, simplifying communication and ownership of the alert. + +After completing their portion of investigating or fixing the alert, users can +unassign their account from the alert when their role is complete. +The alert status can be updated on the [Alert list](alerts.md) to +reflect if the alert has been resolved. + +## View an Alert's logs + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/217768) in GitLab 13.3. + +To view the logs for an alert: + + 1. Sign in as a user with Developer or higher [permissions](../../user/permissions.md). + 1. Navigate to **{cloud-gear}** **Operations > Alerts**. + 1. Click the alert you want to view. + 1. Below the title of the alert, click the **Metrics** tab. + 1. Click the [menu](../metrics/dashboards/index.md#chart-context-menu) of the metric chart to view options. + 1. Click **View logs**. + +Read [View logs from metrics panel](#view-logs-from-metrics-panel) for additional information. + +## Embed metrics in incidents and issues + +You can embed metrics anywhere [GitLab Markdown](../../user/markdown.md) is used, such as descriptions, +comments on issues, and merge requests. Embedding metrics helps you share them +when discussing incidents or performance issues. You can output the dashboard directly +into any issue, merge request, epic, or any other Markdown text field in GitLab +by [copying and pasting the link to the metrics dashboard](../metrics/embed.md#embedding-gitlab-managed-kubernetes-metrics). + +You can embed both +[GitLab-hosted metrics](../metrics/embed.md) and +[Grafana metrics](../metrics/embed_grafana.md) +in incidents and issue templates. + +### Context menu + +You can view more details about an embedded metrics panel from the context menu. +To access the context menu, click the **{ellipsis_v}** **More actions** dropdown box +above the upper right corner of the panel. For a list of options, see +[Chart context menu](../metrics/dashboards/index.md#chart-context-menu). + +#### View logs from metrics panel + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/201846) in GitLab Ultimate 12.8. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/25455) to [GitLab Core](https://about.gitlab.com/pricing/) 12.9. + +Viewing logs from a metrics panel can be useful if you're triaging an application +incident and need to [explore logs](../metrics/dashboards/index.md#chart-context-menu) +from across your application. These logs help you understand what is affecting +your application's performance and resolve any problems. diff --git a/doc/operations/incident_management/alerts.md b/doc/operations/incident_management/alerts.md new file mode 100644 index 00000000000..5a5fc59d5e3 --- /dev/null +++ b/doc/operations/incident_management/alerts.md @@ -0,0 +1,118 @@ +--- +stage: Monitor +group: Health +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Create and manage alerts in GitLab + +Users with at least Developer [permissions](../../user/permissions.md) can access +the Alert Management list at **{cloud-gear}** **Operations > Alerts** in your +project's sidebar. The Alert Management list displays alerts sorted by start time, +but you can change the sort order by clicking the headers in the Alert Management list. +([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/217745) in GitLab 13.1.) + +The alert list displays the following information: + + + +- **Search** - The alert list supports a simple free text search on the title, + description, monitoring tool, and service fields. + ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/213884) in GitLab 13.1.) +- **Severity** - The current importance of a alert and how much attention it should + receive. For a listing of all statuses, read [Alert Management severity](#alert-severity). +- **Start time** - How long ago the alert fired. This field uses the standard + GitLab pattern of `X time ago`, but is supported by a granular date/time tooltip + depending on the user's locale. +- **Alert description** - The description of the alert, which attempts to capture the most meaningful data. +- **Event count** - The number of times that an alert has fired. +- **Issue** - A link to the incident issue that has been created for the alert. +- **Status** - The current status of the alert: + - **Triggered**: No one has begun investigation. + - **Acknowledged**: Someone is actively investigating the problem. + - **Resolved**: No further work is required. + +## Enable Alerts + +NOTE: **Note:** +You need at least Maintainer [permissions](../../user/permissions.md) to enable +the Alerts feature. + +There are several ways to accept alerts into your GitLab project. +Enabling any of these methods enables the Alert list. After configuring +alerts, visit **{cloud-gear}** **Operations > Alerts** in your project's sidebar +to view the list of alerts. + +### Enable GitLab-managed Prometheus alerts + +You can install the GitLab-managed Prometheus application on your Kubernetes +cluster. For more information, read +[Managed Prometheus on Kubernetes](../../user/project/integrations/prometheus.md#managed-prometheus-on-kubernetes). +When GitLab-managed Prometheus is installed, the [Alerts list](alerts.md) +is also enabled. + +To populate the alerts with data, read +[GitLab-Managed Prometheus instances](../metrics/alerts.md#managed-prometheus-instances). + +### Enable external Prometheus alerts + +You can configure an externally-managed Prometheus instance to send alerts +to GitLab. To set up this configuration, read the [configuring Prometheus](../metrics/alerts.md#external-prometheus-instances) documentation. Activating the external Prometheus +configuration also enables the [Alerts list](alerts.md). + +To populate the alerts with data, read +[External Prometheus instances](../metrics/alerts.md#external-prometheus-instances). + +### Enable a Generic Alerts endpoint + +GitLab provides the Generic Alerts endpoint so you can accept alerts from a third-party +alerts service. Read the +[instructions for toggling generic alerts](../../user/project/integrations/generic_alerts.md#setting-up-generic-alerts) +to add this option. After configuring the endpoint, the +[Alerts list](alerts.md) is enabled. + +To populate the alerts with data, read [Customizing the payload](../../user/project/integrations/generic_alerts.md#customizing-the-payload) for requests to the alerts endpoint. + +### Opsgenie integration **(PREMIUM)** + +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3066) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.2. + +A new way of monitoring Alerts via a GitLab integration is with +[Opsgenie](https://www.atlassian.com/software/opsgenie). + +NOTE: **Note:** +If you enable the Opsgenie integration, you can't have other GitLab alert services, +such as [Generic Alerts](../../user/project/integrations/generic_alerts.md) or +Prometheus alerts, active at the same time. + +To enable Opsgenie integration: + +1. Sign in as a user with Maintainer or Owner [permissions](../../user/permissions.md). +1. Navigate to **{cloud-gear}** **Operations > Alerts**. +1. In the **Integrations** select box, select Opsgenie. +1. Click the **Active** toggle. +1. In the **API URL**, enter the base URL for your Opsgenie integration, such + as `https://app.opsgenie.com/alert/list`. +1. Click **Save changes**. + +After enabling the integration, navigate to the Alerts list page at +**{cloud-gear}** **Operations > Alerts**, and click **View alerts in Opsgenie**. + +## Alert severity + +Each level of alert contains a uniquely shaped and color-coded icon to help +you identify the severity of a particular alert. These severity icons help you +immediately identify which alerts you should prioritize investigating: + + + +Alerts contain one of the following icons: + +| Severity | Icon | Color (hexadecimal) | +|---|---|---| +| Critical | **{severity-critical}** | `#8b2615` | +| High | **{severity-high}** | `#c0341d` | +| Medium | **{severity-medium}** | `#fca429` | +| Low | **{severity-low}** | `#fdbc60` | +| Info | **{severity-info}** | `#418cd8` | +| Unknown | **{severity-unknown}** | `#bababa` | diff --git a/doc/operations/incident_management/img/alert_detail_add_todo_v13_1.png b/doc/operations/incident_management/img/alert_detail_add_todo_v13_1.png Binary files differnew file mode 100644 index 00000000000..39aa9e33728 --- /dev/null +++ b/doc/operations/incident_management/img/alert_detail_add_todo_v13_1.png diff --git a/doc/operations/incident_management/img/alert_detail_added_todo_v13_1.png b/doc/operations/incident_management/img/alert_detail_added_todo_v13_1.png Binary files differnew file mode 100644 index 00000000000..ae874706895 --- /dev/null +++ b/doc/operations/incident_management/img/alert_detail_added_todo_v13_1.png diff --git a/doc/user/project/operations/img/alert_detail_full_v13_1.png b/doc/operations/incident_management/img/alert_detail_full_v13_1.png Binary files differindex 18a6f4fb67b..18a6f4fb67b 100644 --- a/doc/user/project/operations/img/alert_detail_full_v13_1.png +++ b/doc/operations/incident_management/img/alert_detail_full_v13_1.png diff --git a/doc/user/project/operations/img/alert_detail_metrics_v13_2.png b/doc/operations/incident_management/img/alert_detail_metrics_v13_2.png Binary files differindex 84d83365ea8..84d83365ea8 100644 --- a/doc/user/project/operations/img/alert_detail_metrics_v13_2.png +++ b/doc/operations/incident_management/img/alert_detail_metrics_v13_2.png diff --git a/doc/user/project/operations/img/alert_detail_overview_v13_1.png b/doc/operations/incident_management/img/alert_detail_overview_v13_1.png Binary files differindex 10c945d3810..10c945d3810 100644 --- a/doc/user/project/operations/img/alert_detail_overview_v13_1.png +++ b/doc/operations/incident_management/img/alert_detail_overview_v13_1.png diff --git a/doc/user/project/operations/img/alert_detail_system_notes_v13_1.png b/doc/operations/incident_management/img/alert_detail_system_notes_v13_1.png Binary files differindex 2a6d0320a54..2a6d0320a54 100644 --- a/doc/user/project/operations/img/alert_detail_system_notes_v13_1.png +++ b/doc/operations/incident_management/img/alert_detail_system_notes_v13_1.png diff --git a/doc/user/project/operations/img/alert_details_assignees_v13_1.png b/doc/operations/incident_management/img/alert_details_assignees_v13_1.png Binary files differindex dab4eac384a..dab4eac384a 100644 --- a/doc/user/project/operations/img/alert_details_assignees_v13_1.png +++ b/doc/operations/incident_management/img/alert_details_assignees_v13_1.png diff --git a/doc/user/project/operations/img/alert_list_assignees_v13_1.png b/doc/operations/incident_management/img/alert_list_assignees_v13_1.png Binary files differindex db1e0d8dcb7..db1e0d8dcb7 100644 --- a/doc/user/project/operations/img/alert_list_assignees_v13_1.png +++ b/doc/operations/incident_management/img/alert_list_assignees_v13_1.png diff --git a/doc/user/project/operations/img/alert_list_search_v13_1.png b/doc/operations/incident_management/img/alert_list_search_v13_1.png Binary files differindex ba993fe530b..ba993fe530b 100644 --- a/doc/user/project/operations/img/alert_list_search_v13_1.png +++ b/doc/operations/incident_management/img/alert_list_search_v13_1.png diff --git a/doc/user/project/operations/img/alert_list_sort_v13_1.png b/doc/operations/incident_management/img/alert_list_sort_v13_1.png Binary files differindex 8e06c3478f7..8e06c3478f7 100644 --- a/doc/user/project/operations/img/alert_list_sort_v13_1.png +++ b/doc/operations/incident_management/img/alert_list_sort_v13_1.png diff --git a/doc/user/project/operations/img/alert_management_severity_v13_0.png b/doc/operations/incident_management/img/alert_management_severity_v13_0.png Binary files differindex f996d6e88f4..f996d6e88f4 100644 --- a/doc/user/project/operations/img/alert_management_severity_v13_0.png +++ b/doc/operations/incident_management/img/alert_management_severity_v13_0.png diff --git a/doc/user/project/operations/img/alert_todo_assignees_v13_1.png b/doc/operations/incident_management/img/alert_todo_assignees_v13_1.png Binary files differindex 637f8be5d25..637f8be5d25 100644 --- a/doc/user/project/operations/img/alert_todo_assignees_v13_1.png +++ b/doc/operations/incident_management/img/alert_todo_assignees_v13_1.png diff --git a/doc/operations/incident_management/img/incident_list.png b/doc/operations/incident_management/img/incident_list.png Binary files differnew file mode 100644 index 00000000000..0498fec6c9c --- /dev/null +++ b/doc/operations/incident_management/img/incident_list.png diff --git a/doc/operations/incident_management/img/incident_list_create_v13_3.png b/doc/operations/incident_management/img/incident_list_create_v13_3.png Binary files differnew file mode 100644 index 00000000000..a000c849099 --- /dev/null +++ b/doc/operations/incident_management/img/incident_list_create_v13_3.png diff --git a/doc/operations/incident_management/img/incident_list_search_v13_3.png b/doc/operations/incident_management/img/incident_list_search_v13_3.png Binary files differnew file mode 100644 index 00000000000..293268986cc --- /dev/null +++ b/doc/operations/incident_management/img/incident_list_search_v13_3.png diff --git a/doc/operations/incident_management/img/incident_list_sort_v13_3.png b/doc/operations/incident_management/img/incident_list_sort_v13_3.png Binary files differnew file mode 100644 index 00000000000..4a263aa188e --- /dev/null +++ b/doc/operations/incident_management/img/incident_list_sort_v13_3.png diff --git a/doc/operations/incident_management/img/incident_management_settings_v13_3.png b/doc/operations/incident_management/img/incident_management_settings_v13_3.png Binary files differnew file mode 100644 index 00000000000..c9520860414 --- /dev/null +++ b/doc/operations/incident_management/img/incident_management_settings_v13_3.png diff --git a/doc/operations/incident_management/img/pagerduty_incidents_integration_v13_3.png b/doc/operations/incident_management/img/pagerduty_incidents_integration_v13_3.png Binary files differnew file mode 100644 index 00000000000..0991e963e02 --- /dev/null +++ b/doc/operations/incident_management/img/pagerduty_incidents_integration_v13_3.png diff --git a/doc/user/project/img/status_page_detail_link_v13_1.png b/doc/operations/incident_management/img/status_page_detail_link_v13_1.png Binary files differindex f3d1005447c..f3d1005447c 100644 --- a/doc/user/project/img/status_page_detail_link_v13_1.png +++ b/doc/operations/incident_management/img/status_page_detail_link_v13_1.png diff --git a/doc/user/project/img/status_page_detail_v12_10.png b/doc/operations/incident_management/img/status_page_detail_v12_10.png Binary files differindex d8dbbb539e6..d8dbbb539e6 100644 --- a/doc/user/project/img/status_page_detail_v12_10.png +++ b/doc/operations/incident_management/img/status_page_detail_v12_10.png diff --git a/doc/user/project/img/status_page_incidents_v12_10.png b/doc/operations/incident_management/img/status_page_incidents_v12_10.png Binary files differindex 3540fbffcf8..3540fbffcf8 100644 --- a/doc/user/project/img/status_page_incidents_v12_10.png +++ b/doc/operations/incident_management/img/status_page_incidents_v12_10.png diff --git a/doc/operations/incident_management/incidents.md b/doc/operations/incident_management/incidents.md new file mode 100644 index 00000000000..0668dc72c22 --- /dev/null +++ b/doc/operations/incident_management/incidents.md @@ -0,0 +1,103 @@ +--- +stage: Monitor +group: Health +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Create and manage incidents in GitLab + +While no configuration is required to use the [manual features](#create-an-incident-manually) +of incident management, some simple [configuration](#configure-incidents) is needed to automate incident creation. + +For users with at least Developer [permissions](../../user/permissions.md), the +Incident Management list is available at **Operations > Incidents** +in your project's sidebar. The list contains the following metrics: + + + +- **Status** - To filter incidents by their status, click **Open**, **Closed**, + or **All** above the incident list. +- **Search** - The Incident list supports a simple free text search, which filters + on the **Title** and **Incident** fields. +- **Incident** - The description of the incident, which attempts to capture the + most meaningful data. +- **Date created** - How long ago the incident was created. This field uses the + standard GitLab pattern of `X time ago`, but is supported by a granular date/time + tooltip depending on the user's locale. +- **Assignees** - The user assigned to the incident. +- **Published** - Displays a green check mark (**{check-circle}**) if the incident is published + to a [Status Page](status_page.md).. **(ULTIMATE)** + +The Incident list displays incidents sorted by incident created date. +([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/229534) to GitLab core in 13.3).) +To see if a column is sortable, point your mouse at the header. Sortable columns +display an arrow next to the column name. + +NOTE: **Note:** +Incidents share the [Issues API](../../user/project/issues/index.md). + +## Configure incidents + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/4925) in GitLab Ultimate 11.11. + +With Maintainer or higher [permissions](../../user/permissions.md), you can enable +or disable Incident Management features in the GitLab user interface +to create issues when alerts are triggered: + +1. Navigate to **Settings > Operations > Incidents** and expand + **Incidents**: + +  + +1. For GitLab versions 11.11 and greater, you can select the **Create an issue** + checkbox to create an issue based on your own + [issue templates](../../user/project/description_templates.md#creating-issue-templates). + For more information, see + [Trigger actions from alerts](../metrics/alerts.md#trigger-actions-from-alerts-ultimate) **(ULTIMATE)**. +1. To create issues from alerts, select the template in the **Issue Template** + select box. +1. To send [separate email notifications](index.md#notify-developers-of-alerts) to users + with [Developer permissions](../../user/permissions.md), select + **Send a separate email notification to Developers**. +1. Click **Save changes**. + +Appropriately configured alerts include an +[embedded chart](../metrics/embed.md#embedding-metrics-based-on-alerts-in-incident-issues) +for the query corresponding to the alert. You can also configure GitLab to +[close issues](../metrics/alerts.md#trigger-actions-from-alerts-ultimate) +when you receive notification that the alert is resolved. + +## Create an incident manually + +> [Moved](https://gitlab.com/gitlab-org/monitor/health/-/issues/24) to GitLab core in 13.3. + +For users with at least Developer [permissions](../../user/permissions.md), to create a Incident you can take any of the following actions: + +- Navigate to **Operations > Incidents** and click **Create Incident**. +- Create a new issue using the `incident` template available when creating it. +- Create a new issue and assign the `incident` label to it. + + + +## Configure PagerDuty integration + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/119018) in GitLab 13.3. + +You can set up a webhook with PagerDuty to automatically create a GitLab issue +for each PagerDuty incident. This configuration requires you to make changes +in both PagerDuty and GitLab: + +1. Sign in as a user with Maintainer [permissions](../../user/permissions.md). +1. Navigate to **Settings > Operations > Incidents** and expand **Incidents**. +1. Select the **PagerDuty integration** tab: + +  + +1. Activate the integration, and save the changes in GitLab. +1. Copy the value of **Webhook URL** for use in a later step. +1. Follow the steps described in the + [PagerDuty documentation](https://support.pagerduty.com/docs/webhooks) + to add the webhook URL to a PagerDuty webhook integration. + +To confirm the integration is successful, trigger a test incident from PagerDuty to +confirm that a GitLab issue is created from the incident. diff --git a/doc/operations/incident_management/index.md b/doc/operations/incident_management/index.md new file mode 100644 index 00000000000..53a6b47ec4b --- /dev/null +++ b/doc/operations/incident_management/index.md @@ -0,0 +1,68 @@ +--- +stage: Monitor +group: Health +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Incident management + +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/2877) in GitLab 13.0. + +Alert Management enables developers to easily discover and view the alerts +generated by their application. By surfacing alert information where the code is +being developed, efficiency and awareness can be increased. + +GitLab offers solutions for handling incidents in your applications and services, +such as [setting up Prometheus alerts](#configure-prometheus-alerts), +[displaying metrics](alertdetails.md#embed-metrics-in-incidents-and-issues), and sending notifications. + +## Alert notifications + +### Slack Notifications + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/216326) in GitLab 13.1. + +You can be alerted via a Slack message when a new alert has been received. + +See the [Slack Notifications Service docs](../../user/project/integrations/slack.md) for information on how to set this up. + +### Notify developers of alerts + +GitLab can react to the alerts triggered from your applications and services +by creating issues and alerting developers through email. By default, GitLab +sends these emails to [owners and maintainers](../../user/permissions.md) of the project. +These emails contain details of the alert, and a link for more information. + +To send separate email notifications to users with +[Developer permissions](../../user/permissions.md), see +[Configure incidents](incidents.md#configure-incidents). + +## Configure Prometheus alerts + +You can set up Prometheus alerts in: + +- [GitLab-managed Prometheus](../metrics/alerts.md) installations. +- [Self-managed Prometheus](../metrics/alerts.md#external-prometheus-instances) installations. + +Prometheus alerts are created by the special Alert Bot user. You can't remove this +user, but it does not count toward your license limit. + +## Configure external generic alerts + +GitLab can accept alerts from any source through a generic webhook receiver. When +[configuring the generic alerts integration](../../user/project/integrations/generic_alerts.md), +GitLab creates a unique endpoint which receives a JSON-formatted, customizable payload. + +## Integrate incidents with Slack + +Slack slash commands allow you to control GitLab and view GitLab content without leaving Slack. + +Learn how to [set up Slack slash commands](../../user/project/integrations/slack_slash_commands.md) +and how to [use the available slash commands](../../user/project/slash_commands.md). + +## Integrate issues with Zoom + +GitLab enables you to [associate a Zoom meeting with an issue](../../user/project/issues/associate_zoom_meeting.md) +for synchronous communication during incident management. After starting a Zoom +call for an incident, you can associate the conference call with an issue. Your +team members can join the Zoom call without requesting a link. diff --git a/doc/operations/incident_management/status_page.md b/doc/operations/incident_management/status_page.md new file mode 100644 index 00000000000..e376607d86f --- /dev/null +++ b/doc/operations/incident_management/status_page.md @@ -0,0 +1,179 @@ +--- +stage: Monitor +group: Health +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# GitLab Status Page **(ULTIMATE)** + +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/2479) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.10. + +With a GitLab Status Page, you can create and deploy a static website to communicate +efficiently to users during an incident. The Status Page landing page displays an +overview of recent incidents: + + + +Clicking an incident displays a detail page with more information about a particular incident: + + + +- Status on the incident, including when the incident was last updated. +- The incident title, including any emojis. +- The description of the incident, including emojis. +- Any file attachments provided in the incident description, or comments with a + valid image extension. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/205166) in GitLab 13.1. +- A chronological ordered list of updates to the incident. + +## Set up a GitLab Status Page + +To configure a GitLab Status Page you must: + +1. [Configure GitLab](#configure-gitlab-with-cloud-provider-information) with your + cloud provider information. +1. [Configure your AWS account](#configure-your-aws-account). +1. [Create a Status Page project](#create-a-status-page-project) on GitLab. +1. [Sync incidents to the Status Page](#sync-incidents-to-the-status-page). + +### Configure GitLab with cloud provider information + +To provide GitLab with the AWS account information needed to push content to your Status Page: + +NOTE: **Note:** +Only AWS S3 is supported as a deploy target. + +1. Sign into GitLab as a user with Maintainer or greater [permissions](../../user/permissions.md). +1. Navigate to **{settings}** **Settings > Operations**. Next to **Status Page**, + click **Expand**. +1. Click **Active** to enable the Status Page feature. +1. In **Status Page URL**, provide the URL to your external status page. +1. Provide the **S3 Bucket name**. For more information, see + [Bucket configuration documentation](https://docs.aws.amazon.com/AmazonS3/latest/dev/HostingWebsiteOnS3Setup.html). +1. Provide the **AWS region** for your bucket. For more information, see the + [AWS documentation](https://github.com/aws/aws-sdk-ruby#configuration). +1. Provide your **AWS access key ID** and **AWS Secret access key**. +1. Click **Save changes**. + +### Configure your AWS account + +1. Within your AWS account, create two new IAM policies, using the following files + as examples: + - [Create bucket](https://gitlab.com/gitlab-org/status-page/-/blob/master/deploy/etc/s3_create_policy.json). + - [Update bucket contents](https://gitlab.com/gitlab-org/status-page/-/blob/master/deploy/etc/s3_update_bucket_policy.json) (Remember replace `S3_BUCKET_NAME` with your bucket name). +1. Create a new AWS access key with the permissions policies created in the first step. + +### Create a status page project + +After configuring your AWS account, you must add the Status Page project and configure +the necessary CI/CD variables to deploy the Status Page to AWS S3: + +1. Fork the [Status Page](https://gitlab.com/gitlab-org/status-page) project. + You can do this through [Repository Mirroring](https://gitlab.com/gitlab-org/status-page#repository-mirroring), + which ensures you get the up-to-date Status Page features. +1. Navigate to **{settings}** **Settings > CI/CD**. +1. Scroll to **Variables**, and click **Expand**. +1. Add the following variables from your Amazon Console: + - `S3_BUCKET_NAME` - The name of the Amazon S3 bucket. + + NOTE: **Note:** + If no bucket with the provided name exists, the first pipeline run creates + one and configures it for + [static website hosting](https://docs.aws.amazon.com/AmazonS3/latest/dev/HostingWebsiteOnS3Setup.html). + + - `AWS_DEFAULT_REGION` - The AWS region. + - `AWS_ACCESS_KEY_ID` - The AWS access key ID. + - `AWS_SECRET_ACCESS_KEY` - The AWS secret. +1. Navigate to **CI / CD > Pipelines > Run Pipeline**, and run the pipeline to + deploy the Status Page to S3. + +CAUTION: **Caution:** +Consider limiting who can access issues in this project, as any user who can view +the issue can potentially [publish comments to your GitLab Status Page](#publish-comments-on-incidents). + +### Sync incidents to the Status Page + +After creating the CI/CD variables, configure the Project you want to use for +Incident issues: + +1. To view the [Operations Settings](../../user/project/settings/#operations-settings) + page, navigate to **{settings}** **Settings > Operations > Status Page**. +1. Fill in your cloud provider's credentials and make sure the **Active** checkbox is checked. +1. Click **Save changes**. + +## How to use your GitLab Status Page + +After configuring your GitLab instance, relevant updates trigger a background job +that pushes JSON-formatted data about the incident to your external cloud provider. +Your status page website periodically fetches this JSON-formatted data. It formats +and displays it to users, providing information about ongoing incidents without +extra effort from your team: + +```mermaid +graph TB + subgraph GitLab Instance + issues(issue updates) -- trigger --> middleware(Background job: JSON generation) + end + subgraph Cloud Provider + middleware --saves data --> c1(Cloud Bucket stores JSON file) + end + subgraph Status Page + d(Static Site on CDN) -- fetches data --> c1 + end +``` + +### Publish an incident + +To publish an incident: + +1. Create an issue in the project you enabled the GitLab Status Page settings in. +1. A [project or group owner](../../user/permissions.md) must use the + `/publish` [quick action](../../user/project/quick_actions.md) to publish the + issue to the GitLab Status Page. + + NOTE: **Note:** + Confidential issues can't be published. + +A background worker publishes the issue onto the Status Page using the credentials +you provided during setup. As part of publication, GitLab will: + +- Anonymize user and group mentions with `Incident Responder`. +- Remove titles of non-public [GitLab references](../../user/markdown.md#special-gitlab-references). +- Publish any files attached to incident issue descriptions, up to 5000 per issue. + ([Introduced in GitLab 13.1](https://gitlab.com/gitlab-org/gitlab/-/issues/205166).) + +After publication, you can access the incident's details page by clicking the +**Published on status page** button displayed under the Incident's title. + + + +### Update an incident + +To publish an update to the Incident, update the incident issue's description. + +CAUTION: **Caution:** +When referenced issues are changed (such as title or confidentiality) the incident +they were referenced in is not updated. + +### Publish comments on incidents + +To publish comments to the Status Page Incident: + +- Create a comment on the incident issue. +- When you're ready to publish the comment, mark the comment for publication by + adding a microphone [award emoji](../../user/award_emojis.md) + reaction (`:microphone:` 🎤) to the comment. +- Any files attached to the comment (up to 5000 per issue) are also published. + ([Introduced in GitLab 13.1](https://gitlab.com/gitlab-org/gitlab/-/issues/205166).) + +CAUTION: **Caution:** +Anyone with access to view the Issue can add an emoji award to a comment, so +consider limiting access to issues to team members only. + +### Update the incident status + +To change the incident status from `open` to `closed`, close the incident issue +within GitLab. Closing the issue triggers a background worker to update the +GitLab Status Page website. + +If you make a published issue confidential, GitLab unpublishes it from your +GitLab Status Page website. diff --git a/doc/operations/index.md b/doc/operations/index.md index 314a1b231ba..bcc3f89f47f 100644 --- a/doc/operations/index.md +++ b/doc/operations/index.md @@ -4,17 +4,96 @@ group: APM info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# Project operations +# Project operations **(CORE)** GitLab provides a variety of tools to help operate and maintain your applications: +## Measure reliability and stability with metrics + +Metrics help you understand the health and performance of your infrastructure, +applications, and systems by providing insights into your application's reliability, +stability, and performance. GitLab provides a dashboard out-of-the-box, which you +can extend with custom metrics, and augment with additional custom dashboards. You +can track the metrics that matter most to your team, generate automated alerts when +performance degrades, and manage those alerts - all within GitLab. + - Collect [Prometheus metrics](../user/project/integrations/prometheus_library/index.md). +- Monitor application status with the [out-of-the-box metrics dashboard](metrics/index.md), + which you can [customize](metrics/dashboards/settings.md). +- Create [custom performance alerts](metrics/alerts.md). +- Create [custom metrics](metrics/index.md#adding-custom-metrics) and + [custom dashboards](metrics/dashboards/index.md). + +## Manage alerts and incidents + +GitLab helps reduce alert fatigue for IT responders by providing tools to identify +issues across multiple systems and aggregate alerts in a centralized place. Your +team needs a single, central interface where they can easily investigate alerts +using metrics and logs, and promote the critical alerts to incidents. + +Are your alerts too noisy? Alerts configured on GitLab metrics can configured +and fine-tuned in GitLab immediately following a fire-fight. + +- [Manage alerts and incidents](../user/incident_management/index.md) in GitLab. +- [Configure alerts for metrics](metrics/alerts.md#set-up-alerts-for-prometheus-metrics-core) in GitLab. +- Create a [status page](incident_management/status_page.md) + to communicate efficiently to your users during an incident. + +## Track errors in your application + +GitLab integrates with [Sentry](https://sentry.io/welcome/) to aggregate errors +from your application and surface them in the GitLab UI with the sorting and filtering +features you need to help identify which errors are the most critical. Through the +entire triage process, your users can create GitLab issues to track critical errors +and the work required to fix them - all without leaving GitLab. + +- Discover and view errors generated by your applications with + [Error Tracking](error_tracking.md). + +## Trace application health and performance **(ULTIMATE)** + +Application tracing in GitLab is a way to measure an application's performance and +health while it's running. After configuring your application to enable tracing, you +gain in-depth insight into your application's layers. With application tracing, +you can measure the execution time of a user journey for troubleshooting or +optimization purposes. + +GitLab integrates with [Jaeger](https://www.jaegertracing.io/) - an open-source, +end-to-end distributed tracing system tool used for monitoring and troubleshooting +microservices-based distributed systems - and displays results within GitLab. + +- [Trace the performance and health](tracing.md) of a deployed application. **(ULTIMATE)** + +## Aggregate and store logs + +Developers need to troubleshoot application changes in development, and incident +responders need aggregated, real-time logs when troubleshooting problems with +production services. GitLab provides centralized, aggregated log storage for your +distributed application, enabling you to collect logs across multiple services and +infrastructure. + +- [View logs of pods or managed applications](../user/project/clusters/kubernetes_pod_logs.md) + in connected Kubernetes clusters. + +## Manage your infrastructure in code + +GitLab stores and executes your infrastructure as code, whether it's +defined in Ansible, Puppet or Chef. We also offer native integration with +[Terraform](https://www.terraform.io/), uniting your GitOps and +Infrastructure-as-Code (IaC) workflows with GitLab's authentication, authorization, +and user interface. By lowering the barrier to entry for adopting Terraform, you +can manage and provision infrastructure through machine-readable definition files, +rather than physical hardware configuration or interactive configuration tools. +Definitions are stored in version control, extending proven coding techniques to +your infrastructure, and blurring the line between what is an application and what is +an environment. + +- Learn how to [manage your infrastructure with GitLab and Terraform](../user/infrastructure/index.md). + +## More features + - Deploy to different [environments](../ci/environments/index.md). -- Manage your [Alerts](../user/project/operations/alert_management.md) and [Incidents](../user/incident_management/index.md). - Connect your project to a [Kubernetes cluster](../user/project/clusters/index.md). -- Manage your infrastructure with [Infrastructure as Code](../user/infrastructure/index.md) approaches. -- Discover and view errors generated by your applications with [Error Tracking](../user/project/operations/error_tracking.md). +- See how your application is used and analyze events with [Product Analytics](product_analytics.md). - Create, toggle, and remove [Feature Flags](feature_flags.md). **(PREMIUM)** -- [Trace](tracing.md) the performance and health of a deployed application. **(ULTIMATE)** -- Change the [settings of the Monitoring Dashboard](../user/project/operations/dashboard_settings.md). diff --git a/doc/operations/metrics/alerts.md b/doc/operations/metrics/alerts.md index 43debbd1b78..2ed8de9396a 100644 --- a/doc/operations/metrics/alerts.md +++ b/doc/operations/metrics/alerts.md @@ -4,7 +4,7 @@ group: APM info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# Set up alerts for Prometheus metrics +# Set up alerts for Prometheus metrics **(CORE)** After [configuring metrics for your CI/CD environment](index.md), you can set up alerting for Prometheus metrics depending on the location of your instances, and @@ -13,23 +13,36 @@ your team when environment performance falls outside of the boundaries you set. ## Managed Prometheus instances -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/6590) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.2 for [custom metrics](index.md#adding-custom-metrics), and 11.3 for [library metrics](../../user/project/integrations/prometheus_library/metrics.md). +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/6590) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.2 for [custom metrics](index.md#adding-custom-metrics), and GitLab 11.3 for [library metrics](../../user/project/integrations/prometheus_library/metrics.md). For managed Prometheus instances using auto configuration, you can [configure alerts for metrics](index.md#adding-custom-metrics) directly in the [metrics dashboard](index.md). To set an alert: -1. In your project, navigate to **{cloud-gear}** **Operations > Metrics**, +1. In your project, navigate to **Operations > Metrics**, 1. Identify the metric you want to create the alert for, and click the **ellipsis** **{ellipsis_v}** icon in the top right corner of the metric. 1. Choose **Alerts**. 1. Set threshold and operator. +1. (Optional) Add a Runbook URL. 1. Click **Add** to save and activate the alert. - + To remove the alert, click back on the alert icon for the desired metric, and click **Delete**. +### Link runbooks to alerts + +> - Runbook URLs [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/39315) in GitLab 13.3. + +When creating alerts from the metrics dashboard for [managed Prometheus instances](#managed-prometheus-instances), +you can also link a runbook. When the alert triggers, the +[chart context menu](dashboards/index.md#chart-context-menu) on the metrics chart +links to the runbook, making it easy for you to locate and access the correct runbook +as soon as the alert fires: + + + ## External Prometheus instances >- [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9258) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.8. @@ -37,11 +50,11 @@ To remove the alert, click back on the alert icon for the desired metric, and cl For manually configured Prometheus servers, GitLab provides a notify endpoint for use with Prometheus webhooks. If you have manual configuration enabled, an -**Alerts** section is added to **{settings}** **Settings > Integrations > Prometheus**. +**Alerts** section is added to **Settings > Integrations > Prometheus**. This section contains the **URL** and **Authorization Key** you will need. The **Reset Key** button will invalidate the key and generate a new one. - + To send GitLab alert notifications, copy the **URL** and **Authorization Key** into the [`webhook_configs`](https://prometheus.io/docs/alerting/latest/configuration/#webhook_config) @@ -75,22 +88,22 @@ Prometheus server to use the Alerts can be used to trigger actions, like opening an issue automatically (disabled by default since `13.1`). To configure the actions: -1. Navigate to your project's **{settings}** **Settings > Operations > Incidents**. +1. Navigate to your project's **Settings > Operations > Incidents**. 1. Enable the option to create issues. 1. Choose the [issue template](../../user/project/description_templates.md) to create the issue from. 1. Optionally, select whether to send an email notification to the developers of the project. 1. Click **Save changes**. After enabling, GitLab automatically opens an issue when an alert is triggered containing -values extracted from [alert's payload](https://prometheus.io/docs/alerting/latest/configuration/#webhook_config): +values extracted from the [`alerts` field in webhook payload](https://prometheus.io/docs/alerting/latest/configuration/#webhook_config): - Issue author: `GitLab Alert Bot` -- Issue title: Extract from `annotations/title`, `annotations/summary` or `labels/alertname` -- Alert `Summary`: A list of properties - - `starts_at`: Alert start time via `startsAt` - - `full_query`: Alert query extracted from `generatorURL` +- Issue title: Extracted from the alert payload fields `annotations/title`, `annotations/summary`, or `labels/alertname`. +- Alert `Summary`: A list of properties from the alert's payload. + - `starts_at`: Alert start time from the payload's `startsAt` field + - `full_query`: Alert query extracted from the payload's `generatorURL` field - Optional list of attached annotations extracted from `annotations/*` -- Alert [GFM](../../user/markdown.md): GitLab Flavored Markdown from `annotations/gitlab_incident_markdown` +- Alert [GFM](../../user/markdown.md): GitLab Flavored Markdown from the payload's `annotations/gitlab_incident_markdown` field. When GitLab receives a **Recovery Alert**, it closes the associated issue. This action is recorded as a system message on the issue indicating that it diff --git a/doc/operations/metrics/dashboards/default.md b/doc/operations/metrics/dashboards/default.md new file mode 100644 index 00000000000..f086d7737bd --- /dev/null +++ b/doc/operations/metrics/dashboards/default.md @@ -0,0 +1,38 @@ +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# GitLab-defined metrics dashboards **(CORE)** + +GitLab provides some dashboards out-of-the-box for any project with +[Prometheus available](../../../user/project/integrations/prometheus.md). You can +[duplicate these GitLab-defined dashboards](index.md#duplicate-a-gitlab-defined-dashboard): + +- [Overview dashboard](#overview-dashboard). +- [Kubernetes pod health dashboard](#kubernetes-pod-health-dashboard). + +To learn about the components of a dashboard, read +[Metrics dashboard for your CI/CD environment](../index.md). + +## Overview dashboard + +This dashboard is the default metrics dashboard. It displays a large number of +metrics about the [deployed application](../index.md#configure-prometheus-to-gather-metrics). + + + +## Kubernetes pod health dashboard + +NOTE: **Note:** +This dashboard requires Kubernetes v1.14 or higher, due to the +[change in metric labels](https://github.com/kubernetes/kubernetes/pull/69099) +in Kubernetes 1.14. + +This dashboard displays CPU, memory, network and disk metrics for the pods in your +[connected K8s cluster](../../../user/project/clusters/index.md). It provides a +[variable selector](templating_variables.md#metric_label_values-variable-type) +at the top of the dashboard to select which pod's metrics to display. + + diff --git a/doc/operations/metrics/dashboards/develop.md b/doc/operations/metrics/dashboards/develop.md new file mode 100644 index 00000000000..b621f5fd727 --- /dev/null +++ b/doc/operations/metrics/dashboards/develop.md @@ -0,0 +1,33 @@ +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Developing templates for custom dashboards **(CORE)** + +GitLab provides a template to make it easier for you to create templates for +[custom dashboards](index.md). Templates provide helpful guidance and +commented-out examples you can use. + +## Apply a dashboard template + +Navigate to the browser-based editor of your choice: + +- In the **Repository view**: + + 1. Navigate to **{doc-text}** **Repository > Files**. + 1. Click **{plus}** **Add to tree** and select **New file**, + then click **Select a template type** to see a list of available templates: +  + +- In the **[Web IDE](../../../user/project/web_ide/index.md)**: + + 1. Click **Web IDE** when viewing your repository. + 1. Click **{doc-new}** **New file**, then click **Choose a template** to see a list of available templates: +  + +## Custom dashboard templates **(PREMIUM ONLY)** + +To enable and use a custom dashboard templates on your GitLab instance, read the +[guide for creating custom templates](../../../user/admin_area/settings/instance_template_repository.md). diff --git a/doc/operations/metrics/dashboards/img/actions_menu_create_add_panel_v13_3.png b/doc/operations/metrics/dashboards/img/actions_menu_create_add_panel_v13_3.png Binary files differnew file mode 100644 index 00000000000..e03fbef3b35 --- /dev/null +++ b/doc/operations/metrics/dashboards/img/actions_menu_create_add_panel_v13_3.png diff --git a/doc/operations/metrics/dashboards/img/actions_menu_create_new_dashboard_v13_3.png b/doc/operations/metrics/dashboards/img/actions_menu_create_new_dashboard_v13_3.png Binary files differnew file mode 100644 index 00000000000..ba4780b730b --- /dev/null +++ b/doc/operations/metrics/dashboards/img/actions_menu_create_new_dashboard_v13_3.png diff --git a/doc/user/project/operations/img/dashboard_external_link_v13_1.png b/doc/operations/metrics/dashboards/img/dashboard_external_link_v13_1.png Binary files differindex 3e8d792c53e..3e8d792c53e 100644 --- a/doc/user/project/operations/img/dashboard_external_link_v13_1.png +++ b/doc/operations/metrics/dashboards/img/dashboard_external_link_v13_1.png diff --git a/doc/user/project/operations/img/dashboard_local_timezone_v13_1.png b/doc/operations/metrics/dashboards/img/dashboard_local_timezone_v13_1.png Binary files differindex 8d45607a940..8d45607a940 100644 --- a/doc/user/project/operations/img/dashboard_local_timezone_v13_1.png +++ b/doc/operations/metrics/dashboards/img/dashboard_local_timezone_v13_1.png diff --git a/doc/user/project/operations/img/external_dashboard_link.png b/doc/operations/metrics/dashboards/img/external_dashboard_link.png Binary files differindex 82c5e05e467..82c5e05e467 100644 --- a/doc/user/project/operations/img/external_dashboard_link.png +++ b/doc/operations/metrics/dashboards/img/external_dashboard_link.png diff --git a/doc/user/project/integrations/img/heatmap_chart_too_much_data_v_13_2.png b/doc/operations/metrics/dashboards/img/heatmap_chart_too_much_data_v_13_2.png Binary files differindex c3a391b06c7..c3a391b06c7 100644 --- a/doc/user/project/integrations/img/heatmap_chart_too_much_data_v_13_2.png +++ b/doc/operations/metrics/dashboards/img/heatmap_chart_too_much_data_v_13_2.png diff --git a/doc/user/project/integrations/img/heatmap_panel_type.png b/doc/operations/metrics/dashboards/img/heatmap_panel_type.png Binary files differindex a2b3911ec68..a2b3911ec68 100644 --- a/doc/user/project/integrations/img/heatmap_panel_type.png +++ b/doc/operations/metrics/dashboards/img/heatmap_panel_type.png diff --git a/doc/operations/metrics/dashboards/img/k8s_pod_health_dashboard_v13_3.png b/doc/operations/metrics/dashboards/img/k8s_pod_health_dashboard_v13_3.png Binary files differnew file mode 100644 index 00000000000..dc0bc951500 --- /dev/null +++ b/doc/operations/metrics/dashboards/img/k8s_pod_health_dashboard_v13_3.png diff --git a/doc/user/project/integrations/img/metrics_dashboard_annotations_ui_v13.0.png b/doc/operations/metrics/dashboards/img/metrics_dashboard_annotations_ui_v13.0.png Binary files differindex a042fbbcf4e..a042fbbcf4e 100644 --- a/doc/user/project/integrations/img/metrics_dashboard_annotations_ui_v13.0.png +++ b/doc/operations/metrics/dashboards/img/metrics_dashboard_annotations_ui_v13.0.png diff --git a/doc/operations/metrics/dashboards/img/metrics_dashboard_panel_preview_v13_3.png b/doc/operations/metrics/dashboards/img/metrics_dashboard_panel_preview_v13_3.png Binary files differnew file mode 100644 index 00000000000..4f6d3b3dfa4 --- /dev/null +++ b/doc/operations/metrics/dashboards/img/metrics_dashboard_panel_preview_v13_3.png diff --git a/doc/operations/metrics/dashboards/img/metrics_dashboard_template_selection_v13_3.png b/doc/operations/metrics/dashboards/img/metrics_dashboard_template_selection_v13_3.png Binary files differnew file mode 100644 index 00000000000..bd8401a1747 --- /dev/null +++ b/doc/operations/metrics/dashboards/img/metrics_dashboard_template_selection_v13_3.png diff --git a/doc/operations/metrics/dashboards/img/metrics_dashboard_template_selection_web_ide_v13_3.png b/doc/operations/metrics/dashboards/img/metrics_dashboard_template_selection_web_ide_v13_3.png Binary files differnew file mode 100644 index 00000000000..650f66e9a30 --- /dev/null +++ b/doc/operations/metrics/dashboards/img/metrics_dashboard_template_selection_web_ide_v13_3.png diff --git a/doc/operations/metrics/dashboards/img/metrics_settings_button_v13_3.png b/doc/operations/metrics/dashboards/img/metrics_settings_button_v13_3.png Binary files differnew file mode 100644 index 00000000000..9c0eac12a3f --- /dev/null +++ b/doc/operations/metrics/dashboards/img/metrics_settings_button_v13_3.png diff --git a/doc/operations/metrics/dashboards/img/panel_context_menu_v13_3.png b/doc/operations/metrics/dashboards/img/panel_context_menu_v13_3.png Binary files differnew file mode 100644 index 00000000000..1259917608b --- /dev/null +++ b/doc/operations/metrics/dashboards/img/panel_context_menu_v13_3.png diff --git a/doc/user/project/integrations/img/prometheus_dashboard_anomaly_panel_type.png b/doc/operations/metrics/dashboards/img/prometheus_dashboard_anomaly_panel_type.png Binary files differindex 5cba6fa9038..5cba6fa9038 100644 --- a/doc/user/project/integrations/img/prometheus_dashboard_anomaly_panel_type.png +++ b/doc/operations/metrics/dashboards/img/prometheus_dashboard_anomaly_panel_type.png diff --git a/doc/user/project/integrations/img/prometheus_dashboard_area_panel_type_v12_8.png b/doc/operations/metrics/dashboards/img/prometheus_dashboard_area_panel_type_v12_8.png Binary files differindex 8c5663fef12..8c5663fef12 100644 --- a/doc/user/project/integrations/img/prometheus_dashboard_area_panel_type_v12_8.png +++ b/doc/operations/metrics/dashboards/img/prometheus_dashboard_area_panel_type_v12_8.png diff --git a/doc/user/project/integrations/img/prometheus_dashboard_bar_chart_panel_type_v12.10.png b/doc/operations/metrics/dashboards/img/prometheus_dashboard_bar_chart_panel_type_v12.10.png Binary files differindex 593e31477f4..593e31477f4 100644 --- a/doc/user/project/integrations/img/prometheus_dashboard_bar_chart_panel_type_v12.10.png +++ b/doc/operations/metrics/dashboards/img/prometheus_dashboard_bar_chart_panel_type_v12.10.png diff --git a/doc/user/project/integrations/img/prometheus_dashboard_column_panel_type.png b/doc/operations/metrics/dashboards/img/prometheus_dashboard_column_panel_type.png Binary files differindex 985f2b04ef3..985f2b04ef3 100644 --- a/doc/user/project/integrations/img/prometheus_dashboard_column_panel_type.png +++ b/doc/operations/metrics/dashboards/img/prometheus_dashboard_column_panel_type.png diff --git a/doc/operations/metrics/dashboards/img/prometheus_dashboard_gauge_panel_type_v13_3.png b/doc/operations/metrics/dashboards/img/prometheus_dashboard_gauge_panel_type_v13_3.png Binary files differnew file mode 100644 index 00000000000..547c565c6f9 --- /dev/null +++ b/doc/operations/metrics/dashboards/img/prometheus_dashboard_gauge_panel_type_v13_3.png diff --git a/doc/user/project/integrations/img/prometheus_dashboard_label_variable_shorthand.png b/doc/operations/metrics/dashboards/img/prometheus_dashboard_label_variable_shorthand.png Binary files differindex 15111a97464..15111a97464 100644 --- a/doc/user/project/integrations/img/prometheus_dashboard_label_variable_shorthand.png +++ b/doc/operations/metrics/dashboards/img/prometheus_dashboard_label_variable_shorthand.png diff --git a/doc/user/project/integrations/img/prometheus_dashboard_label_variables.png b/doc/operations/metrics/dashboards/img/prometheus_dashboard_label_variables.png Binary files differindex 9b94d0c6afa..9b94d0c6afa 100644 --- a/doc/user/project/integrations/img/prometheus_dashboard_label_variables.png +++ b/doc/operations/metrics/dashboards/img/prometheus_dashboard_label_variables.png diff --git a/doc/user/project/integrations/img/prometheus_dashboard_repeated_label.png b/doc/operations/metrics/dashboards/img/prometheus_dashboard_repeated_label.png Binary files differindex d43a890f0fa..d43a890f0fa 100644 --- a/doc/user/project/integrations/img/prometheus_dashboard_repeated_label.png +++ b/doc/operations/metrics/dashboards/img/prometheus_dashboard_repeated_label.png diff --git a/doc/user/project/integrations/img/prometheus_dashboard_single_stat_panel_type.png b/doc/operations/metrics/dashboards/img/prometheus_dashboard_single_stat_panel_type.png Binary files differindex 2d7dfb27b49..2d7dfb27b49 100644 --- a/doc/user/project/integrations/img/prometheus_dashboard_single_stat_panel_type.png +++ b/doc/operations/metrics/dashboards/img/prometheus_dashboard_single_stat_panel_type.png diff --git a/doc/user/project/integrations/img/prometheus_dashboard_stacked_column_panel_type_v12_8.png b/doc/operations/metrics/dashboards/img/prometheus_dashboard_stacked_column_panel_type_v12_8.png Binary files differindex ba67509bcf3..ba67509bcf3 100644 --- a/doc/user/project/integrations/img/prometheus_dashboard_stacked_column_panel_type_v12_8.png +++ b/doc/operations/metrics/dashboards/img/prometheus_dashboard_stacked_column_panel_type_v12_8.png diff --git a/doc/user/project/integrations/img/prometheus_dashboard_yaml_validation_v13_1.png b/doc/operations/metrics/dashboards/img/prometheus_dashboard_yaml_validation_v13_1.png Binary files differindex 08d7d6603d2..08d7d6603d2 100644 --- a/doc/user/project/integrations/img/prometheus_dashboard_yaml_validation_v13_1.png +++ b/doc/operations/metrics/dashboards/img/prometheus_dashboard_yaml_validation_v13_1.png diff --git a/doc/user/project/integrations/img/related_links_v13_1.png b/doc/operations/metrics/dashboards/img/related_links_v13_1.png Binary files differindex 4dc141f0e7f..4dc141f0e7f 100644 --- a/doc/user/project/integrations/img/related_links_v13_1.png +++ b/doc/operations/metrics/dashboards/img/related_links_v13_1.png diff --git a/doc/operations/metrics/dashboards/index.md b/doc/operations/metrics/dashboards/index.md index a46abdee2dc..ffcb7dc92c6 100644 --- a/doc/operations/metrics/dashboards/index.md +++ b/doc/operations/metrics/dashboards/index.md @@ -4,61 +4,154 @@ group: APM info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# Using the Metrics Dashboard +# Custom dashboards **(CORE)** -## Manage the metrics dashboard settings +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/59974) in GitLab 12.1. -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/223204) in GitLab 13.2. +By default, all projects include a [GitLab-defined Prometheus dashboard](default.md), which +includes a few key metrics, but you can also define your own custom dashboards. -To manage the settings for your metrics dashboard: +You may create a [new dashboard from scratch](#add-a-new-dashboard-to-your-project) +or [duplicate a GitLab-defined Prometheus dashboard](#duplicate-a-gitlab-defined-dashboard). + +NOTE: **Note:** +The metrics as defined below do not support alerts, unlike +[custom metrics](../index.md#adding-custom-metrics). + +## Add a new dashboard to your project -1. Sign in as a user with project Maintainer or Admin +> UI option [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/228856) in GitLab 13.3. + +You can configure a custom dashboard by adding a new YAML file into your project's +`.gitlab/dashboards/` directory. For the dashboard to display on your project's +**Operations > Metrics** page, the files must have a `.yml` +extension and be present in your project's **default** branch. + +To create a new dashboard from the GitLab user interface: + +1. Sign in to GitLab as a user with Maintainer or Owner [permissions](../../../user/permissions.md#project-members-permissions). -1. Navigate to your dashboard at **{cloud-gear}** **Operations > Metrics**. -1. In the top-right corner of your dashboard, click **{settings}** **Metrics Settings**: +1. Navigate to your dashboard at **Operations > Metrics**. +1. In the top-right corner of your dashboard, click the **{ellipsis_v}** **More actions** menu, + and select **Create new**: +  +1. In the modal window, click **Open Repository**, then follow the instructions + for creating a new dashboard from the command line. -  +To create a new dashboard from the command line: -## Chart Context Menu +1. Create `.gitlab/dashboards/prom_alerts.yml` under your repository's root + directory. Each YAML file should define the layout of the dashboard and the + Prometheus queries used to populate data. This example dashboard displays a + single area chart: -From each of the panels in the dashboard, you can access the context menu by clicking the **{ellipsis_v}** **More actions** dropdown box above the upper right corner of the panel to take actions related to the chart's data. + ```yaml + dashboard: 'Dashboard Title' + panel_groups: + - group: 'Group Title' + panels: + - type: area-chart + title: "Chart Title" + y_label: "Y-Axis" + y_axis: + format: number + precision: 0 + metrics: + - id: my_metric_id + query_range: 'http_requests_total' + label: "Instance: {{instance}}, method: {{method}}" + unit: "count" + ``` - +1. Save the file, commit, and push to your repository. The file must be present in your **default** branch. +1. Navigate to your project's **Operations > Metrics** and choose the custom + dashboard from the dropdown. -The options are: +Your custom dashboard is available at `https://example.com/project/-/metrics/custom_dashboard_name.yml`. -- [Expand panel](#expand-panel) -- [View logs](#view-logs-ultimate) -- [Download CSV](#downloading-data-as-csv) -- [Copy link to chart](../embed.md#embedding-gitlab-managed-kubernetes-metrics) -- [Alerts](../alerts.md) +NOTE: **Note:** +Configuration files nested under subdirectories of `.gitlab/dashboards` are not +supported and won't be available in the UI. -### View and edit the source file of a custom dashboard +## Add a new metrics panel to a dashboard -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/34779) in GitLab 12.5. +> UI option [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/228761) in GitLab 13.3. -When viewing a custom dashboard of a project, you can view the original -`.yml` file by clicking on the **Edit dashboard** button. +The metrics dashboard supports various [multiple panel types](../../../operations/metrics/dashboards/panel_types.md). +You can quickly test how a panel configuration would display in your metrics dashboard +with the **Add Panel** page: -### Expand panel +1. Sign in to GitLab as a user with Maintainer or Owner + [permissions](../../../user/permissions.md#project-members-permissions). +1. Click **Add panel** in the **{ellipsis_v}** **More actions** menu. + + NOTE: **Note:** + You can add panel only to custom dashboards. -> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3100) in GitLab 13.0. +  +1. In the **Define and preview panel** section, paste in the YAML you want to + preview in the **Panel YAML** field. +1. Click **Preview panel**, and GitLab displays a preview of the chart below the + `Define and preview panel` section: +  + +## Duplicate a GitLab-defined dashboard + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/37238) in GitLab 12.7. +> - From [GitLab 12.8 onwards](https://gitlab.com/gitlab-org/gitlab/-/issues/39505), custom metrics are also duplicated when you duplicate a dashboard. + +You can save a complete copy of a GitLab defined dashboard along with all custom metrics added to it. +The resulting `.yml` file can be customized and adapted to your project. +You can decide to save the dashboard `.yml` file in the project's **default** branch or in a +new branch. + +1. Click **Duplicate current dashboard** in the **{ellipsis_v}** **More actions** menu. + + NOTE: **Note:** + You can duplicate only GitLab-defined dashboards. + +1. Enter the filename and other information, such as the new commit's message, and click **Duplicate**. +1. Select a branch to add your dashboard to: + - *If you select your **default** branch,* the new dashboard becomes immediately available. + - *If you select another branch,* this branch should be merged to your **default** branch first. + +Your custom dashboard is available at `https://example.com/project/-/metrics/custom_dashboard_name.yml`. + +## Manage the metrics dashboard settings + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/223204) in GitLab 13.2. + +To manage the settings for your metrics dashboard: + +1. Sign in as a user with project Maintainer or Administrator + [permissions](../../../user/permissions.md#project-members-permissions). +1. Navigate to your dashboard at **Operations > Metrics**. +1. In the top-right corner of your dashboard, click **Metrics Settings**: -To view a larger version of a visualization, expand the panel by clicking the -**{ellipsis_v}** **More actions** icon and selecting **Expand panel**. +  -To return to the metrics dashboard, click the **Back** button in your -browser, or pressing the <kbd>Esc</kbd> key. +## Chart Context Menu -### View Logs **(ULTIMATE)** +You can take action related to a chart's data by clicking the +**{ellipsis_v}** **More actions** dropdown box above the upper right corner of +any chart on a dashboard: -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/122013) in GitLab 12.8. + -If you have [Logs](../../../user/project/clusters/kubernetes_pod_logs.md) enabled, -you can navigate from the charts in the dashboard to view Logs by -clicking on the context menu in the upper-right corner. +The options are: -If you use the **Timeline zoom** function at the bottom of the chart, logs will narrow down to the time range you selected. +- **Expand panel** - Displays a larger version of a visualization. To return to + the dashboard, click the **Back** button in your browser, or press the <kbd>Esc</kbd> key. + ([Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3100) in GitLab 13.0.) +- **View logs** **(ULTIMATE)** - Displays [Logs](../../../user/project/clusters/kubernetes_pod_logs.md), + if they are enabled. If used in conjunction with the [timeline zoom](#timeline-zoom-and-url-sharing) + feature, logs narrow down to the selected time range. ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/122013) in GitLab 12.8.) +- **Download CSV** - Data from Prometheus charts on the metrics dashboard can be downloaded as CSV. +- [Copy link to chart](../embed.md#embedding-gitlab-managed-kubernetes-metrics) +- **Alerts** - Display any [alerts](../alerts.md) configured for this metric. +- **View Runbook** - Displays the runbook for an alert. For information about configuring + runbooks, read [Set up alerts for Prometheus metrics](../alerts.md). + ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/211844) in GitLab 13.3.) ### Timeline zoom and URL sharing @@ -69,10 +162,6 @@ on a date and time of your choice. When you click and drag the sliders to select a different beginning or end date of data to display, GitLab adds your selected start and end times to the URL, enabling you to share specific timeframes more easily. -### Downloading data as CSV - -Data from Prometheus charts on the metrics dashboard can be downloaded as CSV. - ## Dashboard Annotations > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/211330) in GitLab 12.10 (enabled by feature flag `metrics_dashboard_annotations`). @@ -88,9 +177,9 @@ its description. You can create annotations by making requests to the [Metrics dashboard annotations API](../../../api/metrics_dashboard_annotations.md) - + -### Retention policy +### Annotation retention policy > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/211433) in GitLab 13.01. @@ -124,7 +213,7 @@ The dashboard's time range is appended to the `url` as URL parameters. The following example shows two related links (`GitLab.com` and `GitLab Documentation`) added to a dashboard: - + ### Links Syntax @@ -139,99 +228,6 @@ links: type: grafana ``` -## Defining custom dashboards per project - -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/59974) in GitLab 12.1. - -By default, all projects include a GitLab-defined Prometheus dashboard, which -includes a few key metrics, but you can also define your own custom dashboards. - -You may create a new file from scratch or duplicate a GitLab-defined Prometheus -dashboard. - -NOTE: **Note:** -The metrics as defined below do not support alerts, unlike -[custom metrics](../index.md#adding-custom-metrics). - -### Adding a new dashboard to your project - -> UI option [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/223204) in GitLab 13.2. - -You can configure a custom dashboard by adding a new YAML file into your project's -`.gitlab/dashboards/` directory. In order for the dashboards to be displayed on -the project's **{cloud-gear}** **Operations > Metrics** page, the files must have a `.yml` -extension and should be present in the project's **default** branch. - -To create a new dashboard from the GitLab user interface: - -1. Sign in to GitLab as a user with Maintainer or Owner - [permissions](../../../user/permissions.md#project-members-permissions). -1. Navigate to your dashboard at **{cloud-gear}** **Operations > Metrics**. -1. In the top-right corner of your dashboard, click the **{file-addition-solid}** **Actions** menu, - and select **Create new**: -  -1. In the modal window, click **Open Repository**, then follow the instructions - for creating a new dashboard from the command line. - -To create a new dashboard from the command line: - -1. Create `.gitlab/dashboards/prom_alerts.yml` under your repository's root - directory. Each YAML file should define the layout of the dashboard and the - Prometheus queries used to populate data. This example dashboard displays a - single area chart: - - ```yaml - dashboard: 'Dashboard Title' - panel_groups: - - group: 'Group Title' - panels: - - type: area-chart - title: "Chart Title" - y_label: "Y-Axis" - y_axis: - format: number - precision: 0 - metrics: - - id: my_metric_id - query_range: 'http_requests_total' - label: "Instance: {{instance}}, method: {{method}}" - unit: "count" - ``` - -1. Save the file, commit, and push to your repository. The file must be present in your **default** branch. -1. Navigate to your project's **Operations > Metrics** and choose the custom - dashboard from the dropdown. - -NOTE: **Note:** -Configuration files nested under subdirectories of `.gitlab/dashboards` are not -supported and will not be available in the UI. - -### Navigating to a custom dashboard - -Custom dashboards are uniquely identified by their filenames. In order to quickly view the custom dashboard, -just use the dashboard filename in the URL this way: -`https://gitlab-instance.example.com/project/-/metrics/custom_dashboard_name.yml`. - -### Duplicating a GitLab-defined dashboard - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/37238) in GitLab 12.7. -> - From [GitLab 12.8 onwards](https://gitlab.com/gitlab-org/gitlab/-/issues/39505), custom metrics are also duplicated when you duplicate a dashboard. - -You can save a complete copy of a GitLab defined dashboard along with all custom metrics added to it. -Resulting `.yml` file can be customized and adapted to your project. -You can decide to save the dashboard `.yml` file in the project's **default** branch or in a -new branch. - -1. Click **Duplicate dashboard** in the dashboard dropdown or in the actions menu. - - NOTE: **Note:** - You can duplicate only GitLab-defined dashboards. - -1. Enter the file name and other information, such as the new commit's message, and click **Duplicate**. - -If you select your **default** branch, the new dashboard becomes immediately available. -If you select another branch, this branch should be merged to your **default** branch first. - ## Troubleshooting When troubleshooting issues with a managed Prometheus app, it is often useful to diff --git a/doc/operations/metrics/dashboards/panel_types.md b/doc/operations/metrics/dashboards/panel_types.md index 0397218fe0e..b2cbdcb88d9 100644 --- a/doc/operations/metrics/dashboards/panel_types.md +++ b/doc/operations/metrics/dashboards/panel_types.md @@ -4,7 +4,7 @@ group: APM info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# Panel types for dashboards +# Panel types for dashboards **(CORE)** The below panel types are supported in monitoring dashboards. @@ -37,7 +37,7 @@ Note the following properties: | type | string | no | Type of panel to be rendered. Optional for area panel types | | query_range | string | required | For area panel types, you must use a [range query](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries) | - + Starting in [version 12.8](https://gitlab.com/gitlab-org/gitlab/-/issues/202696), the y-axis values will automatically scale according to the data. Previously, it always started from 0. @@ -81,7 +81,7 @@ Note the following properties: | type | string | required | Must be `anomaly-chart` for anomaly panel types | | query_range | yes | required | For anomaly panel types, you must use a [range query](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries) in every metric. | - + ## Bar chart @@ -110,7 +110,7 @@ Note the following properties: | `type` | string | yes | Type of panel to be rendered. For bar chart types, set to `bar` | | `query_range` | yes | yes | For bar chart, you must use a [range query](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries) - + ## Column chart @@ -137,7 +137,7 @@ Note the following properties: | type | string | yes | Type of panel to be rendered. For column panel types, set to `column` | | query_range | yes | yes | For column panel types, you must use a [range query](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries) | - + ## Stacked column @@ -169,7 +169,7 @@ panel_groups: series_name: 'group 2' ``` - + | Property | Type | Required | Description | | ------ | ------ | ------ | ------ | @@ -202,7 +202,7 @@ Note the following properties: | field | string | no | Panels display the value of a metric. For a panel to display the value of a label instead, put the name of the label in this key. | | query | string | yes | For single stat panel types, you must use an [instant query](https://prometheus.io/docs/prometheus/latest/querying/api/#instant-queries) | - + ## Percentile based results @@ -227,6 +227,57 @@ panel_groups: For example, if you have a query value of `53.6`, adding `%` as the unit results in a single stat value of `53.6%`, but if the maximum expected value of the query is `120`, the value would be `44.6%`. Adding the `max_value` causes the correct percentage value to display. +## Gauge + +CAUTION: **Warning:** +This panel type is an _alpha_ feature, and is subject to change at any time +without prior notice! + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/207044) in GitLab 13.3. + +To add a gauge panel type to a dashboard, look at the following sample dashboard file: + +```yaml +dashboard: 'Dashboard Title' +panel_groups: + - group: 'Group Title' + panels: + - title: "Gauge" + type: "gauge" + min_value: 0 + max_value: 1000 + split: 5 + thresholds: + values: [60, 90] + mode: "percentage" + format: "kilobytes" + metrics: + - id: 10 + query: 'floor(max(prometheus_http_response_size_bytes_bucket)/1000)' + unit: 'kb' +``` + +Note the following properties: + +| Property | Type | Required | Description | +| ------ | ------ | ------ | ------ | +| type | string | yes | Type of panel to be rendered. For gauge panel types, set to `gauge`. | +| min_value | number | no, defaults to `0` | The minimum value of the gauge chart axis. If either of `min_value` or `max_value` are not set, they both get their default values. | +| max_value | number | no, defaults to `100` | The maximum value of the gauge chart axis. If either of `min_value` or `max_value` are not set, they both get their default values. | +| split | number | no, defaults to `10` | The amount of split segments on the gauge chart axis. | +| thresholds | object | no | Thresholds configuration for the gauge chart axis. | +| format | string | no, defaults to `engineering` | Unit format used. See the [full list of units](yaml_number_format.md). | +| query | string | yes | For gauge panel types, you must use an [instant query](https://prometheus.io/docs/prometheus/latest/querying/api/#instant-queries). | + +### Thresholds properties + +| Property | Type | Required | Description | +| ------ | ------ | ------ | ------ | +| values | array | no, defaults to 95% of the range between `min_value` and `max_value`| An array of gauge chart axis threshold values. | +| mode | string | no, defaults to `absolute` | The mode in which the thresholds are interpreted in relation to `min_value` and `max_value`. Can be either `percentage` or `absolute`. | + + + ## Heatmaps > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/30581) in GitLab 12.5. @@ -254,9 +305,9 @@ Note the following properties: | type | string | yes | Type of panel to be rendered. For heatmap panel types, set to `heatmap` | | query_range | yes | yes | For area panel types, you must use a [range query](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries) | - + CAUTION: **Warning:** When a query returns too many data points, the heatmap data bucket dimensions tend downwards to 0, making the chart's data invisible, as shown in the image below. To fix this problem, limit the amount of data returned by changing the time range filter on the metrics dashboard UI, or adding the **step** property to your dashboard's YAML file. - + diff --git a/doc/operations/metrics/dashboards/settings.md b/doc/operations/metrics/dashboards/settings.md new file mode 100644 index 00000000000..a4aef6b1674 --- /dev/null +++ b/doc/operations/metrics/dashboards/settings.md @@ -0,0 +1,52 @@ +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Dashboard settings + +You can configure your [Monitoring dashboard](../index.md) to +display the time zone of your choice, and the links of your choice. + +To configure these settings you must have Manage Project +Operations [permissions](../../../user/permissions.md). + +## Change the dashboard time zone + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/214370) in GitLab 13.1. + +By default, your monitoring dashboard displays dates and times in your local +time zone, but you can display dates and times in UTC format. To change the +time zone: + +1. Sign in as a user with Manage Project Operations [permissions](../../../user/permissions.md). +1. Navigate to **Settings > Operations**, and scroll to + **Metrics Dashboard**. +1. In the **Dashboard timezone** select box, select *User's local timezone* + or *UTC*: + +  +1. Click **Save changes**. + +## Link to an external dashboard + +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/57171) in GitLab 12.0. + +You can add a button on your monitoring dashboard that links directly to your +existing external dashboards: + +1. Sign in as a user with Manage Project Operations [permissions](../../../user/permissions.md). +1. Navigate to **Settings > Operations**, and scroll to + **Metrics Dashboard**. +1. In **External dashboard URL**, provide the URL to your external dashboard: + +  + +1. Click **Save changes**. + +GitLab displays a **View full dashboard** button in the top right corner of your +[monitoring dashboard](../../../ci/environments/index.md#monitoring-environments) +which opens the URL you provided: + + diff --git a/doc/operations/metrics/dashboards/templating_variables.md b/doc/operations/metrics/dashboards/templating_variables.md index a515742ea92..71025d41281 100644 --- a/doc/operations/metrics/dashboards/templating_variables.md +++ b/doc/operations/metrics/dashboards/templating_variables.md @@ -4,7 +4,7 @@ group: APM info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# Templating variables for metrics dashboards +# Templating variables for metrics dashboards **(CORE)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/214539) in GitLab 13.0. diff --git a/doc/operations/metrics/dashboards/variables.md b/doc/operations/metrics/dashboards/variables.md index 19b77a1ed87..8b0d7f37052 100644 --- a/doc/operations/metrics/dashboards/variables.md +++ b/doc/operations/metrics/dashboards/variables.md @@ -4,9 +4,9 @@ group: APM info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# Using Variables +# Using variables **(CORE)** -## Query Variables +## Query variables Variables can be specified using double curly braces, such as `"{{ci_environment_slug}}"` ([added](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/20793) in GitLab 12.7). @@ -41,7 +41,7 @@ For example, if the dashboard time range is set to 8 hours, the value of [Variables can be defined](../../../operations/metrics/dashboards/yaml.md#templating-templating-properties) in a custom dashboard YAML file. -## Query Variables from URL +## Query variables from URL > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/214500) in GitLab 13.0. diff --git a/doc/operations/metrics/dashboards/yaml.md b/doc/operations/metrics/dashboards/yaml.md index 6a4158798bc..f92ba4079e9 100644 --- a/doc/operations/metrics/dashboards/yaml.md +++ b/doc/operations/metrics/dashboards/yaml.md @@ -4,7 +4,7 @@ group: APM info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# Dashboard YAML properties +# Dashboard YAML properties **(CORE)** Dashboards have several components: @@ -43,24 +43,34 @@ Read the documentation on [links](index.md#add-related-links-to-custom-dashboard ## **Panel group (`panel_groups`) properties** +Dashboards display panel groups in the order they are listed in the dashboard YAML file. + +NOTE: **Note:** +In GitLab versions 13.3 and below, panel groups were ordered by a `priority` key, which +is no longer used. + | Property | Type | Required | Description | | ------ | ------ | ------ | ------ | | `group` | string | required | Heading for the panel group. | -| `priority` | number | optional, defaults to order in file | Order to appear on the dashboard. Higher number means higher priority, which will be higher on the page. Numbers do not need to be consecutive. | | `panels` | array | required | The panels which should be in the panel group. | Panels in a panel group are laid out in rows consisting of two panels per row. An exception to this rule are single panels on a row: these panels will take the full width of their containing row. ## **Panel (`panels`) properties** +Dashboards display panels in the order they are listed in the dashboard YAML file. + +NOTE: **Note:** +In GitLab versions 13.3 and below, panels were ordered by a `weight` key, which +is no longer used. + | Property | Type | Required | Description | | ------ | ------ | ------ | ------- | -| `type` | enum | no, defaults to `area-chart` | Specifies the panel type to use, for example `area-chart`, `line-chart` or `anomaly-chart`. [View documentation on all panel types.](panel_types.md) | +| `type` | string | no, defaults to `area-chart` | Specifies the panel type to use, for example `area-chart`, `line-chart` or `anomaly-chart`. Only types listed among [all panel types](panel_types.md) are allowed. | | `title` | string | yes | Heading for the panel. | | `y_label` | string | no, but highly encouraged | Y-Axis label for the panel. | | `y_axis` | string | no | Y-Axis configuration for the panel. | | `max_value` | number | no | Denominator value used for calculating [percentile based results](panel_types.md#percentile-based-results) | -| `weight` | number | no, defaults to order in file | Order to appear within the grouping. Lower number means higher priority, which will be higher on the page. Numbers do not need to be consecutive. | | `metrics` | array | yes | The metrics which should be displayed in the panel. Any number of metrics can be displayed when `type` is `area-chart` or `line-chart`, whereas only 3 can be displayed when `type` is `anomaly-chart`. | | `links` | array | no | Add links to display on the chart's [context menu](index.md#chart-context-menu). | @@ -79,8 +89,8 @@ Panels in a panel group are laid out in rows consisting of two panels per row. A | `id` | string | no | Used for associating dashboard metrics with database records. Must be unique across dashboard configuration files. Required for [alerting](../alerts.md) (support not yet enabled, see [relevant issue](https://gitlab.com/gitlab-org/gitlab/-/issues/27980)). | | `unit` | string | yes | Defines the unit of the query's return data. | | `label` | string | no, but highly encouraged | Defines the legend-label for the query. Should be unique within the panel's metrics. Can contain time series labels as interpolated variables. | -| `query` | string | yes if `query_range` is not defined | Defines the Prometheus query to be used to populate the chart/panel. If defined, the `query` endpoint of the [Prometheus API](https://prometheus.io/docs/prometheus/latest/querying/api/) will be utilized. | -| `query_range` | string | yes if `query` is not defined | Defines the Prometheus query to be used to populate the chart/panel. If defined, the `query_range` endpoint of the [Prometheus API](https://prometheus.io/docs/prometheus/latest/querying/api/) will be utilized. | +| `query` | string/number | yes if `query_range` is not defined | Defines the Prometheus query to be used to populate the chart/panel. If defined, the `query` endpoint of the [Prometheus API](https://prometheus.io/docs/prometheus/latest/querying/api/) will be utilized. | +| `query_range` | string/number | yes if `query` is not defined | Defines the Prometheus query to be used to populate the chart/panel. If defined, the `query_range` endpoint of the [Prometheus API](https://prometheus.io/docs/prometheus/latest/querying/api/) will be utilized. | | `step` | number | no, value is calculated if not defined | Defines query resolution step width in float number of seconds. Metrics on the same panel should use the same `step` value. | ## Dynamic labels @@ -99,7 +109,7 @@ metrics: This may render a legend like this: - + For labels to be more explicit, using variables that reflect time series labels is a good practice. The variables will be replaced by the values of the time series labels when the legend is rendered: @@ -113,7 +123,7 @@ metrics: The resulting rendered legend will look like this: - + There is also a shorthand value for dynamic dashboard labels that make use of only one time series label: @@ -127,7 +137,7 @@ metrics: This works by lowercasing the value of `label` and, if there are more words separated by spaces, replacing those spaces with an underscore (`_`). The transformed value is then checked against the labels of the time series returned by the Prometheus query. If a time series label is found that is equal to the transformed value, then the label value will be used and rendered in the legend like this: - + ## Dashboard YAML syntax validation @@ -142,19 +152,19 @@ To confirm your dashboard definition contains valid YAML syntax: Files with valid syntax display **Metrics Dashboard YAML definition is valid**, and files with invalid syntax display **Metrics Dashboard YAML definition is invalid**. - + When **Metrics Dashboard YAML definition is invalid** at least one of the following messages is displayed: -1. `dashboard: can't be blank` [learn more](../../../operations/metrics/dashboards/yaml.md#dashboard-top-level-properties) -1. `panel_groups: should be an array of panel_groups objects` [learn more](../../../operations/metrics/dashboards/yaml.md#dashboard-top-level-properties) -1. `group: can't be blank` [learn more](../../../operations/metrics/dashboards/yaml.md#panel-group-panel_groups-properties) -1. `panels: should be an array of panels objects` [learn more](../../../operations/metrics/dashboards/yaml.md#panel-group-panel_groups-properties) -1. `title: can't be blank` [learn more](../../../operations/metrics/dashboards/yaml.md#panel-panels-properties) -1. `metrics: should be an array of metrics objects` [learn more](../../../operations/metrics/dashboards/yaml.md#panel-panels-properties) -1. `query: can't be blank` [learn more](../../../operations/metrics/dashboards/yaml.md#metrics-metrics-properties) -1. `query_range: can't be blank` [learn more](../../../operations/metrics/dashboards/yaml.md#metrics-metrics-properties) -1. `unit: can't be blank` [learn more](../../../operations/metrics/dashboards/yaml.md#metrics-metrics-properties) +1. `dashboard: can't be blank` [learn more](#dashboard-top-level-properties) +1. `panel_groups: should be an array of panel_groups objects` [learn more](#dashboard-top-level-properties) +1. `group: can't be blank` [learn more](#panel-group-panel_groups-properties) +1. `panels: should be an array of panels objects` [learn more](#panel-group-panel_groups-properties) +1. `title: can't be blank` [learn more](#panel-panels-properties) +1. `metrics: should be an array of metrics objects` [learn more](#panel-panels-properties) +1. `query: can't be blank` [learn more](#metrics-metrics-properties) +1. `query_range: can't be blank` [learn more](#metrics-metrics-properties) +1. `unit: can't be blank` [learn more](#metrics-metrics-properties) 1. `YAML syntax: The parsed YAML is too big` This is displayed when the YAML file is larger than 1 MB. diff --git a/doc/operations/metrics/dashboards/yaml_number_format.md b/doc/operations/metrics/dashboards/yaml_number_format.md index ae0cd9fff64..1a8bd6f4257 100644 --- a/doc/operations/metrics/dashboards/yaml_number_format.md +++ b/doc/operations/metrics/dashboards/yaml_number_format.md @@ -4,7 +4,7 @@ group: APM info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# Unit formats reference +# Unit formats reference **(CORE)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/201999) in GitLab 12.9. diff --git a/doc/operations/metrics/embed.md b/doc/operations/metrics/embed.md index 5ee9b0859b9..62d60921c85 100644 --- a/doc/operations/metrics/embed.md +++ b/doc/operations/metrics/embed.md @@ -4,33 +4,37 @@ group: APM info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# Embedding metric charts within GitLab Flavored Markdown +# Embedding metric charts within GitLab-flavored Markdown **(CORE)** + +You can display metrics charts within +[GitLab Flavored Markdown](../../user/markdown.md#gitlab-flavored-markdown-gfm) +fields such as issue or merge request descriptions. The maximum number of embedded +charts allowed in a GitLab Flavored Markdown field is 100. +Embedding charts is useful when sharing an application incident or performance +metrics to others, and you want to have relevant information directly available. ## Embedding GitLab-managed Kubernetes metrics > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/29691) in GitLab 12.2. -It is possible to display metrics charts within [GitLab Flavored Markdown](../../user/markdown.md#gitlab-flavored-markdown-gfm) fields such as issue or merge request descriptions. The maximum number of embedded charts allowed in a GitLab Flavored Markdown field is 100. - -This can be useful if you are sharing an application incident or performance -metrics to others and want to have relevant information directly available. - NOTE: **Note:** Requires [Kubernetes](../../user/project/integrations/prometheus_library/kubernetes.md) metrics. -To display metric charts, include a link of the form `https://<root_url>/<project>/-/environments/<environment_id>/metrics`: +To display metric charts, include a link of the form +`https://<root_url>/<project>/-/environments/<environment_id>/metrics` in a field +that supports GitLab-flavored Markdown: - + GitLab unfurls the link as an embedded metrics panel: - + You can also embed a single chart. To get a link to a chart, click the **{ellipsis_v}** **More actions** menu in the upper right corner of the chart, and select **Copy link to chart**, as shown in this example: - + The following requirements must be met for the metric to unfurl: @@ -40,27 +44,33 @@ The following requirements must be met for the metric to unfurl: - The user must be allowed access to the monitoring dashboard for the environment ([Reporter or higher](../../user/permissions.md)). - The dashboard must have data within the last 8 hours. - If all of the above are true, then the metric will unfurl as seen below: + If all of the above are true, then the metric unfurls as seen below: - + Metric charts may also be hidden: - + -You can open the link directly into your browser for a [detailed view of the data](dashboards/index.md#expand-panel). +You can open the link directly into your browser for a +[detailed view of the data](dashboards/index.md#chart-context-menu). ## Embedding metrics in issue templates -It is also possible to embed either the default dashboard metrics or individual metrics in issue templates. For charts to render side-by-side, links to the entire metrics dashboard or individual metrics should be separated by either a comma or a space. +You can also embed either the overview dashboard metrics or individual metrics in +issue templates. For charts to render side-by-side, separate links to the entire metrics +dashboard or individual metrics by either a comma or a space. - + ## Embedding metrics based on alerts in incident issues -For [GitLab-managed alerting rules](alerts.md), the issue will include an embedded chart for the query corresponding to the alert. The chart displays an hour of data surrounding the starting point of the incident, 30 minutes before and after. +For [GitLab-managed alerting rules](alerts.md), the issue includes an embedded +chart for the query corresponding to the alert. The chart displays an hour of data +surrounding the starting point of the incident, 30 minutes before and after. -For [manually configured Prometheus instances](../../user/project/integrations/prometheus.md#manual-configuration-of-prometheus), a chart corresponding to the query can be included if these requirements are met: +For [manually configured Prometheus instances](../../user/project/integrations/prometheus.md#manual-configuration-of-prometheus), +a chart corresponding to the query can be included if these requirements are met: - The alert corresponds to an environment managed through GitLab. - The alert corresponds to a [range query](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries). @@ -69,25 +79,31 @@ For [manually configured Prometheus instances](../../user/project/integrations/p | Attributes | Required | Description | | ---------- | -------- | ----------- | | `annotations/gitlab_environment_name` | Yes | Name of the GitLab-managed environment corresponding to the alert | -| One of `annotations/title`, `annotations/summary`, `labels/alertname` | Yes | Will be used as the chart title | -| `annotations/gitlab_y_label` | No | Will be used as the chart's y-axis label | +| One of `annotations/title`, `annotations/summary`, `labels/alertname` | Yes | Used as the chart title | +| `annotations/gitlab_y_label` | No | Used as the chart's y-axis label | ## Embedding cluster health charts > - [Introduced](<https://gitlab.com/gitlab-org/gitlab/-/issues/40997>) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.9. > - [Moved](<https://gitlab.com/gitlab-org/gitlab/-/issues/208224>) to GitLab core in 13.2. -[Cluster Health Metrics](../../user/project/clusters/index.md#visualizing-cluster-health) can also be embedded in [GitLab-flavored Markdown](../../user/markdown.md). +[Cluster Health Metrics](../../user/project/clusters/index.md#visualizing-cluster-health) +can also be embedded in [GitLab-flavored Markdown](../../user/markdown.md). -To embed a metric chart, include a link to that chart in the form `https://<root_url>/<project>/-/cluster/<cluster_id>?<query_params>` anywhere that GitLab-flavored Markdown is supported. To generate and copy a link to the chart, follow the instructions in the [Cluster Health Metric documentation](../../user/project/clusters/index.md#visualizing-cluster-health). +To embed a metric chart, include a link to that chart in the form +`https://<root_url>/<project>/-/cluster/<cluster_id>?<query_params>` anywhere that +GitLab-flavored Markdown is supported. To generate and copy a link to the chart, +follow the instructions in the +[Cluster Health Metric documentation](../../user/project/clusters/index.md#visualizing-cluster-health). The following requirements must be met for the metric to unfurl: - The `<cluster_id>` must correspond to a real cluster. - Prometheus must be monitoring the cluster. - The user must be allowed access to the project cluster metrics. -- The dashboards must be reporting data on the [Cluster Health Page](../../user/project/clusters/index.md#visualizing-cluster-health) +- The dashboards must be reporting data on the + [Cluster Health Page](../../user/project/clusters/index.md#visualizing-cluster-health) - If the above requirements are met, then the metric will unfurl as seen below. + If the above requirements are met, then the metric unfurls as seen below. - + diff --git a/doc/operations/metrics/embed_grafana.md b/doc/operations/metrics/embed_grafana.md index 427ad866442..2843a4319a8 100644 --- a/doc/operations/metrics/embed_grafana.md +++ b/doc/operations/metrics/embed_grafana.md @@ -4,62 +4,76 @@ group: APM info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# Embedding Grafana charts +# Embedding Grafana charts **(CORE)** Grafana metrics can be embedded in [GitLab Flavored Markdown](../../user/markdown.md). -## Embedding charts via Grafana Rendered Images +## Embedding charts via Grafana rendered images -It is possible to embed live [Grafana](https://docs.gitlab.com/omnibus/settings/grafana.html) charts in issues, as a [direct linked rendered image](https://grafana.com/docs/grafana/latest/reference/share_panel/#direct-link-rendered-image). +You can embed live [Grafana](https://docs.gitlab.com/omnibus/settings/grafana.html) +charts in issues as a +[direct linked rendered image](https://grafana.com/docs/grafana/latest/reference/share_panel/#direct-link-rendered-image). +The **Direct link rendered image** sharing dialog within Grafana provides the link: -The sharing dialog within Grafana provides the link, as highlighted below. - - + NOTE: **Note:** -For this embed to display correctly, the Grafana instance must be available to the target user, either as a public dashboard, or on the same network. +For this embed to display correctly, the Grafana instance must be available to the +target user, either as a public dashboard or on the same network. -Copy the link and add an image tag as [inline HTML](../../user/markdown.md#inline-html) in your Markdown. You may tweak the query parameters as required. For instance, removing the `&from=` and `&to=` parameters will give you a live chart. Here is example markup for a live chart from GitLab's public dashboard: +Copy the link and add an image tag as [inline HTML](../../user/markdown.md#inline-html) +in your Markdown. You can tweak the query parameters to meet your needs, such as +removing the `&from=` and `&to=` parameters to display a live chart. Here is example +markup for a live chart from GitLab's public dashboard: ```html <img src="https://dashboards.gitlab.com/d/RZmbBr7mk/gitlab-triage?orgId=1&refresh=30s&var-env=gprd&var-environment=gprd&var-prometheus=prometheus-01-inf-gprd&var-prometheus_app=prometheus-app-01-inf-gprd&var-backend=All&var-type=All&var-stage=main&from=1580444107655&to=1580465707655"/> ``` -This will render like so: +This markup renders a graph of `5xx` errors, like this: - + ## Embedding charts via integration with Grafana HTTP API > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/31376) in GitLab 12.5. -Each project can support integration with one Grafana instance. This configuration allows a user to copy a link to a panel in Grafana, then paste it into a GitLab Markdown field. The chart will be rendered in the GitLab chart format. - -Prerequisites for embedding from a Grafana instance: +Each project can support integration with one Grafana instance. This configuration +enables you to copy a link to a panel in Grafana, then paste it into a GitLab Markdown +field. The chart renders in the GitLab chart format. To embed charts +from a Grafana instance, the data source must: -1. The datasource must be a Prometheus instance. -1. The datasource must be proxyable, so the HTTP Access setting should be set to `Server`. +1. Be a Prometheus instance. +1. Be proxyable, so the HTTP Access setting should be set to `Server`: - +  ## Setting up the Grafana integration -1. [Generate an Admin-level API Token in Grafana.](https://grafana.com/docs/grafana/latest/http_api/auth/#create-api-token) +1. In Grafana, [generate an Admin-level API Token](https://grafana.com/docs/grafana/latest/http_api/auth/#create-api-token). 1. In your GitLab project, navigate to **Settings > Operations > Grafana Authentication**. -1. To enable the integration, check the "Active" checkbox. -1. For "Grafana URL", enter the base URL of the Grafana instance. -1. For "API Token", enter the Admin API Token you just generated. +1. To enable the integration, check the **Active** checkbox. +1. For **Grafana URL**, enter the base URL of the Grafana instance. +1. For **API Token**, enter the Admin API Token you just generated. 1. Click **Save Changes**. ## Generating a link to a chart 1. In Grafana, navigate to the dashboard you wish to embed a panel from. -  -1. In the upper-left corner of the page, select a specific value for each variable required for the queries in the chart. -  -1. In Grafana, click on a panel's title, then click **Share** to open the panel's sharing dialog to the **Link** tab. If you click the _dashboard's_ share panel instead, GitLab will attempt to embed the first supported panel on the dashboard (if available). -1. If your Prometheus queries use Grafana's custom template variables, ensure that the "Template variables" option is toggled to **On**. Of Grafana global template variables, only `$__interval`, `$__from`, and `$__to` are currently supported. Toggle **On** the "Current time range" option to specify the time range of the chart. Otherwise, the default range will be the last 8 hours. -  +  +1. In the upper-left corner of the page, select a specific value for each variable + required for the queries in the chart. +  +1. In Grafana, click on a panel's title, then click **Share** to open the panel's + sharing dialog to the **Link** tab. If you click the _dashboard's_ share panel + instead, GitLab attempts to embed the first supported panel on the dashboard (if available). +1. If your Prometheus queries use Grafana's custom template variables, ensure the + **Template variables** option is toggled to **On**. Of Grafana global template + variables, only `$__interval`, `$__from`, and `$__to` are supported. +1. Toggle **On** the **Current time range** option to specify the time range of + the chart. Otherwise, the default range is the last 8 hours. +  1. Click **Copy** to copy the URL to the clipboard. -1. In GitLab, paste the URL into a Markdown field and save. The chart will take a few moments to render. -  +1. In GitLab, paste the URL into a Markdown field and save. The chart takes a few + moments to render. +  diff --git a/doc/user/project/integrations/img/copy_link_to_chart_v12_10.png b/doc/operations/metrics/img/copy_link_to_chart_v12_10.png Binary files differindex fc205240ea5..fc205240ea5 100644 --- a/doc/user/project/integrations/img/copy_link_to_chart_v12_10.png +++ b/doc/operations/metrics/img/copy_link_to_chart_v12_10.png diff --git a/doc/user/project/integrations/img/embed_metrics_issue_template.png b/doc/operations/metrics/img/embed_metrics_issue_template.png Binary files differindex ca39a738d5f..ca39a738d5f 100644 --- a/doc/user/project/integrations/img/embed_metrics_issue_template.png +++ b/doc/operations/metrics/img/embed_metrics_issue_template.png diff --git a/doc/user/project/integrations/img/embedded_metrics_markdown_v12_8.png b/doc/operations/metrics/img/embedded_metrics_markdown_v12_8.png Binary files differindex ffd34705464..ffd34705464 100644 --- a/doc/user/project/integrations/img/embedded_metrics_markdown_v12_8.png +++ b/doc/operations/metrics/img/embedded_metrics_markdown_v12_8.png diff --git a/doc/user/project/integrations/img/embedded_metrics_rendered_v12_8.png b/doc/operations/metrics/img/embedded_metrics_rendered_v12_8.png Binary files differindex b024daaaa8e..b024daaaa8e 100644 --- a/doc/user/project/integrations/img/embedded_metrics_rendered_v12_8.png +++ b/doc/operations/metrics/img/embedded_metrics_rendered_v12_8.png diff --git a/doc/operations/metrics/img/example-dashboard_v13_1.png b/doc/operations/metrics/img/example-dashboard_v13_1.png Binary files differdeleted file mode 100644 index 0cda4ece689..00000000000 --- a/doc/operations/metrics/img/example-dashboard_v13_1.png +++ /dev/null diff --git a/doc/operations/metrics/img/example-dashboard_v13_3.png b/doc/operations/metrics/img/example-dashboard_v13_3.png Binary files differnew file mode 100644 index 00000000000..1178b4a9be7 --- /dev/null +++ b/doc/operations/metrics/img/example-dashboard_v13_3.png diff --git a/doc/user/project/integrations/img/grafana_embedded.png b/doc/operations/metrics/img/grafana_embedded.png Binary files differindex c7946aa4b10..c7946aa4b10 100644 --- a/doc/user/project/integrations/img/grafana_embedded.png +++ b/doc/operations/metrics/img/grafana_embedded.png diff --git a/doc/user/project/integrations/img/grafana_live_embed.png b/doc/operations/metrics/img/grafana_live_embed.png Binary files differindex 91970cd379a..91970cd379a 100644 --- a/doc/user/project/integrations/img/grafana_live_embed.png +++ b/doc/operations/metrics/img/grafana_live_embed.png diff --git a/doc/user/project/integrations/img/grafana_panel_v12_5.png b/doc/operations/metrics/img/grafana_panel_v12_5.png Binary files differindex 18c17b910cd..18c17b910cd 100644 --- a/doc/user/project/integrations/img/grafana_panel_v12_5.png +++ b/doc/operations/metrics/img/grafana_panel_v12_5.png diff --git a/doc/user/project/integrations/img/grafana_sharing_dialog_v12_5.png b/doc/operations/metrics/img/grafana_sharing_dialog_v12_5.png Binary files differindex fae62dd50df..fae62dd50df 100644 --- a/doc/user/project/integrations/img/grafana_sharing_dialog_v12_5.png +++ b/doc/operations/metrics/img/grafana_sharing_dialog_v12_5.png diff --git a/doc/user/project/integrations/img/hide_embedded_metrics_v12_10.png b/doc/operations/metrics/img/hide_embedded_metrics_v12_10.png Binary files differindex 1213029d1d1..1213029d1d1 100644 --- a/doc/user/project/integrations/img/hide_embedded_metrics_v12_10.png +++ b/doc/operations/metrics/img/hide_embedded_metrics_v12_10.png diff --git a/doc/user/project/integrations/img/http_proxy_access_v12_5.png b/doc/operations/metrics/img/http_proxy_access_v12_5.png Binary files differindex 0036a916a12..0036a916a12 100644 --- a/doc/user/project/integrations/img/http_proxy_access_v12_5.png +++ b/doc/operations/metrics/img/http_proxy_access_v12_5.png diff --git a/doc/operations/metrics/img/linked_runbooks_on_charts.png b/doc/operations/metrics/img/linked_runbooks_on_charts.png Binary files differnew file mode 100644 index 00000000000..335ba5dc172 --- /dev/null +++ b/doc/operations/metrics/img/linked_runbooks_on_charts.png diff --git a/doc/operations/metrics/img/prometheus_add_metric.png b/doc/operations/metrics/img/prometheus_add_metric.png Binary files differnew file mode 100644 index 00000000000..32a7cbf3a73 --- /dev/null +++ b/doc/operations/metrics/img/prometheus_add_metric.png diff --git a/doc/operations/metrics/img/prometheus_alert.png b/doc/operations/metrics/img/prometheus_alert.png Binary files differnew file mode 100644 index 00000000000..08680c88b23 --- /dev/null +++ b/doc/operations/metrics/img/prometheus_alert.png diff --git a/doc/user/project/integrations/img/prometheus_cluster_health_embed_v12_9.png b/doc/operations/metrics/img/prometheus_cluster_health_embed_v12_9.png Binary files differindex c669467757f..c669467757f 100644 --- a/doc/user/project/integrations/img/prometheus_cluster_health_embed_v12_9.png +++ b/doc/operations/metrics/img/prometheus_cluster_health_embed_v12_9.png diff --git a/doc/user/project/integrations/img/prometheus_dashboard_edit_metric_link_v_12_9.png b/doc/operations/metrics/img/prometheus_dashboard_edit_metric_link_v_12_9.png Binary files differindex b66b1a9f39b..b66b1a9f39b 100644 --- a/doc/user/project/integrations/img/prometheus_dashboard_edit_metric_link_v_12_9.png +++ b/doc/operations/metrics/img/prometheus_dashboard_edit_metric_link_v_12_9.png diff --git a/doc/operations/metrics/img/prometheus_monitoring_dashboard_v13_3.png b/doc/operations/metrics/img/prometheus_monitoring_dashboard_v13_3.png Binary files differnew file mode 100644 index 00000000000..1178b4a9be7 --- /dev/null +++ b/doc/operations/metrics/img/prometheus_monitoring_dashboard_v13_3.png diff --git a/doc/user/project/integrations/img/prometheus_service_alerts.png b/doc/operations/metrics/img/prometheus_service_alerts.png Binary files differindex 609c5e5196c..609c5e5196c 100644 --- a/doc/user/project/integrations/img/prometheus_service_alerts.png +++ b/doc/operations/metrics/img/prometheus_service_alerts.png diff --git a/doc/user/project/integrations/img/rendered_grafana_embed_v12_5.png b/doc/operations/metrics/img/rendered_grafana_embed_v12_5.png Binary files differindex 6cabe4193bd..6cabe4193bd 100644 --- a/doc/user/project/integrations/img/rendered_grafana_embed_v12_5.png +++ b/doc/operations/metrics/img/rendered_grafana_embed_v12_5.png diff --git a/doc/user/project/integrations/img/select_query_variables_v12_5.png b/doc/operations/metrics/img/select_query_variables_v12_5.png Binary files differindex 23503577327..23503577327 100644 --- a/doc/user/project/integrations/img/select_query_variables_v12_5.png +++ b/doc/operations/metrics/img/select_query_variables_v12_5.png diff --git a/doc/user/project/integrations/img/view_embedded_metrics_v12_10.png b/doc/operations/metrics/img/view_embedded_metrics_v12_10.png Binary files differindex 95bb148ba71..95bb148ba71 100644 --- a/doc/user/project/integrations/img/view_embedded_metrics_v12_10.png +++ b/doc/operations/metrics/img/view_embedded_metrics_v12_10.png diff --git a/doc/operations/metrics/index.md b/doc/operations/metrics/index.md index 12088884f44..92c4a4986bc 100644 --- a/doc/operations/metrics/index.md +++ b/doc/operations/metrics/index.md @@ -4,7 +4,64 @@ group: APM info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# Monitor metrics for your CI/CD environment +# Monitor your CI/CD environment's metrics **(CORE)** + +GitLab helps your team monitor the health and performance of your applications +and infrastructure by turning statistics and log files into charts and graphs +that are easy to understand, especially when time is short and decisions are +critical. For GitLab to display your information in charts, you must: + +1. **Instrument your application** - Collect accurate and complete measurements. + <I class="fa fa-youtube-play youtube" aria-hidden="true"></I> + For an overview, see [How to instrument Prometheus metrics in GitLab](https://www.youtube.com/watch?v=tuI2oJ3TTB4). +1. **Expose metrics for capture** - Make logs, metrics, and traces available for capture. +1. [**Configure Prometheus to gather metrics**](#configure-prometheus-to-gather-metrics) - + Deploy managed applications like Elasticsearch, Prometheus, and Jaeger to gather + the data you've exposed. +1. **GitLab collects metrics** - GitLab uses Prometheus to scrape the data you've + captured in your managed apps, and prepares the data for display. To learn more, read + [Collect and process metrics](#collect-and-process-metrics). +1. **Display charts in the GitLab user interface** - GitLab converts your metrics + into easy-to-read charts on a default dashboard. You can create as many custom charts + and custom dashboards as needed so your team has full insight into your + application's health. + +## Configure Prometheus to gather metrics + +You must connect a Prometheus instance to GitLab to collect metrics. How you configure +your Prometheus integration depends on where your apps are running: + +- **For manually-configured Prometheus** - + [Specify your Prometheus server](../../user/project/integrations/prometheus.md#manual-configuration-of-prometheus), + and define at least one environment. +- **For GitLab-managed Prometheus** - GitLab can + [deploy and manage Prometheus](../../user/project/integrations/prometheus.md#managed-prometheus-on-kubernetes) for you. + You must also complete a code deployment, as described in + [Deploy code with GitLab-managed Prometheus](#deploy-code-with-gitlab-managed-prometheus), + for the **Operations > Metrics** page to contain data. + +### Deploy code with GitLab-managed Prometheus + +For GitLab-managed Prometheus, you can set up [Auto DevOps](../../topics/autodevops/index.md) +to quickly create a deployment: + +1. Navigate to your project's **Operations > Kubernetes** page. +1. Ensure that, in addition to Prometheus, you also have Runner and Ingress + installed. +1. After installing Ingress, copy its endpoint. +1. Navigate to your project's **Settings > CI/CD** page. In the + **Auto DevOps** section, select a deployment strategy and save your changes. +1. On the same page, in the **Variables** section, add a variable named + `KUBE_INGRESS_BASE_DOMAIN` with the value of the Ingress endpoint you + copied previously. Leave the type as **Variable**. +1. Navigate to your project's **{rocket}** **CI/CD > Pipelines** page, and run a + pipeline on any branch. +1. When the pipeline has run successfully, graphs are available on the + **Operations > Metrics** page. + + + +## Collect and process metrics After [configuring Prometheus for a cluster](../../user/project/integrations/prometheus.md), GitLab attempts to retrieve performance metrics for any [environment](../../ci/environments/index.md) with @@ -15,17 +72,17 @@ and NGINX, and attempts to identify individual environments. To learn more about the supported metrics and scan processes, see the [Prometheus Metrics Library documentation](../../user/project/integrations/prometheus_library/index.md). -To view the metrics dashboard for an environment that has -[completed at least one deployment](#populate-your-metrics-dashboard): +To view the [default metrics dashboard](dashboards/default.md) for an environment that is +[configured to gather metrics](#configure-prometheus-to-gather-metrics): 1. *If the metrics dashboard is only visible to project members,* sign in to GitLab as a member of a project. Learn more about [metrics dashboard visibility](#metrics-dashboard-visibility). -1. In your project, navigate to **{cloud-gear}** **Operations > Metrics**. +1. In your project, navigate to **Operations > Metrics**. -GitLab displays the default metrics dashboard for the environment, like the -following example: +GitLab displays the [default metrics dashboard](dashboards/default.md) for the environment, +like the following example: - + The top of the dashboard contains a navigation bar. From left to right, the navigation bar contains: @@ -37,38 +94,22 @@ navigation bar contains: - **Range** - The time period of data to display. - **Refresh dashboard** **{retry}** - Reload the dashboard with current data. - **Set refresh rate** - Set a time frame for refreshing the data displayed. -- **Star dashboard** **{star-o}** - Click to mark a dashboard as a favorite. +- **More actions** **{ellipsis_v}** - More dashboard actions + - **Add metric** - Adds a [custom metric](#adding-custom-metrics). Only available on GitLab-defined dashboards. + ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/34779) in GitLab 12.5.) + - **Edit dashboard YAML** - Edit the source YAML file of a custom dashboard. Only available on + [custom dashboards](dashboards/index.md). + ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/34779) in GitLab 12.5.) + - **Duplicate current dashboard** - Save a [complete copy of a dashboard](dashboards/index.md#duplicate-a-gitlab-defined-dashboard). Only available on GitLab-defined dashboards. + - **Star dashboard** **{star-o}** - Click to mark a dashboard as a favorite. Starred dashboards display a solid star **{star}** button, and display first in the **Dashboard** dropdown list. ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/214582) in GitLab 13.0.) -- **Create dashboard** **{file-addition-solid}** - Create a - [new custom dashboard for your project](dashboards/index.md#adding-a-new-dashboard-to-your-project). -- **Metrics settings** **{settings}** - Configure the + - **Create new dashboard** - Create a [new custom dashboard for your project](dashboards/index.md#add-a-new-dashboard-to-your-project). + ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/228856) in GitLab 13.3.) +- **Metrics settings** - Configure the [settings for this dashboard](dashboards/index.md#manage-the-metrics-dashboard-settings). -## Populate your metrics dashboard - -After [configuring Prometheus for a cluster](../../user/project/integrations/prometheus.md), -you must also deploy code for the **{cloud-gear}** **Operations > Metrics** page -to contain data. Setting up [Auto DevOps](../../topics/autodevops/index.md) -helps quickly create a deployment: - -1. Navigate to your project's **{cloud-gear}** **Operations > Kubernetes** page. -1. Ensure that, in addition to Prometheus, you also have Runner and Ingress - installed. -1. After installing Ingress, copy its endpoint. -1. Navigate to your project's **{settings}** **Settings > CI/CD** page. In the - **Auto DevOps** section, select a deployment strategy and save your changes. -1. On the same page, in the **Variables** section, add a variable named - `KUBE_INGRESS_BASE_DOMAIN` with the value of the Ingress endpoint you - copied previously. Leave the type as **Variable**. -1. Navigate to your project's **{rocket}** **CI/CD > Pipelines** page, and run a - pipeline on any branch. -1. When the pipeline has run successfully, graphs are available on the - **{cloud-gear}** **Operations > Metrics** page. - - - ## Customize your metrics dashboard After creating your dashboard, you can customize it to meet your needs: @@ -101,7 +142,7 @@ After saving them, they display on the environment metrics dashboard provided th [Prometheus installed on the cluster](../../user/project/integrations/prometheus.md#enabling-prometheus-integration). - Prometheus is [manually configured](../../user/project/integrations/prometheus.md#manual-configuration-of-prometheus). - + A few fields are required: @@ -124,7 +165,7 @@ suggested if this feature is used. You can edit existing additional custom metrics for your dashboard by clicking the **{ellipsis_v}** **More actions** dropdown and selecting **Edit metric**. - + ## Keyboard shortcuts for charts diff --git a/doc/operations/product_analytics.md b/doc/operations/product_analytics.md new file mode 100644 index 00000000000..96f1a45167b --- /dev/null +++ b/doc/operations/product_analytics.md @@ -0,0 +1,82 @@ +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Product Analytics **(CORE)** + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/225167) in GitLab 13.3. +> - It's deployed behind a feature flag, disabled by default. +> - It's disabled on GitLab.com. +> - It's able to be enabled or disabled per-project. +> - It's not recommended for production use. +> - To use it in GitLab self-managed instances, ask a GitLab administrator to enable it. + +GitLab allows you to go from planning an application to getting feedback. Feedback +is not just observability, but also knowing how people use your product. +Product Analytics uses events sent from your application to know how they are using it. +It's based on [Snowplow](https://github.com/snowplow/snowplow), the best open-source +event tracker. With Product Analytics, you can receive and analyze the Snowplow data +inside GitLab. + +## Enable or disable Product Analytics + +Product Analytics is under development and not ready for production use. It's +deployed behind a feature flag that's **disabled by default**. +[GitLab administrators with access to the GitLab Rails console](../administration/feature_flags.md) +can enable it for your instance. Product Analytics can be enabled or disabled per-project. + +To enable it: + +```ruby +# Instance-wide +Feature.enable(:product_analytics) +# or by project +Feature.enable(:product_analytics, Project.find(<project id>)) +``` + +To disable it: + +```ruby +# Instance-wide +Feature.disable(:product_analytics) +# or by project +Feature.disable(:product_analytics, Project.find(<project id>)) +``` + +## Access Product Analytics + +After enabling the feature flag for Product Analytics, you can access the +user interface: + +1. Sign in to GitLab as a user with Reporter or greater + [permissions](../user/permissions.md). +1. Navigate to **Operations > Product Analytics** + +The user interface contains: + +- An Events page that shows the recent events and a total count. +- A test page that sends a sample event. +- A setup page containing the code to implement in your application. + +## Rate limits for Product Analytics + +While Product Analytics is under development, it's rate-limited to +**100 events per minute** per project. This limit prevents the events table in the +database from growing too quickly. + +## Data storage for Product Analytics + +Product Analytics stores events are stored in GitLab database. + +CAUTION: **Caution:** +This data storage is experimental, and GitLab is likely to remove this data during +future development. + +## Event collection + +Events are collected by [Rails collector](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/36443), +allowing GitLab to ship the feature fast. Due to scalability issue, GitLab plans +to switch to a separate application, such as +[snowplow-go-collector](https://gitlab.com/gitlab-org/snowplow-go-collector), for event collection. diff --git a/doc/policy/maintenance.md b/doc/policy/maintenance.md index 5b3fc3e0021..7f3f75d933d 100644 --- a/doc/policy/maintenance.md +++ b/doc/policy/maintenance.md @@ -112,9 +112,9 @@ Please see the table below for some examples: | Target version | Your version | Recommended upgrade path | Note | | --------------------- | ------------ | ------------------------ | ---- | -| `13.2.0` | `11.5.0` | `11.5.0` -> `11.11.8` -> `12.0.12` -> `12.10.6` -> `13.0.0` -> `13.2.0` | Four intermediate versions are required: the final `11.11`, `12.0`, and `12.10` releases, plus `13.0`. | -| `13.0.1` | `11.10.8` | `11.10.5` -> `11.11.8` -> `12.0.12` -> `12.10.6` -> `13.0.1` | Three intermediate versions are required: `11.11`, `12.0`, and `12.10`. | -| `12.10.6` | `11.3.4` | `11.3.4` -> `11.11.8` -> `12.0.12` -> `12.10.6` | Two intermediate versions are required: `11.11` and `12.0` | +| `13.2.3` | `11.5.0` | `11.5.0` -> `11.11.8` -> `12.0.12` -> `12.10.14` -> `13.0.12` -> `13.2.3` | Four intermediate versions are required: the final `11.11`, `12.0`, and `12.10` releases, plus `13.0`. | +| `13.0.12` | `11.10.8` | `11.10.5` -> `11.11.8` -> `12.0.12` -> `12.10.14` -> `13.0.12` | Three intermediate versions are required: `11.11`, `12.0`, and `12.10`. | +| `12.10.14` | `11.3.4` | `11.3.4` -> `11.11.8` -> `12.0.12` -> `12.10.14` | Two intermediate versions are required: `11.11` and `12.0` | | `12.9.5` | `10.4.5` | `10.4.5` -> `10.8.7` -> `11.11.8` -> `12.0.12` -> `12.9.5` | Three intermediate versions are required: `10.8`, `11.11`, and `12.0`, then `12.9.5` | | `12.2.5` | `9.2.6` | `9.2.6` -> `9.5.10` -> `10.8.7` -> `11.11.8` -> `12.0.12` -> `12.2.5` | Four intermediate versions are required: `9.5`, `10.8`, `11.11`, `12.0`, then `12.2`. | | `11.3.4` | `8.13.4` | `8.13.4` -> `8.17.7` -> `9.5.10` -> `10.8.7` -> `11.3.4` | `8.17.7` is the last version in version 8, `9.5.10` is the last version in version 9, `10.8.7` is the last version in version 10. | diff --git a/doc/push_rules/push_rules.md b/doc/push_rules/push_rules.md index f958eaa1b21..6a3939b0de8 100644 --- a/doc/push_rules/push_rules.md +++ b/doc/push_rules/push_rules.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" type: reference, howto --- @@ -98,7 +101,7 @@ The following options are available. | Maximum file size | **Starter** 7.12 | Pushes that contain added or updated files that exceed this file size (in MB) are rejected. Set to 0 to allow files of any size. Files tracked by Git LFS are exempted. | TIP: **Tip:** -GitLab uses [RE2 syntax](https://github.com/google/re2/wiki/Syntax) for regular expressions in push rules, and you can test them at the [GoLang regex tester](https://regex-golang.appspot.com/assets/html/index.html). +GitLab uses [RE2 syntax](https://github.com/google/re2/wiki/Syntax) for regular expressions in push rules, and you can test them at the [regex101 regex tester](https://regex101.com/). ## Prevent pushing secrets to the repository diff --git a/doc/raketasks/README.md b/doc/raketasks/README.md index b7cfc18534b..54b504bbfe8 100644 --- a/doc/raketasks/README.md +++ b/doc/raketasks/README.md @@ -35,6 +35,7 @@ The following are available Rake tasks: | [Praefect Rake tasks](../administration/raketasks/praefect.md) | [Praefect](../administration/gitaly/praefect.md)-related tasks. | | [Project import/export](../administration/raketasks/project_import_export.md) | Prepare for [project exports and imports](../user/project/settings/import_export.md). | | [Sample Prometheus data](generate_sample_prometheus_data.md) | Generate sample Prometheus data. | +| [SPDX license list import](spdx.md) **(PREMIUM ONLY)** | Import a local copy of the [SPDX license list](https://spdx.org/licenses/) for matching [License Compliance policies](../user/compliance/license_compliance/index.md).| | | [Repository storage](../administration/raketasks/storage.md) | List and migrate existing projects and attachments from legacy storage to hashed storage. | | [Uploads migrate](../administration/raketasks/uploads/migrate.md) | Migrate uploads between storage local and object storage. | | [Uploads sanitize](../administration/raketasks/uploads/sanitize.md) | Remove EXIF data from images uploaded to earlier versions of GitLab. | diff --git a/doc/raketasks/backup_restore.md b/doc/raketasks/backup_restore.md index d8522cc19a0..2163cf22944 100644 --- a/doc/raketasks/backup_restore.md +++ b/doc/raketasks/backup_restore.md @@ -9,6 +9,11 @@ You can only restore a backup to **exactly the same version and type (CE/EE)** of GitLab on which it was created. The best way to migrate your repositories from one server to another is through backup restore. +CAUTION: **Warning:** +GitLab will not backup items that are not stored on the +filesystem. If using [object storage](../administration/object_storage.md), +remember to enable backups with your object storage provider if desired. + ## Requirements In order to be able to backup and restore, you need two essential tools @@ -290,6 +295,30 @@ For installations from source: sudo -u git -H bundle exec rake gitlab:backup:create SKIP=tar RAILS_ENV=production ``` +#### Back up Git repositories concurrently + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/37158) in GitLab 13.3. + +Repositories can be backed up concurrently to help fully utilise CPU time. The following variables +are available to modify the default behavior of the Rake task: + +- `GITLAB_BACKUP_MAX_CONCURRENCY` sets the maximum number of projects to backup at the same time. + Defaults to 1. +- `GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY` sets the maximum number of projects to backup at the same time on each storage. This allows the repository backups to be spread across storages. + Defaults to 1. + +For example, for Omnibus GitLab installations: + +```shell +sudo gitlab-backup create GITLAB_BACKUP_MAX_CONCURRENCY=4 GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY=1 +``` + +For example, for installations from source: + +```shell +sudo -u git -H bundle exec rake gitlab:backup:create GITLAB_BACKUP_MAX_CONCURRENCY=4 GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY=1 +``` + #### Uploading backups to a remote (cloud) storage Starting with GitLab 7.4 you can let the backup script upload the `.tar` file it creates. @@ -516,7 +545,7 @@ directory that you want to copy the tarballs to is the root of your mounted directory, just use `.` instead. NOTE: **Note:** -Since file system performance may affect GitLab's overall performance, we do not recommend using EFS for storage. See the [relevant documentation](../administration/high_availability/nfs.md#avoid-using-awss-elastic-file-system-efs) for more details. +Since file system performance may affect GitLab's overall performance, we do not recommend using EFS for storage. See the [relevant documentation](../administration/nfs.md#avoid-using-awss-elastic-file-system-efs) for more details. For Omnibus GitLab packages: @@ -717,7 +746,7 @@ sure these directories are empty before attempting a restore. Otherwise GitLab will attempt to move these directories before restoring the new data and this would cause an error. -Read more on [configuring NFS mounts](../administration/high_availability/nfs.md) +Read more on [configuring NFS mounts](../administration/nfs.md) ### Restore for installation from source @@ -960,11 +989,10 @@ For more information see similar questions on PostgreSQL issue tracker [here](ht ### When the secrets file is lost -If you have failed to [back up the secrets file](#storing-configuration-files), -then users with 2FA enabled will not be able to log into GitLab. In that case, -you need to [disable 2FA for everyone](../security/two_factor_authentication.md#disabling-2fa-for-everyone). +If you have failed to [back up the secrets file](#storing-configuration-files), you'll +need to perform a number of steps to get GitLab working properly again. -The secrets file is also responsible for storing the encryption key for several +The secrets file is responsible for storing the encryption key for several columns containing sensitive information. If the key is lost, GitLab will be unable to decrypt those columns. This will break a wide range of functionality, including (but not restricted to): @@ -972,7 +1000,7 @@ including (but not restricted to): - [CI/CD variables](../ci/variables/README.md) - [Kubernetes / GCP integration](../user/project/clusters/index.md) - [Custom Pages domains](../user/project/pages/custom_domains_ssl_tls_certification/index.md) -- [Project error tracking](../user/project/operations/error_tracking.md) +- [Project error tracking](../operations/error_tracking.md) - [Runner authentication](../ci/runners/README.md) - [Project mirroring](../user/project/repository/repository_mirroring.md) - [Web hooks](../user/project/integrations/webhooks.md) @@ -983,17 +1011,28 @@ experience some unexpected behavior such as: - Stuck jobs. - 500 errors. -You can check whether you have undecryptable values in the database using -the [Secrets Doctor Rake task](../administration/raketasks/doctor.md). - In this case, you are required to reset all the tokens for CI/CD variables and Runner Authentication, which is described in more detail below. After resetting the tokens, you should be able to visit your project and the jobs -will have started running again. +will have started running again. Use the information in the following sections at your own risk. + +#### Check for undecryptable values + +You can check whether you have undecryptable values in the database using +the [Secrets Doctor Rake task](../administration/raketasks/doctor.md). + +#### Take a backup + +You will need to directly modify GitLab data to work around your lost secrets file. CAUTION: **Warning:** -Use the following commands at your own risk, and make sure you've taken a -backup beforehand. +Make sure you've taken a backup beforehand, particularly a full database backup. + +#### Disable user two-factor authentication (2FA) + +Users with 2FA enabled will not be able to log into GitLab. In that case, +you need to [disable 2FA for everyone](../security/two_factor_authentication.md#disabling-2fa-for-everyone) +and then users will have to reactivate 2FA from scratch. #### Reset CI/CD variables @@ -1090,6 +1129,35 @@ A similar strategy can be employed for the remaining features - by removing the data that cannot be decrypted, GitLab can be brought back into working order, and the lost data can be manually replaced. +#### Fix project integrations + +If you've lost your secrets, the +[projects' integrations settings pages](../user/project/integrations/index.md) +are probably generating 500 errors. + +The fix is to truncate the `web_hooks` table: + +1. Enter the DB console: + + For Omnibus GitLab packages: + + ```shell + sudo gitlab-rails dbconsole + ``` + + For installations from source: + + ```shell + sudo -u git -H bundle exec rails dbconsole -e production + ``` + +1. Truncate the table + + ```sql + -- truncate web_hooks table + TRUNCATE web_hooks CASCADE; + ``` + ### Container Registry push failures after restoring from a backup If you use the [Container Registry](../user/packages/container_registry/index.md), you diff --git a/doc/raketasks/cleanup.md b/doc/raketasks/cleanup.md index cf4edea383b..c4046b36c55 100644 --- a/doc/raketasks/cleanup.md +++ b/doc/raketasks/cleanup.md @@ -142,6 +142,10 @@ I, [2018-08-02T10:26:47.764356 #45087] INFO -- : Moved to lost and found: @hash > - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/29681) in GitLab 12.1. > - [`ionice` support fixed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/28023) in GitLab 12.10. +NOTE: **Note:** +These commands will not work for artifacts stored on +[object storage](../administration/object_storage.md). + When you notice there are more job artifacts files on disk than there should be, you can run: diff --git a/doc/raketasks/spdx.md b/doc/raketasks/spdx.md new file mode 100644 index 00000000000..23eb27eb059 --- /dev/null +++ b/doc/raketasks/spdx.md @@ -0,0 +1,18 @@ +# SPDX license list import **(PREMIUM ONLY)** + +GitLab provides a Rake task for uploading a fresh copy of the [SPDX license list](https://spdx.org/licenses/) +to a GitLab instance. This list is needed for matching the names of [License Compliance policies](../user/compliance/license_compliance/index.md). + +To import a fresh copy of the PDX license list, run: + +```shell +# omnibus-gitlab +sudo gitlab-rake gitlab:spdx:import + +# source installations +bundle exec rake gitlab:spdx:import RAILS_ENV=production +``` + +To perform this task in the [offline environment](../user/application_security/offline_deployments/#defining-offline-environments), +an outbound connection to [`licenses.json`](https://spdx.org/licenses/licenses.json) should be +allowed. diff --git a/doc/security/README.md b/doc/security/README.md index e2375c0f0b5..bbc7db54b14 100644 --- a/doc/security/README.md +++ b/doc/security/README.md @@ -7,6 +7,7 @@ type: index - [Password storage](password_storage.md) - [Password length limits](password_length_limits.md) +- [Generated passwords for users created through integrated authentication](passwords_for_integrated_authentication_methods.md) - [Restrict SSH key technologies and minimum length](ssh_keys_restrictions.md) - [Rate limits](rate_limits.md) - [Webhooks and insecure internal web services](webhooks.md) diff --git a/doc/security/cicd_environment_variables.md b/doc/security/cicd_environment_variables.md index ea597ea05f2..b8fe14e2d3b 100644 --- a/doc/security/cicd_environment_variables.md +++ b/doc/security/cicd_environment_variables.md @@ -1,5 +1,7 @@ --- -type: reference +stage: Release +group: Release Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # CI/CD Environment Variables diff --git a/doc/security/passwords_for_integrated_authentication_methods.md b/doc/security/passwords_for_integrated_authentication_methods.md new file mode 100644 index 00000000000..704af49b2d2 --- /dev/null +++ b/doc/security/passwords_for_integrated_authentication_methods.md @@ -0,0 +1,14 @@ +--- +type: reference +--- + +# Generated passwords for users created through integrated authentication + +GitLab allows users to set up accounts through integration with external [authentication and authorization providers](../administration/auth/README.md). + +These authentication methods do not require the user to explicitly create a password for their accounts. +However, to maintain data consistency, GitLab requires passwords for all user accounts. + +For such accounts, we use the [`friendly_token`](https://github.com/heartcombo/devise/blob/f26e05c20079c9acded3c0ee16da0df435a28997/lib/devise.rb#L492) method provided by the Devise gem to generate a random, unique and secure password and sets it as the account password during sign up. + +The length of the generated password is the set based on the value of [maximum password length](password_length_limits.md#modify-maximum-password-length-using-configuration-file) as set in the Devise configuation. The default value is 128 characters. diff --git a/doc/security/ssh_keys_restrictions.md b/doc/security/ssh_keys_restrictions.md index 47eccf665d3..903a28136ad 100644 --- a/doc/security/ssh_keys_restrictions.md +++ b/doc/security/ssh_keys_restrictions.md @@ -1,5 +1,8 @@ --- type: reference, howto +stage: Manage +group: Access +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # Restrict allowed SSH key technologies and minimum length diff --git a/doc/security/two_factor_authentication.md b/doc/security/two_factor_authentication.md index 886154aac6d..9d49e1d3af2 100644 --- a/doc/security/two_factor_authentication.md +++ b/doc/security/two_factor_authentication.md @@ -1,5 +1,8 @@ --- type: howto +stage: Manage +group: Access +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # Enforce Two-factor Authentication (2FA) @@ -36,8 +39,9 @@ period to `0`. If you want to enforce 2FA only for certain groups, you can: -1. Enable it in the group's **Settings > General** page. -1. Optionally specify a grace period as above. +1. Enable it in the group's **Settings > General** page. Navigate to **Permissions, LFS, 2FA > Two-factor authentication**. +You can then check the **Require all users in this group to setup Two-factor authentication** option. +1. You can also specify a grace period in the **Time before enforced** option. To change this setting, you need to be administrator or owner of the group. diff --git a/doc/security/user_email_confirmation.md b/doc/security/user_email_confirmation.md index a493b374d66..6260c76bff9 100644 --- a/doc/security/user_email_confirmation.md +++ b/doc/security/user_email_confirmation.md @@ -1,5 +1,8 @@ --- type: howto +stage: Manage +group: Access +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # User email confirmation at sign-up diff --git a/doc/security/user_file_uploads.md b/doc/security/user_file_uploads.md index 9fc8f7ec985..662e115d1ed 100644 --- a/doc/security/user_file_uploads.md +++ b/doc/security/user_file_uploads.md @@ -1,5 +1,8 @@ --- type: reference +stage: Manage +group: Access +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # User File Uploads @@ -15,7 +18,7 @@ notification emails, which are often read from email clients that are not authenticated with GitLab, such as Outlook, Apple Mail, or the Mail app on your mobile device. ->**Note:** +NOTE: **Note:** Non-image attachments do require authentication to be viewed. <!-- ## Troubleshooting diff --git a/doc/security/webhooks.md b/doc/security/webhooks.md index af9be499e80..3d7aa3026ab 100644 --- a/doc/security/webhooks.md +++ b/doc/security/webhooks.md @@ -5,7 +5,7 @@ type: concepts, reference, howto # Webhooks and insecure internal web services NOTE: **Note:** -On GitLab.com the [maximum number of webhooks](../user/gitlab_com/index.md#maximum-number-of-webhooks) per project is limited. +On GitLab.com, the [maximum number of webhooks and their size](../user/gitlab_com/index.md#webhooks) per project, and per group, is limited. If you have non-GitLab web services running on your GitLab server or within its local network, these may be vulnerable to exploitation via Webhooks. diff --git a/doc/ssh/README.md b/doc/ssh/README.md index 980a20ade3e..b4b6f9a70e3 100644 --- a/doc/ssh/README.md +++ b/doc/ssh/README.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" type: howto, reference --- @@ -133,14 +136,14 @@ Enter file in which to save the key (/home/user/.ssh/id_rsa): ``` If you don't already have an SSH key pair and are not generating a [deploy key](#deploy-keys), -accept the suggested file and directory. Your SSH client will use +accept the suggested file and directory. Your SSH client uses the resulting SSH key pair with no additional configuration. Alternatively, you can save the new SSH key pair in a different location. -You can assign the directory and file name of your choice. +You can assign the directory and filename of your choice. You can also dedicate that SSH key pair to a [specific host](#working-with-non-default-ssh-key-pair-paths). -After assigning a file to save your SSH key, you'll get a chance to set up +After assigning a file to save your SSH key, you can set up a [passphrase](https://www.ssh.com/ssh/passphrase/) for your SSH key: ```plaintext @@ -221,7 +224,7 @@ Now you can copy the SSH key you created to your GitLab account. To do so, follo 1. Include an (optional) expiry date for the key under "Expires at" section. (Introduced in [GitLab 12.9](https://gitlab.com/gitlab-org/gitlab/-/issues/36243).) 1. Click the **Add key** button. -SSH keys that have "expired" using this procedure will still be valid in GitLab workflows. +SSH keys that have "expired" using this procedure are valid in GitLab workflows. As the GitLab-configured expiration date is not included in the SSH key itself, you can still export public SSH keys as needed. @@ -238,7 +241,7 @@ your terminal (replacing `gitlab.com` with your GitLab's instance domain): ssh -T git@gitlab.com ``` -The first time you connect to GitLab via SSH, you will be asked to verify the +The first time you connect to GitLab via SSH, you should verify the authenticity of the GitLab host that you're connecting to. For example, when connecting to GitLab.com, answer `yes` to add GitLab.com to the list of trusted hosts: @@ -253,10 +256,10 @@ Warning: Permanently added 'gitlab.com' (ECDSA) to the list of known hosts. NOTE: **Note:** For GitLab.com, consult the [SSH host keys fingerprints](../user/gitlab_com/index.md#ssh-host-keys-fingerprints), -section to make sure you're connecting to the correct server. For example, you'll see +section to make sure you're connecting to the correct server. For example, you can see the ECDSA key fingerprint shown above in the linked section. -Once added to the list of known hosts, you won't be asked to validate the +Once added to the list of known hosts, you should validate the authenticity of GitLab's host again. Run the above command once more, and you should only receive a _Welcome to GitLab, `@username`!_ message. @@ -294,8 +297,8 @@ Host gitlab.company.com IdentityFile ~/.ssh/example_com_rsa ``` -Public SSH keys need to be unique to GitLab, as they will bind to your account. -Your SSH key is the only identifier you'll have when pushing code via SSH, +Public SSH keys need to be unique to GitLab, as they bind to your account. +Your SSH key is the only identifier you have when pushing code via SSH, that's why it needs to uniquely map to a single user. ## Per-repository SSH keys @@ -307,7 +310,7 @@ on, you can issue the following command while inside your repository: git config core.sshCommand "ssh -o IdentitiesOnly=yes -i ~/.ssh/private-key-filename-for-this-repository -F /dev/null" ``` -This will not use the SSH Agent and requires at least Git 2.10. +This does not use the SSH Agent and requires at least Git 2.10. ## Multiple accounts on a single GitLab instance @@ -315,7 +318,7 @@ The [per-repository](#per-repository-ssh-keys) method also works for using multiple accounts within a single GitLab instance. Alternatively, it is possible to directly assign aliases to hosts in -`~.ssh/config`. SSH and, by extension, Git will fail to log in if there is +`~.ssh/config`. SSH and, by extension, Git fails to log in if there is an `IdentityFile` set outside of a `Host` block in `.ssh/config`. This is due to how SSH assembles `IdentityFile` entries and is not changed by setting `IdentitiesOnly` to `yes`. `IdentityFile` entries should point to @@ -385,14 +388,14 @@ GitLab integrates with the system-installed SSH daemon, designating a user connecting to the GitLab server over SSH are identified by their SSH key instead of their username. -SSH *client* operations performed on the GitLab server will be executed as this +SSH *client* operations performed on the GitLab server are executed as this user. Although it is possible to modify the SSH configuration for this user to, e.g., provide a private SSH key to authenticate these requests by, this practice is **not supported** and is strongly discouraged as it presents significant security risks. -The GitLab check process includes a check for this condition, and will direct you -to this section if your server is configured like this, e.g.: +The GitLab check process includes a check for this condition, and directs you +to this section if your server is configured like this, for example: ```shell $ gitlab-rake gitlab:check diff --git a/doc/subscriptions/index.md b/doc/subscriptions/index.md index 64bf2e0b50b..35559c9b970 100644 --- a/doc/subscriptions/index.md +++ b/doc/subscriptions/index.md @@ -76,7 +76,7 @@ Every occupied seat, whether by person, job, or bot is counted in the subscripti renewal of a subscription won't be counted as active users for the renewal subscription. They may count as active users in the subscription period in which they were originally added. - Members with Guest permissions on an Ultimate subscription. -- GitLab-created service accounts: `Ghost User` and `Support Bot`. +- GitLab-created service accounts: `Ghost User`, `Support Bot` and [`Project bot users`](../user/project/settings/project_access_tokens.md#project-bot-users). ##### Users statistics @@ -126,38 +126,78 @@ If you're purchasing a subscription for an existing **Core** self-managed instance, ensure you're purchasing enough seats to [cover your users](../user/admin_area/index.md#administering-users). +### Credit card declined + +If your credit card is declined when purchasing a GitLab subscription, possible reasons include: + +- The credit card details provided are incorrect. +- The credit card account has insufficient funds. +- You are using a virtual credit card and it has insufficient funds, or has expired. +- The transaction exceeds the credit limit. +- The transaction exceeds the credit card's maximum transaction amount. + +Check with your financial institution to confirm if any of these reasons apply. If they don't +apply, contact [GitLab Support](https://support.gitlab.com/hc/en-us/requests/new?ticket_form_id=360000071293). + ## Manage your GitLab account With the [Customers Portal](https://customers.gitlab.com/) you can: -- [Change billing and company information](#change-billing-information) -- [Change the payment method](#change-payment-method) +- [Change your personal details](#change-your-personal-details) +- [Change your company details](#change-your-company-details) +- [Change your payment method](#change-your-payment-method) - [Change the linked account](#change-the-linked-account) - [Change the associated namespace](#change-the-associated-namespace) - [Change customers portal account password](#change-customer-portal-account-password) -### Change billing information +### Change your personal details + +Your personal details are used on invoices. Your email address is used for the Customers Portal +login and license-related email. -To change billing information: +To change your personal details, including name and billing address: 1. Log in to the [Customers Portal](https://customers.gitlab.com/customers/sign_in). -1. Select the **My account** drop-down and click on **Payment methods**. -1. Make the required changes to the **Account Details** information. +1. Select **My account > Account details**. +1. Expand the **Personal details** section. +1. Edit your personal details. 1. Click **Save changes**. -NOTE: **Note:** -Future purchases will use the information in this section. -The email listed in this section is used for the Customers Portal -login and for license-related email communication. +### Change your company details + +To change your company details, including company name and VAT number: + +1. Log in to the [Customers Portal](https://customers.gitlab.com/customers/sign_in). +1. Select **My account > Account details**. +1. Expand the **Company details** section. +1. Edit the company details. +1. Click **Save changes**. + +### Change your payment method + +Purchases in the Customers Portal require a credit card on record as a payment method. You can add +multiple credit cards to your account, so that purchases for different products are charged to the +correct card. -### Change payment method +If you would like to use an alternative method to pay, please [contact our Sales +team](https://about.gitlab.com/sales/). -To change payment method or update credit card information: +To change your payment method: 1. Log in to the [Customers Portal](https://customers.gitlab.com/customers/sign_in). -1. Select the **My account** drop-down and click on **Payment methods**. -1. **Edit** the existing payment method information or **Add new payment method**. -1. Save changes. +1. Select **My account > Payment methods**. +1. **Edit** an existing payment method's information or **Add new payment method**. +1. Click **Save Changes**. + +#### Set a default payment method + +Automatic renewal of a subscription is charged to your default payment method. To mark a payment +method as the default: + +1. Log in to the [Customers Portal](https://customers.gitlab.com/customers/sign_in). +1. Select **My account > Payment methods**. +1. **Edit** the selected payment method and check the **Make default payment method** checkbox. +1. Click **Save Changes**. ### Change the linked account @@ -196,32 +236,38 @@ To change the password for this customers portal account: ## View your subscription +You can view details of your subscription in either GitLab.com or your self-managed instance: + +- [View your GitLab.com subscription](#view-your-gitlabcom-subscription) +- [View your self-managed subscription](#view-your-self-managed-subscription) + ### View your GitLab.com subscription -To see the status of your GitLab.com subscription, log into GitLab.com and go to the **Billing** section of the relevant namespace: +To see the status of your GitLab.com subscription, log in to GitLab.com and go to the **Billing** section of the relevant namespace: - For individuals: 1. Go to **User Avatar > Settings**. 1. Click **Billing**. - For groups: 1. From the group page (*not* from a project within the group), go to **Settings > Billing**. + You must have Owner level permission to view a group's billing page. The following table describes details of your subscription for groups: -| Field | Description | -| ------ | ------ | -| Seats in subscription | If this is a paid plan, represents the number of seats you've paid to support in your group. | -| Seats currently in use | Number of active seats currently in use. | -| Max seats used | Highest number of seats you've used. If this exceeds the seats in subscription, you may owe an additional fee for the additional users. | -| Seats owed | If your maximum seats used exceeds the seats in your subscription, you'll owe an additional fee for the users you've added. | -| Subscription start date | Date your subscription started. If this is for a Free plan, is the date you transitioned off your group's paid plan. | -| Subscription end date | Date your current subscription will end. Does not apply to Free plans. | +| Field | Description | +|-------------------------|-----------------------------------------------------------------------------------------------------------------------------------------| +| **Seats in subscription** | If this is a paid plan, represents the number of seats you've paid to support in your group. | +| **Seats currently in use** | Number of active seats currently in use. | +| **Max seats used** | Highest number of seats you've used. If this exceeds the seats in subscription, you may owe an additional fee for the additional users. | +| **Seats owed** | If your maximum seats used exceeds the seats in your subscription, you'll owe an additional fee for the users you've added. | +| **Subscription start date** | Date your subscription started. If this is for a Free plan, is the date you transitioned off your group's paid plan. | +| **Subscription end date** | Date your current subscription will end. Does not apply to Free plans. | ### View your self-managed subscription -To view the status of your self-managed subscription, log into the self-managed instance and go to the **License** page. +To view the status of your self-managed subscription, log in to the self-managed instance and go to the **License** page. - 1. Go to **{admin}** **Admin Area**. + 1. Go to **Admin Area**. 1. From the left-hand menu, select **License**. ## Renew your subscription @@ -353,8 +399,7 @@ You can view the exact JSON payload in the administration panel. To view the pay Seat Link is enabled by default. -To disable this feature, go to -**{admin}** **Admin Area > Settings > Metrics and profiling**, uncheck the **Enable Seat Link** checkbox > **Save changes**. +To disable this feature, go to **Admin Area > Settings > Metrics and profiling**, uncheck the **Enable Seat Link** checkbox > **Save changes**. To disable Seat Link in an Omnibus GitLab installation, and prevent it from being configured in the future through the administration panel, set the following in @@ -409,7 +454,7 @@ We recommend following these steps during renewal: 1. Enter the number of [users over license](#users-over-license) in the second box for the user overage incurred in your previous subscription term. TIP: **Tip:** - You can find the _users over license_ in your instance's **Admin** dashboard by clicking on **{admin}** (**Admin Area**) in the top bar, or going to `/admin`. + You can find the _users over license_ in your instance's **Admin** dashboard by clicking on the **Admin Area** in the top bar, or going to `/admin`. The following table describes details of your admin dashboard and renewal terms: @@ -486,8 +531,8 @@ CI pipeline minutes are the execution time for your [pipelines](../ci/pipelines/ Quotas apply to: -- Groups, where the minutes are shared across all members of the group, its subgroups, and nested projects. To view the group's usage, navigate to the group, then **{settings}** **Settings** > **Usage Quotas**. -- Your personal account, where the minutes are available for your personal projects. To view and buy personal minutes, click your avatar, then **{settings}** **Settings** > **[Usage Quotas](https://gitlab.com/profile/usage_quotas#pipelines-quota-tab)**. +- Groups, where the minutes are shared across all members of the group, its subgroups, and nested projects. To view the group's usage, navigate to the group, then **Settings > Usage Quotas**. +- Your personal account, where the minutes are available for your personal projects. To view and buy personal minutes, click your avatar, then **Settings > [Usage Quotas](https://gitlab.com/profile/usage_quotas#pipelines-quota-tab)**. Only pipeline minutes for GitLab shared runners are restricted. If you have a specific runner set up for your projects, there is no limit to your build time on GitLab.com. @@ -507,11 +552,11 @@ main quota. You can find pricing for additional CI/CD minutes in the [GitLab Cus To purchase additional minutes for your group on GitLab.com: -1. From your group, go to **{settings}** **Settings > Usage Quotas**. +1. From your group, go to **Settings > Usage Quotas**. 1. Select **Buy additional minutes** and you will be directed to the Customers Portal. 1. Locate the subscription card that's linked to your group on GitLab.com, click **Buy more CI minutes**, and complete the details about the transaction. 1. Once we have processed your payment, the extra CI minutes will be synced to your group namespace. -1. To confirm the available CI minutes, go to your group, then **{settings}** **Settings > Usage Quotas**. +1. To confirm the available CI minutes, go to your group, then **Settings > Usage Quotas**. The **Additional minutes** displayed now includes the purchased additional CI minutes, plus any minutes rolled over from last month. diff --git a/doc/topics/authentication/index.md b/doc/topics/authentication/index.md index 6ce5e203bbf..9e37ebb699b 100644 --- a/doc/topics/authentication/index.md +++ b/doc/topics/authentication/index.md @@ -44,5 +44,4 @@ This page gathers all the resources for the topic **Authentication** within GitL - [Kanboard Plugin GitLab Authentication](https://github.com/kanboard/plugin-gitlab-auth) - [Jenkins GitLab OAuth Plugin](https://wiki.jenkins.io/display/JENKINS/GitLab+OAuth+Plugin) -- [How to customize GitLab to support OpenID authentication](https://blog.eric.van-der-vlist.com/2013/11/23/how-to-customize-gitlab-to-support-openid-authentication/) - [OKD - Configuring Authentication and User Agent](https://docs.okd.io/3.11/install_config/configuring_authentication.html#GitLab) diff --git a/doc/topics/autodevops/customize.md b/doc/topics/autodevops/customize.md index 2d6da4d322b..b5952494201 100644 --- a/doc/topics/autodevops/customize.md +++ b/doc/topics/autodevops/customize.md @@ -5,7 +5,8 @@ almost everything to fit your needs. Auto DevOps offers everything from custom [buildpacks](#custom-buildpacks), to [Dockerfiles](#custom-dockerfile), and [Helm charts](#custom-helm-chart). You can even copy the complete [CI/CD configuration](#customizing-gitlab-ciyml) into your project to enable -staging and canary deployments, and more. +staging and canary deployments, +[manage Auto DevOps with GitLab APIs](customize.md#extend-auto-devops-with-the-api), and more. ## Custom buildpacks @@ -77,6 +78,16 @@ Avoid passing secrets as Docker build arguments if possible, as they may be persisted in your image. See [this discussion of best practices with secrets](https://github.com/moby/moby/issues/13490) for details. +## Extend Auto DevOps with the API + +You can extend and manage your Auto DevOps configuration with GitLab APIs: + +- [Settings that can be accessed with API calls](../../api/settings.md#list-of-settings-that-can-be-accessed-via-api-calls), + which include `auto_devops_enabled`, to enable Auto DevOps on projects by default. +- [Creating a new project](../../api/projects.md#create-project). +- [Editing groups](../../api/groups.md#update-group). +- [Editing projects](../../api/projects.md#edit-project). + ## Forward CI variables to the build environment > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/25514) in GitLab 12.3, but available in versions 11.9 and above. @@ -381,7 +392,7 @@ these prefixed variables available to the deployed application as environment va To configure your application variables: -1. Go to your project's **{settings}** **Settings > CI/CD**, then expand the +1. Go to your project's **Settings > CI/CD**, then expand the **Variables** section. 1. Create a CI/CD variable, ensuring the key is prefixed with diff --git a/doc/topics/autodevops/index.md b/doc/topics/autodevops/index.md index 01e61095fe2..e8a344a41d7 100644 --- a/doc/topics/autodevops/index.md +++ b/doc/topics/autodevops/index.md @@ -104,8 +104,10 @@ knowledge of the following: - [GitLab Runner](https://docs.gitlab.com/runner/) - [Prometheus](https://prometheus.io/docs/introduction/overview/) -Auto DevOps provides great defaults for all the stages and makes use of [CI templates](https://gitlab.com/gitlab-org/gitlab/-/tree/master/lib/gitlab/ci/templates); you can, however, -[customize](customize.md) almost everything to your needs. +Auto DevOps provides great defaults for all the stages and makes use of +[CI templates](https://gitlab.com/gitlab-org/gitlab/-/tree/master/lib/gitlab/ci/templates). You can, however, +[customize](customize.md) almost everything to your needs, and +[manage Auto DevOps with GitLab APIs](customize.md#extend-auto-devops-with-the-api). For an overview on the creation of Auto DevOps, read more [in this blog post](https://about.gitlab.com/blog/2017/06/29/whats-next-for-gitlab-ci/). @@ -130,7 +132,7 @@ any of the following places: [groups](../../user/group/clusters/index.md#base-domain) - or at the project level as a variable: `KUBE_INGRESS_BASE_DOMAIN` - or at the group level as a variable: `KUBE_INGRESS_BASE_DOMAIN` -- or as an instance-wide fallback in **{admin}** **Admin Area > Settings** under the +- or as an instance-wide fallback in **Admin Area > Settings** under the **Continuous Integration and Delivery** section The base domain variable `KUBE_INGRESS_BASE_DOMAIN` follows the same order of precedence @@ -185,7 +187,7 @@ instance level. If enabling, check that your project does not have a `.gitlab-ci.yml`, or if one exists, remove it. -1. Go to your project's **{settings}** **Settings > CI/CD > Auto DevOps**. +1. Go to your project's **Settings > CI/CD > Auto DevOps**. 1. Select the **Default to Auto DevOps pipeline** checkbox to enable it. 1. (Optional, but recommended) When enabling, you can add in the [base domain](#auto-devops-base-domain) Auto DevOps uses to @@ -207,7 +209,7 @@ is specifically enabled or disabled on the subgroup or project. To enable or disable Auto DevOps at the group level: -1. Go to your group's **{settings}** **Settings > CI/CD > Auto DevOps** page. +1. Go to your group's **Settings > CI/CD > Auto DevOps** page. 1. Select the **Default to Auto DevOps pipeline** checkbox to enable it. 1. Click **Save changes** for the changes to take effect. @@ -216,7 +218,7 @@ To enable or disable Auto DevOps at the group level: Even when disabled at the instance level, group owners and project maintainers can still enable Auto DevOps at the group and project level, respectively. -1. Go to **{admin}** **Admin Area > Settings > Continuous Integration and Deployment**. +1. Go to **Admin Area > Settings > Continuous Integration and Deployment**. 1. Select **Default to Auto DevOps pipeline for all projects** to enable it. 1. (Optional) You can set up the Auto DevOps [base domain](#auto-devops-base-domain), for Auto Deploy and Auto Review Apps to use. @@ -227,7 +229,7 @@ Auto DevOps at the group and project level, respectively. > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/38542) in GitLab 11.0. You can change the deployment strategy used by Auto DevOps by going to your -project's **{settings}** **Settings > CI/CD > Auto DevOps**. The following options +project's **Settings > CI/CD > Auto DevOps**. The following options are available: - **Continuous deployment to production**: Enables [Auto Deploy](stages.md#auto-deploy) @@ -276,14 +278,14 @@ The following table is an example of how to configure the three different cluste To add a different cluster for each environment: -1. Navigate to your project's **{cloud-gear}** **Operations > Kubernetes**. +1. Navigate to your project's **Operations > Kubernetes**. 1. Create the Kubernetes clusters with their respective environment scope, as described from the table above. 1. After creating the clusters, navigate to each cluster and install Ingress. Wait for the Ingress IP address to be assigned. 1. Make sure you've [configured your DNS](#auto-devops-base-domain) with the specified Auto DevOps domains. -1. Navigate to each cluster's page, through **{cloud-gear}** **Operations > Kubernetes**, +1. Navigate to each cluster's page, through **Operations > Kubernetes**, and add the domain based on its Ingress IP address. After completing configuration, you can test your setup by creating a merge request diff --git a/doc/topics/autodevops/quick_start_guide.md b/doc/topics/autodevops/quick_start_guide.md index 4f8074f047e..6b9b461e76e 100644 --- a/doc/topics/autodevops/quick_start_guide.md +++ b/doc/topics/autodevops/quick_start_guide.md @@ -57,7 +57,7 @@ to deploy this project to. ## Create a Kubernetes cluster from within GitLab 1. On your project's landing page, click **Add Kubernetes cluster** - (note that this option is also available when you navigate to **{cloud-gear}** **Operations > Kubernetes**). + (note that this option is also available when you navigate to **Operations > Kubernetes**).  @@ -127,7 +127,7 @@ While Auto DevOps is enabled by default, Auto DevOps can be disabled at both the instance level (for self-managed instances) and the group level. Complete these steps to enable Auto DevOps if it's disabled: -1. Navigate to **{settings}** **Settings > CI/CD > Auto DevOps**, and click **Expand**. +1. Navigate to **Settings > CI/CD > Auto DevOps**, and click **Expand**. 1. Select **Default to Auto DevOps pipeline** to display more options. 1. In **Deployment strategy**, select your desired [continuous deployment strategy](index.md#deployment-strategy) to deploy the application to production after the pipeline successfully runs on the `master` branch. @@ -194,7 +194,7 @@ to monitor it. After successfully deploying your application, you can view its website and check on its health on the **Environments** page by navigating to -**{cloud-gear}** **Operations > Environments**. This page displays details about +**Operations > Environments**. This page displays details about the deployed applications, and the right-hand column displays icons that link you to common environment tasks: @@ -221,7 +221,7 @@ takes you to the pod's logs page. TIP: **Tip:** The example shows only one pod hosting the application at the moment, but you can add more pods by defining the [`REPLICAS` variable](customize.md#environment-variables) -in **{settings}** **Settings > CI/CD > Environment variables**. +in **Settings > CI/CD > Environment variables**. ### Work with branches diff --git a/doc/topics/autodevops/stages.md b/doc/topics/autodevops/stages.md index bf1594130f4..7c6cf043820 100644 --- a/doc/topics/autodevops/stages.md +++ b/doc/topics/autodevops/stages.md @@ -90,6 +90,12 @@ Check the [currently supported languages](#currently-supported-languages). Auto Test uses tests you already have in your application. If there are no tests, it's up to you to add them. +NOTE: **Note:** +Not all buildpacks supported by [Auto Build](#auto-build) are supported by Auto Test. +Auto Test uses [Herokuish](https://gitlab.com/gitlab-org/gitlab/-/issues/212689), *not* +Cloud Native Buildpacks, and only buildpacks that implement the +[Testpack API](https://devcenter.heroku.com/articles/testpack-api) are supported. + ### Currently supported languages Note that not all buildpacks support Auto Test yet, as it's a relatively new @@ -454,8 +460,9 @@ For example, in a Rails application in an image built with Unless your repository contains a `Dockerfile`, your image is built with Herokuish, and you must prefix commands run in these images with -`/bin/herokuish procfile exec` to replicate the environment where your application -will run. +`/bin/herokuish procfile exec` (for Herokuish) or `/cnb/lifecycle/launcher` +(for Cloud Native Buildpacks) to replicate the environment where your +application runs. ### Upgrade auto-deploy-app Chart @@ -616,6 +623,12 @@ For example, to start a Rails console from the application root directory, run: /bin/herokuish procfile exec bin/rails c ``` +When using Cloud Native Buildpacks, instead of `/bin/herokuish procfile exec`, use + +```shell +/cnb/lifecycle/launcher $COMMAND +``` + ## Auto Monitoring After your application deploys, Auto Monitoring helps you monitor @@ -644,6 +657,6 @@ To use Auto Monitoring: 1. After the pipeline finishes successfully, open the [monitoring dashboard for a deployed environment](../../ci/environments/index.md#monitoring-environments) to view the metrics of your deployed application. To view the metrics of the - whole Kubernetes cluster, navigate to **{cloud-gear}** **Operations > Metrics**. + whole Kubernetes cluster, navigate to **Operations > Metrics**.  diff --git a/doc/topics/cron/index.md b/doc/topics/cron/index.md new file mode 100644 index 00000000000..a3dd3b77c22 --- /dev/null +++ b/doc/topics/cron/index.md @@ -0,0 +1,63 @@ +# Cron + +Cron syntax is used to schedule when jobs should run. + +You may need to use a cron syntax string to +[trigger nightly pipelines](../../ci/triggers/README.md#using-cron-to-trigger-nightly-pipelines), +create a [pipeline schedule](../../api/pipeline_schedules.md#create-a-new-pipeline-schedule), +or to prevent unintentional releases by setting a +[deploy freeze](../../user/project/releases/index.md#prevent-unintentional-releases-by-setting-a-deploy-freeze). + +## Cron syntax + +Cron scheduling uses a series of five numbers, separated by spaces: + +```plaintext +# ┌───────────── minute (0 - 59) +# │ ┌───────────── hour (0 - 23) +# │ │ ┌───────────── day of the month (1 - 31) +# │ │ │ ┌───────────── month (1 - 12) +# │ │ │ │ ┌───────────── day of the week (0 - 6) (Sunday to Saturday) +# │ │ │ │ │ +# │ │ │ │ │ +# │ │ │ │ │ +# * * * * * <command to execute> +``` + +[Source: [Wikipedia](https://en.wikipedia.org/wiki/Cron)] + +In cron syntax, the asterisk (`*`) means 'every,' so the following cron strings +are valid: + +- Run once an hour at the beginning of the hour: `0 * * * *` +- Run once a day at midnight: `0 0 * * *` +- Run once a week at midnight on Sunday morning: `0 0 * * 0` +- Run once a month at midnight of the first day of the month: `0 0 1 * *` +- Run once a year at midnight of 1 January: `0 0 1 1 *` + +For complete cron documentation, refer to the +[crontab(5) — Linux manual page](https://man7.org/linux/man-pages/man5/crontab.5.html). +This documentation is accessible offline by entering `man 5 crontab` in a Linux or MacOS +terminal. + +## Cron examples + +```plaintext +# Run at 7:00pm every day: +0 19 * * * + +# Run every minute on the 10th of June: +* * 3 6 * + +# Run at 06:30 every Friday: +30 6 * * 5 +``` + +More examples of how to write a cron schedule can be found at +[crontab.guru](https://crontab.guru/examples.html). + +## How GitLab parses cron syntax strings + +GitLab uses [fugit](https://github.com/floraison/fugit) to parse cron syntax +strings on the server and [cron-validate](https://github.com/Airfooox/cron-validate) +to validate cron syntax in the browser. diff --git a/doc/topics/git/feature_branch_development.md b/doc/topics/git/feature_branch_development.md index ab3adf54dd7..762eddbc130 100644 --- a/doc/topics/git/feature_branch_development.md +++ b/doc/topics/git/feature_branch_development.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" type: how-tos --- diff --git a/doc/topics/git/how_to_install_git/index.md b/doc/topics/git/how_to_install_git/index.md index 75ea6183a32..7b842b6a409 100644 --- a/doc/topics/git/how_to_install_git/index.md +++ b/doc/topics/git/how_to_install_git/index.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" author: Sean Packham author_gitlab: SeanPackham level: beginner diff --git a/doc/topics/git/img/create_merge_request_v13_1.png b/doc/topics/git/img/create_merge_request_v13_1.png Binary files differindex a725149f6a2..d59cfc74290 100644 --- a/doc/topics/git/img/create_merge_request_v13_1.png +++ b/doc/topics/git/img/create_merge_request_v13_1.png diff --git a/doc/topics/git/img/modify_branches_v13_1.png b/doc/topics/git/img/modify_branches_v13_1.png Binary files differindex dc517dd249f..781f54fc3c0 100644 --- a/doc/topics/git/img/modify_branches_v13_1.png +++ b/doc/topics/git/img/modify_branches_v13_1.png diff --git a/doc/topics/git/index.md b/doc/topics/git/index.md index 89da3dfdbd0..92181fb7bb0 100644 --- a/doc/topics/git/index.md +++ b/doc/topics/git/index.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" type: index --- diff --git a/doc/topics/git/lfs/index.md b/doc/topics/git/lfs/index.md index 1e2f45fd67b..5875cdd4ca1 100644 --- a/doc/topics/git/lfs/index.md +++ b/doc/topics/git/lfs/index.md @@ -1,4 +1,8 @@ --- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference, howto disqus_identifier: 'https://docs.gitlab.com/ee/workflow/lfs/lfs/index.html' --- diff --git a/doc/topics/git/lfs/migrate_from_git_annex_to_git_lfs.md b/doc/topics/git/lfs/migrate_from_git_annex_to_git_lfs.md index 09087fcae13..c62b7e1cc12 100644 --- a/doc/topics/git/lfs/migrate_from_git_annex_to_git_lfs.md +++ b/doc/topics/git/lfs/migrate_from_git_annex_to_git_lfs.md @@ -1,6 +1,13 @@ +--- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference, howto +--- + # Migration guide from Git Annex to Git LFS ->**Note:** +NOTE: **Note:** Git Annex support [has been removed](https://gitlab.com/gitlab-org/gitlab/-/issues/1648) in GitLab Enterprise Edition 9.0 (2017/03/22). @@ -30,7 +37,7 @@ ones that GitLab developed. ## Migration steps ->**Note:** +NOTE: **Note:** Since Git Annex files are stored in a sub-directory of the normal repositories (`.git/annex/objects`) and LFS files are stored outside of the repositories, they are not compatible as they are using a different scheme. Therefore, the diff --git a/doc/topics/git/lfs/migrate_to_git_lfs.md b/doc/topics/git/lfs/migrate_to_git_lfs.md index 3e287c0816d..944f4d8f78d 100644 --- a/doc/topics/git/lfs/migrate_to_git_lfs.md +++ b/doc/topics/git/lfs/migrate_to_git_lfs.md @@ -96,9 +96,10 @@ Consider an example upstream project, `git@gitlab.com:gitlab-tests/test-git-lfs- 1. Clean up the repository: ```shell - # cd path/to/mirror/repo: + # Change into the mirror repo directory: cd test-git-lfs-repo-migration.git - # clean up the repo: + + # Clean up the repo: git reflog expire --expire=now --all && git gc --prune=now --aggressive ``` @@ -128,12 +129,23 @@ Consider an example upstream project, `git@gitlab.com:gitlab-tests/test-git-lfs- 1. Track the files you want with LFS: ```shell - # cd path/to/upstream/repo: + # Change into the /tmp directory + cd /tmp + + # Clone the repo + git clone git@gitlab.com:gitlab-tests/test-git-lfs-repo-migration.git + + # Change into the upstream repo directory: cd test-git-lfs-repo-migration + # You may need to reset your local copy with upstream's `master` after force-pushing from the mirror: git reset --hard origin/master + # Track the files with LFS: git lfs track "*.gif" "*.png" "*.jpg" "*.psd" "*.mp4" "img/" + + # Push up changes to .gitattributes + git add .gitattributes && git commit -m 'Track .gif,.png,.jpg,.psd,.mp4 and img/' && git push ``` Now all existing the files you converted, as well as the new diff --git a/doc/topics/git/numerous_undo_possibilities_in_git/index.md b/doc/topics/git/numerous_undo_possibilities_in_git/index.md index fdf86d8f646..285ab133196 100644 --- a/doc/topics/git/numerous_undo_possibilities_in_git/index.md +++ b/doc/topics/git/numerous_undo_possibilities_in_git/index.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" author: Crt Mori author_gitlab: Letme level: intermediary diff --git a/doc/topics/git/partial_clone.md b/doc/topics/git/partial_clone.md index de25c8a3283..c976eda688a 100644 --- a/doc/topics/git/partial_clone.md +++ b/doc/topics/git/partial_clone.md @@ -1,3 +1,10 @@ +--- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference, howto +--- + # Partial Clone As Git repositories grow in size, they can become cumbersome to work with @@ -182,3 +189,47 @@ For more details, see the Git documentation for # Checkout master git checkout master ``` + +## Remove partial clone filtering + +Git repositories with partial clone filtering can have the filtering removed. To +remove filtering: + +1. Fetch everything that has been excluded by the filters, to make sure that the + repository is complete. If `git sparse-checkout` was used, use + `git sparse-checkout disable` to disable it. See the + [`disable` documentation](https://git-scm.com/docs/git-sparse-checkout#Documentation/git-sparse-checkout.txt-emdisableem) + for more information. + + Then do a regular `fetch` to ensure that the repository is complete. To check if + there are missing objects to fetch, and then fetch them, especially when not using + `git sparse-checkout`, the following commands can be used: + + ```shell + # Show missing objects + git rev-list --objects --all --missing=print | grep -e '^\?' + + # Show missing objects without a '?' character before them (needs GNU grep) + git rev-list --objects --all --missing=print | grep -oP '^\?\K\w+' + + # Fetch missing objects + git fetch origin $(git rev-list --objects --all --missing=print | grep -oP '^\?\K\w+') + + # Show number of missing objects + git rev-list --objects --all --missing=print | grep -e '^\?' | wc -l + ``` + +1. Repack everything. This can be done using `git repack -a -d`, for example. This + should leave only three files in `.git/objects/pack/`: + - A `pack-<SHA1>.pack` file. + - Its corresponding `pack-<SHA1>.idx` file. + - A `pack-<SHA1>.promisor` file. + +1. Delete the `.promisor` file. The above step should have left only one + `pack-<SHA1>.promisor` file, which should be empty and should be deleted. + +1. Remove partial clone configuration. The partial clone-related configuration + variables should be removed from Git config files. Usually only the following + configuration must be removed: + - `remote.origin.promisor`. + - `remote.origin.partialclonefilter`. diff --git a/doc/topics/git/troubleshooting_git.md b/doc/topics/git/troubleshooting_git.md index 010a811bd33..523718e4133 100644 --- a/doc/topics/git/troubleshooting_git.md +++ b/doc/topics/git/troubleshooting_git.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" type: howto --- diff --git a/doc/topics/git/useful_git_commands.md b/doc/topics/git/useful_git_commands.md index 711567dbd06..0bf6075a1ea 100644 --- a/doc/topics/git/useful_git_commands.md +++ b/doc/topics/git/useful_git_commands.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" type: reference --- diff --git a/doc/topics/gitlab_flow.md b/doc/topics/gitlab_flow.md index 04c942ab532..339da40b29d 100644 --- a/doc/topics/gitlab_flow.md +++ b/doc/topics/gitlab_flow.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" disqus_identifier: 'https://docs.gitlab.com/ee/workflow/gitlab_flow.html' --- @@ -266,11 +269,11 @@ One option is to use continuous integration (CI) to merge in `master` at the sta Another option is to only merge in from well-defined points in time, for example, a tagged release. You could also use [feature toggles](https://martinfowler.com/bliki/FeatureToggle.html) to hide incomplete features so you can still merge back into `master` every day. -> **Note:** Don't confuse automatic branch testing with continuous integration. -> Martin Fowler makes this distinction in [his article about feature branches](https://martinfowler.com/bliki/FeatureBranch.html): -> -> "I've heard people say they are doing CI because they are running builds, perhaps using a CI server, on every branch with every commit. -> That's continuous building, and a Good Thing, but there's no *integration*, so it's not CI." +NOTE: **Note:** +Don't confuse automatic branch testing with continuous integration. +Martin Fowler makes this distinction in [his article about feature branches](https://martinfowler.com/bliki/FeatureBranch.html): +"I've heard people say they are doing CI because they are running builds, perhaps using a CI server, on every branch with every commit. +That's continuous building, and a Good Thing, but there's no *integration*, so it's not CI." In conclusion, you should try to prevent merge commits, but not eliminate them. Your codebase should be clean, but your history should represent what actually happened. diff --git a/doc/topics/index.md b/doc/topics/index.md index 634dd70613a..3b92bbb55cb 100644 --- a/doc/topics/index.md +++ b/doc/topics/index.md @@ -10,10 +10,12 @@ tutorials, technical overviews, blog posts) and videos. - [Auto DevOps](autodevops/index.md) - [Authentication](authentication/index.md) - [Continuous Integration (GitLab CI/CD)](../ci/README.md) +- [Cron](cron/index.md) - [Git](git/index.md) - [GitLab Flow](gitlab_flow.md) - [GitLab Installation](../install/README.md) - [GitLab Pages](../user/project/pages/index.md) - [Offline GitLab](offline/index.md) ->**Note:** More topics will be available soon. +NOTE: **Note:** +More topics will be available soon. diff --git a/doc/topics/offline/index.md b/doc/topics/offline/index.md index 6d4c486d350..a8366659c22 100644 --- a/doc/topics/offline/index.md +++ b/doc/topics/offline/index.md @@ -13,126 +13,3 @@ If you plan to deploy a GitLab instance on a physically-isolated and offline net Follow these best practices to use GitLab's features in an offline environment: - [Operating the GitLab Secure scanners in an offline environment](../../user/application_security/offline_deployments/index.md). - -## Loading Docker images onto your offline host - -To use many GitLab features, including -[security scans](../../user/application_security/index.md#working-in-an-offline-environment) -and [Auto DevOps](../autodevops/), the GitLab Runner must be able to fetch the -relevant Docker images. - -The process for making these images available without direct access to the public internet -involves downloading the images then packaging and transferring them to the offline host. Here's an -example of such a transfer: - -1. Download Docker images from public internet. -1. Package Docker images as tar archives. -1. Transfer images to offline environment. -1. Load transferred images into offline Docker registry. - -### Using the official GitLab template - -GitLab provides a [vendored template](../../ci/yaml/README.md#includetemplate) -to ease this process. - -This template should be used in a new, empty project, with a `gitlab-ci.yml` file containing: - -```yaml -include: - - template: Secure-Binaries.gitlab-ci.yml -``` - -The pipeline downloads the Docker images needed for the Security Scanners and saves them as -[job artifacts](../../ci/pipelines/job_artifacts.md) or pushes them to the [Container Registry](../../user/packages/container_registry/index.md) -of the project where the pipeline is executed. These archives can be transferred to another location -and [loaded](https://docs.docker.com/engine/reference/commandline/load/) in a Docker daemon. -This method requires a GitLab Runner with access to both `gitlab.com` (including -`registry.gitlab.com`) and the local offline instance. This runner must run in -[privileged mode](https://docs.gitlab.com/runner/executors/docker.html#use-docker-in-docker-with-privileged-mode) -to be able to use the `docker` command inside the jobs. This runner can be installed in a DMZ or on -a bastion, and used only for this specific project. - -#### Scheduling the updates - -By default, this project's pipeline will run only once, when the `.gitlab-ci.yml` is added to the -repo. To update the GitLab security scanners and signatures, it's necessary to run this pipeline -regularly. GitLab provides a way to [schedule pipelines](../../ci/pipelines/schedules.md). For -example, you can set this up to download and store the Docker images every week. - -Some images can be updated more frequently than others. For example, the [vulnerability database](https://hub.docker.com/r/arminc/clair-db/tags) -for Container Scanning is updated daily. To update this single image, create a new Scheduled -Pipeline that runs daily and set `SECURE_BINARIES_ANALYZERS` to `clair-vulnerabilities-db`. Only -this job will be triggered, and the image will be updated daily and made available in the project -registry. - -#### Using the secure bundle created - -The project using the `Secure-Binaries.gitlab-ci.yml` template should now host all the required -images and resources needed to run GitLab Security features. - -Next, you must tell the offline instance to use these resources instead of the default ones on -GitLab.com. To do so, set the environment variable `SECURE_ANALYZERS_PREFIX` with the URL of the -project [container registry](../../user/packages/container_registry/index.md). - -You can set this variable in the projects' `.gitlab-ci.yml`, or -in the GitLab UI at the project or group level. See the [GitLab CI/CD environment variables page](../../ci/variables/README.md#custom-environment-variables) -for more information. - -#### Variables - -The following table shows which variables you can use with the `Secure-Binaries.gitlab-ci.yml` -template: - -| VARIABLE | Description | Default value | -|-------------------------------------------|-----------------------------------------------|-----------------------------------| -| `SECURE_BINARIES_ANALYZERS` | Comma-separated list of analyzers to download | `"bandit, brakeman, gosec, and so on..."` | -| `SECURE_BINARIES_DOWNLOAD_IMAGES` | Used to disable jobs | `"true"` | -| `SECURE_BINARIES_PUSH_IMAGES` | Push files to the project registry | `"true"` | -| `SECURE_BINARIES_SAVE_ARTIFACTS` | Also save image archives as artifacts | `"false"` | -| `SECURE_BINARIES_ANALYZER_VERSION` | Default analyzer version (Docker tag) | `"2"` | - -### Alternate way without the official template - -If it's not possible to follow the above method, the images can be transferred manually instead: - -#### Example image packager script - -```shell -#!/bin/bash -set -ux - -# Specify needed analyzer images -analyzers=${SAST_ANALYZERS:-"bandit eslint gosec"} -gitlab=registry.gitlab.com/gitlab-org/security-products/analyzers/ - -for i in "${analyzers[@]}" -do - tarname="${i}_2.tar" - docker pull $gitlab$i:2 - docker save $gitlab$i:2 -o ./analyzers/${tarname} - chmod +r ./analyzers/${tarname} -done -``` - -#### Example image loader script - -This example loads the images from a bastion host to an offline host. In certain configurations, -physical media may be needed for such a transfer: - -```shell -#!/bin/bash -set -ux - -# Specify needed analyzer images -analyzers=${SAST_ANALYZERS:-"bandit eslint gosec"} -registry=$GITLAB_HOST:4567 - -for i in "${analyzers[@]}" -do - tarname="${i}_2.tar" - scp ./analyzers/${tarname} ${GITLAB_HOST}:~/${tarname} - ssh $GITLAB_HOST "sudo docker load -i ${tarname}" - ssh $GITLAB_HOST "sudo docker tag $(sudo docker images | grep $i | awk '{print $3}') ${registry}/analyzers/${i}:2" - ssh $GITLAB_HOST "sudo docker push ${registry}/analyzers/${i}:2" -done -``` diff --git a/doc/topics/offline/quick_start_guide.md b/doc/topics/offline/quick_start_guide.md index 0abdd08ffcf..817d7d31180 100644 --- a/doc/topics/offline/quick_start_guide.md +++ b/doc/topics/offline/quick_start_guide.md @@ -34,7 +34,7 @@ Follow these steps to enable SSL for your fresh instance. Note that these steps ```ruby # Update external_url from "http" to "https" - external_url "https://example.gitlab.com" + external_url "https://gitlab.example.com" # Set Let's Encrypt to false letsencrypt['enable'] = false @@ -64,8 +64,8 @@ Follow these steps to enable the container registry. Note that these steps refle ```ruby # Change external_registry_url to match external_url, but append the port 4567 - external_url "https://example.gitlab.com" - registry_external_url "https://example.gitlab.com:4567" + external_url "https://gitlab.example.com" + registry_external_url "https://gitlab.example.com:4567" ``` 1. Reconfigure your instance to apply the changes: diff --git a/doc/university/bookclub/booklist.md b/doc/university/bookclub/booklist.md index 33298e45393..c0251229916 100644 --- a/doc/university/bookclub/booklist.md +++ b/doc/university/bookclub/booklist.md @@ -1,118 +1,5 @@ --- -comments: false -type: index +redirect_to: 'https://docs.gitlab.com' --- -# Books - -List of books and resources that may be worth reading. - -## Papers - -1. **The Humble Programmer** - - Edsger W. Dijkstra, 1972 ([paper](https://dl.acm.org/citation.cfm?id=361591)) - -## Programming - -1. **Design Patterns: Elements of Reusable Object-Oriented Software** - - Erich Gamma, Richard Helm, Ralph Johnson, John Vlissides, 1994 ([amazon](https://www.amazon.com/Design-Patterns-Elements-Reusable-Object-Oriented/dp/0201633612)) - -1. **Clean Code: A Handbook of Agile Software Craftsmanship** - - Robert C. "Uncle Bob" Martin, 2008 ([amazon](https://www.amazon.com/Clean-Code-Handbook-Software-Craftsmanship/dp/0132350882)) - -1. **Code Complete: A Practical Handbook of Software Construction**, 2nd Edition - - Steve McConnell, 2004 ([amazon](https://www.amazon.com/Code-Complete-Practical-Handbook-Construction/dp/0735619670)) - -1. **The Pragmatic Programmer: From Journeyman to Master** - - Andrew Hunt, David Thomas, 1999 ([amazon](https://www.amazon.com/Pragmatic-Programmer-Journeyman-Master/dp/020161622X)) - -1. **Working Effectively with Legacy Code** - - Michael Feathers, 2004 ([amazon](https://www.amazon.com/Working-Effectively-Legacy-Michael-Feathers/dp/0131177052)) - -1. **Eloquent Ruby** - - Russ Olsen, 2011 ([amazon](https://www.amazon.com/Eloquent-Ruby-Addison-Wesley-Professional/dp/0321584104)) - -1. **Domain-Driven Design: Tackling Complexity in the Heart of Software** - - Eric Evans, 2003 ([amazon](https://www.amazon.com/Domain-Driven-Design-Tackling-Complexity-Software/dp/0321125215)) - -1. **How to Solve It: A New Aspect of Mathematical Method** - - Polya G. 1957 ([amazon](https://www.amazon.com/How-Solve-Mathematical-Princeton-Science/dp/069116407X)) - -1. **Software Creativity 2.0** - - Robert L. Glass, 2006 ([amazon](https://www.amazon.com/Software-Creativity-2-0-Robert-Glass/dp/0977213315)) - -1. **Object-Oriented Software Construction** - - Bertrand Meyer, 1997 ([amazon](https://www.amazon.com/Object-Oriented-Software-Construction-Book-CD-ROM/dp/0136291554)) - -1. **Refactoring: Improving the Design of Existing Code** - - Martin Fowler, Kent Beck, 1999 ([amazon](https://www.amazon.com/Refactoring-Improving-Design-Existing-Code/dp/0201485672)) - -1. **Test Driven Development: By Example** - - Kent Beck, 2002 ([amazon](https://www.amazon.com/Test-Driven-Development-Kent-Beck/dp/0321146530)) - -1. **Algorithms in C++: Fundamentals, Data Structure, Sorting, Searching** - - Robert Sedgewick, 1990 ([amazon](https://www.amazon.com/Algorithms-Parts-1-4-Fundamentals-Structure/dp/0201350882)) - -1. **Effective C++** - - Scott Mayers, 1996 ([amazon](https://www.amazon.com/Effective-Specific-Improve-Programs-Designs/dp/0321334876)) - -1. **Extreme Programming Explained: Embrace Change** - - Kent Beck, 1999 ([amazon](https://www.amazon.com/Extreme-Programming-Explained-Embrace-Change/dp/0321278658)) - -1. **The Art of Computer Programming** - - Donald E. Knuth, 1997 ([amazon](https://www.amazon.com/Computer-Programming-Volumes-1-4A-Boxed/dp/0321751043)) - -1. **Writing Efficient Programs** - - Jon Louis Bentley, 1982 ([amazon](https://www.amazon.com/Writing-Efficient-Programs-Prentice-Hall-Software/dp/013970244X)) - -1. **The Mythical Man-Month: Essays on Software Engineering** - - Frederick Phillips Brooks, 1975 ([amazon](https://www.amazon.com/Mythical-Man-Month-Essays-Software-Engineering/dp/0201006502)) - -1. **Peopleware: Productive Projects and Teams** 3rd Edition - - Tom DeMarco, Tim Lister, 2013 ([amazon](https://www.amazon.com/Peopleware-Productive-Projects-Teams-3rd/dp/0321934113)) - -1. **Principles Of Software Engineering Management** - - Tom Gilb, 1988 ([amazon](https://www.amazon.com/Principles-Software-Engineering-Management-Gilb/dp/0201192462)) - -## Other - -1. **Thinking, Fast and Slow** - - Daniel Kahneman, 2013 ([amazon](https://www.amazon.com/Thinking-Fast-Slow-Daniel-Kahneman/dp/0374533555)) - -1. **The Social Animal** 11th Edition - - Elliot Aronson, 2011 ([amazon](https://www.amazon.com/Social-Animal-Elliot-Aronson/dp/1429233419)) - -1. **Influence: Science and Practice** 5th Edition - - Robert B. Cialdini, 2008 ([amazon](https://www.amazon.com/Influence-Practice-Robert-B-Cialdini/dp/0205609996)) - -1. **Getting to Yes: Negotiating Agreement Without Giving In** - - Roger Fisher, William L. Ury, Bruce Patton, 2011 ([amazon](https://www.amazon.com/Getting-Yes-Negotiating-Agreement-Without/dp/0143118757)) - -1. **How to Win Friends & Influence People** - - Dale Carnegie, 1981 ([amazon](https://www.amazon.com/How-Win-Friends-Influence-People/dp/0671027034)) +Visit our [documentation page](https://docs.gitlab.com) for information about GitLab. diff --git a/doc/university/bookclub/index.md b/doc/university/bookclub/index.md index 71dfe7fc3cb..c0251229916 100644 --- a/doc/university/bookclub/index.md +++ b/doc/university/bookclub/index.md @@ -1,24 +1,5 @@ --- -comments: false -type: index +redirect_to: 'https://docs.gitlab.com' --- -# The GitLab Book Club - -The Book Club is a casual meet-up to read and discuss books we like. -We'll find a time that suits most, if not all. - -See the [book list](booklist.md) for additional recommendations. - -## Currently reading : Books about remote work - -1. **Remote: Office not required** - - David Heinemeier Hansson and Jason Fried, 2013 - ([Amazon](https://www.amazon.co.uk/dp/0091954673/ref=cm_sw_r_tw_dp_x_0yy9EbZ2WXJ6Y)) - -1. **The Year Without Pants** - - Scott Berkun, 2013 ([ScottBerkun.com](https://scottberkun.com/yearwithoutpants/)) - -Any other books you'd like to suggest? Edit this page and add them to the queue. +Visit our [documentation page](https://docs.gitlab.com) for information about GitLab. diff --git a/doc/university/glossary/README.md b/doc/university/glossary/README.md index 297b841b283..c0251229916 100644 --- a/doc/university/glossary/README.md +++ b/doc/university/glossary/README.md @@ -1,11 +1,5 @@ --- -comments: false +redirect_to: 'https://docs.gitlab.com' --- -# Glossary - -This page has been removed after an effort to ensure that all applicable GitLab-specific -terms are available in context on the relevant [GitLab Documentation](https://docs.gitlab.com/) -or <https://about.gitlab.com/> pages. - -If you are looking for a definition of a specific term, please search these sites. +Visit our [documentation page](https://docs.gitlab.com) for information about GitLab. diff --git a/doc/university/process/README.md b/doc/university/process/README.md index 0b7fa467230..c0251229916 100644 --- a/doc/university/process/README.md +++ b/doc/university/process/README.md @@ -1,30 +1,5 @@ --- -comments: false +redirect_to: 'https://docs.gitlab.com' --- -# Suggesting improvements - -If you would like to teach a class or participate or help in any way please -submit a merge request and assign it to [Job](https://gitlab.com/JobV). - -If you have suggestions for additional courses you would like to see, -please submit a merge request to add an upcoming class, assign to -[Chad](https://gitlab.com/chadmalchow) and /cc [Job](https://gitlab.com/JobV). - -## Adding classes - -1. All training materials of any kind should be added to [GitLab CE](https://gitlab.com/gitlab-org/gitlab-foss/) - to ensure they are available to a broad audience (don't use any other repo or - storage for training materials). -1. Don't make materials that are needlessly specific to one group of people, try - to keep the wording broad and inclusive (don't make things for only GitLab Inc. - people, only interns, only customers, etc.). -1. To allow people to contribute all content should be in Git. -1. The content can go in a subdirectory under `/doc/university/`. -1. To make, view or modify the slides of the classes use [Deckset](https://www.deckset.com) - or [RevealJS](https://revealjs.com/#/). Do not use PowerPoint or Google - Slides since this prevents everyone from contributing. -1. Please upload any video recordings to our YouTube channel. We prefer them to - be public, if needed they can be unlisted but if so they should be linked from - this page. -1. Please create a merge request and assign to [Erica](https://gitlab.com/Erica). +Visit our [documentation page](https://docs.gitlab.com) for information about GitLab. diff --git a/doc/university/training/end-user/README.md b/doc/university/training/end-user/README.md index 8d25b865855..c0251229916 100644 --- a/doc/university/training/end-user/README.md +++ b/doc/university/training/end-user/README.md @@ -1,370 +1,5 @@ --- -comments: false +redirect_to: 'https://docs.gitlab.com' --- -# Training - -This training material is the Markdown used to generate training slides -which can be found at [End User Slides](https://gitlab-org.gitlab.io/end-user-training-slides/#/) -through it's [RevealJS](https://gitlab.com/gitlab-org/end-user-training-slides) -project. - -## Git Intro - -### What is a Version Control System (VCS) - -- Records changes to a file -- Maintains history of changes -- Disaster Recovery -- Types of VCS: Local, Centralized and Distributed - -### Short Story of Git - -- 1991-2002: The Linux kernel was being maintained by sharing archived files - and patches. -- 2002: The Linux kernel project began using a DVCS called BitKeeper -- 2005: BitKeeper revoked the free-of-charge status and Git was created - -### What is Git - -- Distributed Version Control System -- Great branching model that adapts well to most workflows -- Fast and reliable -- Keeps a complete history -- Disaster recovery friendly -- Open Source - -### Getting Help - -- Use the tools at your disposal when you get stuck. - - Use `git help <command>` command - - Use Google (i.e. StackOverflow, Google groups) - - Read documentation at <https://git-scm.com> - -## Git Setup - -Workshop Time! - -### Setup - -- Windows: Install 'Git for Windows' - - <https://gitforwindows.org> -- Mac: Type `git` in the Terminal application. - - If it's not installed, it will prompt you to install it. -- Linux - - Debian: `sudo apt-get install git-all` - - Red Hat `sudo yum install git-all` - -### Configure - -- One-time configuration of the Git client: - -```shell -git config --global user.name "Your Name" -git config --global user.email you@example.com -``` - -- If you don't use the global flag you can set up a different author for - each project -- Check settings with: - -```shell -git config --global --list -``` - -- You might want or be required to use an SSH key. - - Instructions: [SSH](http://doc.gitlab.com/ce/ssh/README.html) - -### Workspace - -- Choose a directory on you machine easy to access -- Create a workspace or development directory -- This is where we'll be working and adding content - -```shell -mkdir ~/development -cd ~/development - --or- - -mkdir ~/workspace -cd ~/workspace -``` - -## Git Basics - -### Git Workflow - -- Untracked files - - New files that Git has not been told to track previously. -- Working area (Workspace) - - Files that have been modified but are not committed. -- Staging area (Index) - - Modified files that have been marked to go in the next commit. -- Upstream - - Hosted repository on a shared server - -### GitLab - -- GitLab is an application to code, test and deploy. -- Provides repository management with access controls, code reviews, - issue tracking, Merge Requests, and other features. -- The hosted version of GitLab is <https://gitlab.com> - -### New Project - -- Sign in into your <https://gitlab.com> account -- Create a project -- Choose to import from 'Any Repo by URL' and use <https://gitlab.com/gitlab-org/training-examples.git> -- On your machine clone the `training-examples` project - -### Git and GitLab basics - -1. Edit `edit_this_file.rb` in `training-examples` -1. See it listed as a changed file (working area) -1. View the differences -1. Stage the file -1. Commit -1. Push the commit to the remote -1. View the Git log - -```shell -# Edit `edit_this_file.rb` -git status -git diff -git add <file> -git commit -m 'My change' -git push origin master -git log -``` - -### Feature Branching - -1. Create a new feature branch called `squash_some_bugs` -1. Edit `bugs.rb` and remove all the bugs. -1. Commit -1. Push - -```shell -git checkout -b squash_some_bugs -# Edit `bugs.rb` -git status -git add bugs.rb -git commit -m 'Fix some buggy code' -git push origin squash_some_bugs -``` - -## Merge Request - -- When you want feedback create a merge request -- Target is the ‘default’ branch (usually master) -- Assign or mention the person you would like to review -- Add `Draft:` to the title if it's a work in progress -- When accepting, always delete the branch -- Anyone can comment, not just the assignee -- Push corrections to the same branch - -### Merge request example - -- Create your first merge request - - Use the blue button in the activity feed - - View the diff (changes) and leave a comment - - Push a new commit to the same branch - - Review the changes again and notice the update - -### Feedback and Collaboration - -- Merge requests are a time for feedback and collaboration -- Giving feedback is hard -- Be as kind as possible -- Receiving feedback is hard -- Be as receptive as possible -- Feedback is about the best code, not the person. You are not your code -- Feedback and Collaboration - ---- - -- Review the Thoughtbot code-review guide for suggestions to follow when reviewing merge requests: - [Thoughtbot](https://github.com/thoughtbot/guides/tree/master/code-review) -- See GitLab merge requests for examples: [Merge Requests](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests) - -## Merge Conflicts - -- Happen often -- Learning to fix conflicts is hard -- Practice makes perfect -- Force push after fixing conflicts. Be careful! - -### Example Plan - -1. Checkout a new branch and edit conflicts.rb. Add 'Line4' and 'Line5'. -1. Commit and push -1. Checkout master and edit conflicts.rb. Add 'Line6' and 'Line7' below 'Line3'. -1. Commit and push to master -1. Create a merge request and watch it fail -1. Rebase our new branch with master -1. Fix conflicts on the conflicts.rb file. -1. Stage the file and continue rebasing -1. Force push the changes -1. Finally continue with the Merge Request - -### Example 1/2 - -```shell -git checkout -b conflicts_branch - -# vi conflicts.rb -# Add 'Line4' and 'Line5' - -git commit -am "add line4 and line5" -git push origin conflicts_branch - -git checkout master - -# vi conflicts.rb -# Add 'Line6' and 'Line7' -git commit -am "add line6 and line7" -git push origin master -``` - -### Example 2/2 - -Create a merge request on the GitLab web UI. You'll see a conflict warning. - -```shell -git checkout conflicts_branch -git fetch -git rebase master - -# Fix conflicts by editing the files. - -git add conflicts.rb -# No need to commit this file - -git rebase --continue - -# Remember that we have rewritten our commit history so we -# need to force push so that our remote branch is restructured -git push origin conflicts_branch -f -``` - -### Notes - -- When to use `git merge` and when to use `git rebase` -- Rebase when updating your branch with master -- Merge when bringing changes from feature to master -- Reference: <https://www.atlassian.com/git/tutorials/merging-vs-rebasing> - -## Revert and Unstage - -### Unstage - -To remove files from stage use reset HEAD. Where HEAD is the last commit of the current branch: - -```shell -git reset HEAD <file> -``` - -This will unstage the file but maintain the modifications. To revert the file back to the state it was in before the changes we can use: - -```shell -git checkout -- <file> -``` - -To remove a file from disk and repo use `git rm` and to remove a directory use the `-r` flag: - -```shell -git rm '*.txt' -git rm -r <dirname> -``` - -If we want to remove a file from the repository but keep it on disk, say we forgot to add it to our `.gitignore` file then use `--cache`: - -```shell -git rm <filename> --cache -``` - -### Undo Commits - -Undo last commit putting everything back into the staging area: - -```shell -git reset --soft HEAD^ -``` - -Add files and change message with: - -```shell -git commit --amend -m "New Message" -``` - -Undo last and remove changes - -```shell -git reset --hard HEAD^ -``` - -Same as last one but for two commits back: - -```shell -git reset --hard HEAD^^ -``` - -Don't reset after pushing - -### Reset Workflow - -1. Edit file again 'edit_this_file.rb' -1. Check status -1. Add and commit with wrong message -1. Check log -1. Amend commit -1. Check log -1. Soft reset -1. Check log -1. Pull for updates -1. Push changes - -```shell -# Change file edit_this_file.rb -git status -git commit -am "kjkfjkg" -git log -git commit --amend -m "New comment added" -git log -git reset --soft HEAD^ -git log -git pull origin master -git push origin master -``` - -### `git revert` vs `git reset` - -Reset removes the commit while revert removes the changes but leaves the commit -Revert is safer considering we can revert a revert - -```shell -# Changed file -git commit -am "bug introduced" -git revert HEAD -# New commit created reverting changes -# Now we want to re apply the reverted commit -git log # take hash from the revert commit -git revert <rev commit hash> -# reverted commit is back (new commit created again) -``` - -## Questions - -## Instructor Notes - -### Version Control - -- Local VCS was used with a filesystem or a simple db. -- Centralized VCS such as Subversion includes collaboration but - still is prone to data loss as the main server is the single point of - failure. -- Distributed VCS enables the team to have a complete copy of the project - and work with little dependency to the main server. In case of a main - server failing the project can be recovered by any of the latest copies - from the team +Visit our [documentation page](https://docs.gitlab.com) for information about GitLab. diff --git a/doc/university/training/topics/explore_gitlab.md b/doc/university/training/topics/explore_gitlab.md index 4ca931d0e26..8678f8fd9eb 100644 --- a/doc/university/training/topics/explore_gitlab.md +++ b/doc/university/training/topics/explore_gitlab.md @@ -1,12 +1,5 @@ --- -comments: false +redirect_to: '../../../gitlab-basics/README.md' --- -# Explore GitLab projects - -- Dashboard -- User Preferences -- Issues -- Milestones and Labels -- Manage project members -- Project settings +This document was moved to [another location](../../../gitlab-basics/README.md). diff --git a/doc/update/README.md b/doc/update/README.md index fa9fdee4ed4..85fc4363673 100644 --- a/doc/update/README.md +++ b/doc/update/README.md @@ -50,7 +50,8 @@ However, for this to work there are the following requirements: migrations](../development/post_deployment_migrations.md) (included in zero downtime update steps below). - You are using PostgreSQL. Starting from GitLab 12.1, MySQL is not supported. -- Multi-node GitLab instance. Single-node instances may experience brief interruptions as services restart. +- Multi-node GitLab instance. Single-node instances may experience brief interruptions + [as services restart (Puma in particular)](https://docs.gitlab.com/omnibus/update/README.html#single-node-deployment). Most of the time you can safely upgrade from a patch release to the next minor release if the patch release is not the latest. For example, upgrading from @@ -200,7 +201,7 @@ Below you can find some guides to help you change editions easily. ### Community to Enterprise Edition ->**Note:** +NOTE: **Note:** The following guides are for subscribers of the Enterprise Edition only. If you wish to upgrade your GitLab installation from Community to Enterprise @@ -221,6 +222,11 @@ possible. ## Version specific upgrading instructions +### 13.3.0 + +The recommended Git version is Git v2.28. The minimum required version of Git +v2.24 remains the same. + ### 13.2.0 GitLab installations that have multiple web nodes will need to be @@ -232,6 +238,11 @@ After upgrading, if some of your users are unexpectedly encountering 404 or 422 or "blocked" messages when using the command line, their accounts may have been un-confirmed. In that case, please ask them to check their email for a re-confirmation link. +For more information, see our discussion of [Email confirmation issues](../user/upgrade_email_bypass.md). + +GitLab 13.2.0 relies on the `btree_gist` extension for PostgreSQL. For installations with an externally managed PostgreSQL setup, please make sure to +[install the extension manually](https://www.postgresql.org/docs/11/sql-createextension.html) before upgrading GitLab if the database user for GitLab +is not a superuser. This is not necessary for installations using a GitLab managed PostgreSQL database. ### 13.1.0 diff --git a/doc/update/upgrading_from_source.md b/doc/update/upgrading_from_source.md index af461b93910..f8762866a53 100644 --- a/doc/update/upgrading_from_source.md +++ b/doc/update/upgrading_from_source.md @@ -124,7 +124,7 @@ rm go1.13.5.linux-amd64.tar.gz CAUTION: **Caution:** From GitLab 13.1, you must use at least Git v2.24 (previous minimum version was v2.22). -Git v2.26 is recommended. +Git v2.28 is recommended. To check you are running the minimum required Git version, see [Git versions](../install/requirements.md#git-versions). @@ -152,9 +152,9 @@ make install # Download and compile from source cd /tmp -curl --remote-name --location --progress https://www.kernel.org/pub/software/scm/git/git-2.26.0.tar.gz -echo 'aa168c2318e7187cd295a645f7370cc6d71a324aafc932f80f00c780b6a26bed git-2.26.0.tar.gz' | shasum -a256 -c - && tar -xzf git-2.26.0.tar.gz -cd git-2.26.0/ +curl --remote-name --location --progress https://www.kernel.org/pub/software/scm/git/git-2.28.0.tar.gz +echo 'f914c60a874d466c1e18467c864a910dd4ea22281ba6d4d58077cb0c3f115170 git-2.28.0.tar.gz' | shasum -a256 -c - && tar -xzf git-2.28.0.tar.gz +cd git-2.28.0/ ./configure --with-libpcre make prefix=/usr/local all diff --git a/doc/user/admin_area/abuse_reports.md b/doc/user/admin_area/abuse_reports.md index 783aadc0974..0283f1a9eff 100644 --- a/doc/user/admin_area/abuse_reports.md +++ b/doc/user/admin_area/abuse_reports.md @@ -12,6 +12,14 @@ View and resolve abuse reports from GitLab users. GitLab administrators can view and [resolve](#resolving-abuse-reports) abuse reports in the Admin Area. +## Receiving notifications of abuse reports + +To receive notifications of new abuse reports by e-mail, follow these steps: + +1. Select **Admin Area > Settings > Reporting**. +1. Expand the **Abuse reports** section. +1. Provide an email address. + ## Reporting abuse To find out more about reporting abuse, see [abuse reports user diff --git a/doc/user/admin_area/custom_project_templates.md b/doc/user/admin_area/custom_project_templates.md index d194ca42215..f39e5b198e7 100644 --- a/doc/user/admin_area/custom_project_templates.md +++ b/doc/user/admin_area/custom_project_templates.md @@ -1,6 +1,6 @@ --- -stage: Plan -group: Project Management +stage: Manage +group: Import info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference --- diff --git a/doc/user/admin_area/img/scope_mr_approval_settings_v13_1.png b/doc/user/admin_area/img/scope_mr_approval_settings_v13_1.png Binary files differindex c6ca2bac83c..fe85d567a57 100644 --- a/doc/user/admin_area/img/scope_mr_approval_settings_v13_1.png +++ b/doc/user/admin_area/img/scope_mr_approval_settings_v13_1.png diff --git a/doc/user/admin_area/index.md b/doc/user/admin_area/index.md index e63d0c7c882..8aa50bb0496 100644 --- a/doc/user/admin_area/index.md +++ b/doc/user/admin_area/index.md @@ -59,13 +59,13 @@ The Dashboard is the default view of the Admin Area, and is made up of the follo ## Overview section -The following topics document the **{overview}** **Overview** section of the Admin Area. +The following topics document the **Overview** section of the Admin Area. ### Administering Projects You can administer all projects in the GitLab instance from the Admin Area's Projects page. -To access the Projects page, go to **{admin}** **Admin Area >** **{overview}** **Overview > Projects**. +To access the Projects page, go to **Admin Area > Overview > Projects**. Click the **All**, **Private**, **Internal**, or **Public** tab to list only projects of that criteria. @@ -105,7 +105,7 @@ You can combine the filter options. For example, to list only public projects wi You can administer all users in the GitLab instance from the Admin Area's Users page. -To access the Users page, go to **{admin}** **Admin Area >** **{overview}** **Overview > Users**. +To access the Users page, go to **Admin Area > Overview > Users**. To list users matching a specific criteria, click on one of the following tabs on the **Users** page: @@ -157,7 +157,7 @@ reflected in the statistics. You can administer all groups in the GitLab instance from the Admin Area's Groups page. -To access the Groups page, go to **{admin}** **Admin Area >** **{overview}** **Overview > Groups**. +To access the Groups page, go to **Admin Area > Overview > Groups**. For each group, the page displays their name, description, size, number of projects in the group, number of members, and whether the group is private, internal, or public. To edit a group, click @@ -176,7 +176,7 @@ To [Create a new group](../group/index.md#create-a-new-group) click **New group* You can administer all jobs in the GitLab instance from the Admin Area's Jobs page. -To access the Jobs page, go to **{admin}** **Admin Area >** **{overview}** **Overview > Jobs**. +To access the Jobs page, go to **Admin Area > Overview > Jobs**. All jobs are listed, in descending order of job ID. @@ -201,7 +201,7 @@ For each job, the following details are listed: You can administer all Runners in the GitLab instance from the Admin Area's **Runners** page. See [GitLab Runner](https://docs.gitlab.com/runner/) for more information on Runner itself. -To access the **Runners** page, go to **{admin}** **Admin Area >** **{overview}** **Overview > Runners**. +To access the **Runners** page, go to **Admin Area > Overview > Runners**. The **Runners** page features: @@ -247,7 +247,7 @@ You can also edit, pause, or remove each Runner. You can list all Gitaly servers in the GitLab instance from the Admin Area's **Gitaly Servers** page. For more details, see [Gitaly](../../administration/gitaly/index.md). -To access the **Gitaly Servers** page, go to **{admin}** **Admin Area >** **{overview}** **Overview > Gitaly Servers**. +To access the **Gitaly Servers** page, go to **Admin Area > Overview > Gitaly Servers**. For each Gitaly server, the following details are listed: @@ -261,7 +261,7 @@ For each Gitaly server, the following details are listed: ## Monitoring section -The following topics document the **{monitor}** **Monitoring** section of the Admin Area. +The following topics document the **Monitoring** section of the Admin Area. ### System Info diff --git a/doc/user/admin_area/license.md b/doc/user/admin_area/license.md index c5e29612596..2c849db66b1 100644 --- a/doc/user/admin_area/license.md +++ b/doc/user/admin_area/license.md @@ -102,6 +102,14 @@ In order to get back all the previous functionality, a new license must be uploa To fall back to having only the Core features active, you'll need to delete the expired license(s). +### Remove a license + +To remove a license from a self-managed instance: + +1. Go to the [Admin Area](index.md) (click the wrench in the top navigation bar). +1. Click **License** in the left sidebar. +1. Click **Remove License**. + ## License history It's possible to upload and view more than one license, diff --git a/doc/user/admin_area/merge_requests_approvals.md b/doc/user/admin_area/merge_requests_approvals.md index 6d9d634ce14..8f51c03e105 100644 --- a/doc/user/admin_area/merge_requests_approvals.md +++ b/doc/user/admin_area/merge_requests_approvals.md @@ -15,7 +15,7 @@ if they are enabled at an instance level. To enable merge request approval rules for an instance: -1. Navigate to **{admin}** **Admin Area >** **{push-rules}** **Push Rules** and expand **Merge +1. Navigate to **Admin Area >** **{push-rules}** **Push Rules** and expand **Merge requests approvals**. 1. Set the required rule. 1. Click **Save changes**. diff --git a/doc/user/admin_area/settings/account_and_limit_settings.md b/doc/user/admin_area/settings/account_and_limit_settings.md index 167016f1cb5..4651a548ff9 100644 --- a/doc/user/admin_area/settings/account_and_limit_settings.md +++ b/doc/user/admin_area/settings/account_and_limit_settings.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" type: reference --- @@ -26,20 +29,6 @@ If you choose a size larger than what is currently configured for the web server you will likely get errors. See the [troubleshooting section](#troubleshooting) for more details. -## Maximum namespace storage size - -This sets a maximum size limit on each namespace. The following are included in the namespace size: - -- Repository -- Wiki -- LFS objects -- Build artifacts -- Packages -- Snippets - -NOTE: **Note:** -This limit is not currently enforced but will be in a future release. - ## Repository size limit **(STARTER ONLY)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/740) in [GitLab Enterprise Edition 8.12](https://about.gitlab.com/releases/2016/09/22/gitlab-8-12-released/#limit-project-size-ee). @@ -86,7 +75,8 @@ The first push of a new project, including LFS objects, will be checked for size and **will** be rejected if the sum of their sizes exceeds the maximum allowed repository size. -**Note:** The repository size limit includes repository files and LFS, and does not include artifacts. +NOTE: **Note:** +The repository size limit includes repository files and LFS, and does not include artifacts. For details on manually purging files, see [reducing the repository size using Git](../../project/repository/reducing_the_repo_size_using_git.md). @@ -159,19 +149,19 @@ To do this: ### Enable or disable optional enforcement of Personal Access Token expiry Feature **(CORE ONLY)** -Optional Enforcement of Personal Access Token Expiry is under development and not ready for production use. It is deployed behind a feature flag that is **disabled by default**. +Optional Enforcement of Personal Access Token Expiry is deployed behind a feature flag and is **disabled by default**. [GitLab administrators with access to the GitLab Rails console](../../../administration/feature_flags.md) can enable it for your instance from the [rails console](../../../administration/feature_flags.md#start-the-gitlab-rails-console). To enable it: ```ruby -Feature.enable(:enforce_personal_access_token_expiration) +Feature.enable(:enforce_pat_expiration) ``` To disable it: ```ruby -Feature.disable(:enforce_personal_access_token_expiration) +Feature.disable(:enforce_pat_expiration) ``` ## Disabling user profile name changes **(PREMIUM ONLY)** diff --git a/doc/user/admin_area/settings/gitaly_timeouts.md b/doc/user/admin_area/settings/gitaly_timeouts.md index 68f359368f0..2003d02c9b3 100644 --- a/doc/user/admin_area/settings/gitaly_timeouts.md +++ b/doc/user/admin_area/settings/gitaly_timeouts.md @@ -1,4 +1,8 @@ --- +stage: Create +group: Gitaly +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference type: reference --- diff --git a/doc/user/admin_area/settings/img/import_export_rate_limits_v13_2.png b/doc/user/admin_area/settings/img/import_export_rate_limits_v13_2.png Binary files differindex 78c94b3803c..76015ce0ee3 100644 --- a/doc/user/admin_area/settings/img/import_export_rate_limits_v13_2.png +++ b/doc/user/admin_area/settings/img/import_export_rate_limits_v13_2.png diff --git a/doc/user/admin_area/settings/index.md b/doc/user/admin_area/settings/index.md index 8c1e82f838b..db6dbb7f38b 100644 --- a/doc/user/admin_area/settings/index.md +++ b/doc/user/admin_area/settings/index.md @@ -4,7 +4,7 @@ type: index # Admin Area settings **(CORE ONLY)** -As an administrator of a GitLab self-managed instance, you can manage the behavior of your deployment. To do so, select **{admin}** **Admin Area > Settings**. +As an administrator of a GitLab self-managed instance, you can manage the behavior of your deployment. To do so, select **Admin Area > Settings**. The admin area is not accessible on GitLab.com, and settings can only be changed by the GitLab.com administrators. See the [GitLab.com settings](../../gitlab_com/index.md) @@ -12,8 +12,7 @@ documentation for all current settings and limits on the GitLab.com instance. ## General -Access the default page for admin area settings by navigating to -**{admin}** **Admin Area > Settings > General**: +Access the default page for admin area settings by navigating to **Admin Area > Settings > General**: | Option | Description | | ------ | ----------- | @@ -62,7 +61,7 @@ Access the default page for admin area settings by navigating to | ------ | ----------- | | [Continuous Integration and Deployment](continuous_integration.md) | Auto DevOps, runners and job artifacts. | | [Required pipeline configuration](continuous_integration.md#required-pipeline-configuration-premium-only) **(PREMIUM ONLY)** | Set an instance-wide auto included [pipeline configuration](../../../ci/yaml/README.md). This pipeline configuration will be run after the project's own configuration. | -| [Package Registry](continuous_integration.md#package-registry-configuration-premium-only) **(PREMIUM ONLY)**| Settings related to the use and experience of using GitLab's Package Registry. Note there are [risks involved](./../../packages/container_registry/index.md#use-with-external-container-registries) in enabling some of these settings. | +| [Package Registry](continuous_integration.md#package-registry-configuration-premium-only) | Settings related to the use and experience of using GitLab's Package Registry. Note there are [risks involved](./../../packages/container_registry/index.md#use-with-external-container-registries) in enabling some of these settings. | ## Reporting @@ -90,13 +89,13 @@ Access the default page for admin area settings by navigating to | [User and IP rate limits](user_and_ip_rate_limits.md) | Configure limits for web and API requests. | | [Outbound requests](../../../security/webhooks.md) | Allow requests to the local network from hooks and services. | | [Protected Paths](protected_paths.md) | Configure paths to be protected by Rack Attack. | -| [Incident Management](../../incident_management/index.md) Limits | Configure limits on the number of inbound alerts able to be sent to a project. | +| [Incident Management](../../../operations/incident_management/index.md) Limits | Configure limits on the number of inbound alerts able to be sent to a project. | ## Geo | Option | Description | | ------ | ----------- | -| Geo | Geo allows you to replicate your GitLab instance to other geographical locations. Redirects to **{admin}** **Admin Area >** **{location-dot}** **Geo >** **{settings}** **Settings**, and will no longer be available at **{admin}** **Admin Area >** **{settings}** **Settings >** **{location-dot}** **Geo** in [GitLab 13.0](https://gitlab.com/gitlab-org/gitlab/-/issues/36896). | +| Geo | Geo allows you to replicate your GitLab instance to other geographical locations. Redirects to **Admin Area > Geo > Settings**, and will no longer be available at **Admin Area > Settings > Geo** in [GitLab 13.0](https://gitlab.com/gitlab-org/gitlab/-/issues/36896). | ## Preferences diff --git a/doc/user/admin_area/settings/instance_template_repository.md b/doc/user/admin_area/settings/instance_template_repository.md index ae56c67f0ab..97380b93295 100644 --- a/doc/user/admin_area/settings/instance_template_repository.md +++ b/doc/user/admin_area/settings/instance_template_repository.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" type: reference --- @@ -30,12 +33,13 @@ Templates must be added to a specific subdirectory in the repository, corresponding to the kind of template. The following types of custom templates are supported: -| Type | Directory | Extension | -| :---------------: | :-----------: | :-----------: | -| `Dockerfile` | `Dockerfile` | `.dockerfile` | -| `.gitignore` | `gitignore` | `.gitignore` | -| `.gitlab-ci.yml` | `gitlab-ci` | `.yml` | -| `LICENSE` | `LICENSE` | `.txt` | +| Type | Directory | Extension | +| :---------------: | :-----------: | :-----------: | +| `Dockerfile` | `Dockerfile` | `.dockerfile` | +| `.gitignore` | `gitignore` | `.gitignore` | +| `.gitlab-ci.yml` | `gitlab-ci` | `.yml` | +| `LICENSE` | `LICENSE` | `.txt` | +| `metrics-dashboard.yml` | `metrics-dashboards` | `.yml` | Each template must go in its respective subdirectory, have the correct extension and not be empty. So, the hierarchy should look like this: @@ -54,6 +58,9 @@ extension and not be empty. So, the hierarchy should look like this: |-- LICENSE |-- custom_license.txt |-- another_license.txt +|-- metrics-dashboards + |-- custom_metrics-dashboard.yml + |-- another_metrics-dashboard.yml ``` Once this is established, the list of custom templates will be included when diff --git a/doc/user/admin_area/settings/project_integration_management.md b/doc/user/admin_area/settings/project_integration_management.md new file mode 100644 index 00000000000..e4fe7e36139 --- /dev/null +++ b/doc/user/admin_area/settings/project_integration_management.md @@ -0,0 +1,54 @@ +--- +stage: Create +group: Ecosystem +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Project integration management **(CORE ONLY)** + +> [Introduced in](https://gitlab.com/groups/gitlab-org/-/epics/2137) GitLab 13.3. + +Project integrations can be configured and enabled by project administrators. As a GitLab instance administrator, you can set default configuration parameters for a given integration that all projects can inherit and use, enabling the integration for all projects that are not already using custom settings. + +You can update these default settings at any time, changing the settings in use for all projects that are set to use instance-level defaults. This also enables the integration for all projects on which it was not already enabled. + +It is only possible to inherit the complete settings for an integration. Per-field inheritance is [planned](https://gitlab.com/groups/gitlab-org/-/epics/2137), as well as [group-level management](https://gitlab.com/groups/gitlab-org/-/epics/2543) of integration settings. + +## Manage default settings for a project integration + +1. Navigate to **Admin Area > Settings > Integrations**. +1. Select a project integration. +1. Enter configuration details and click **Save changes**. + +CAUTION: **Caution:** +This may affect all or most of the projects on your GitLab instance. Please review the details below. + +If this is the first time you are setting up instance-level settings for an integration: + +- The integration is enabled for all projects that do not already have this integration configured if you have the **Enable integration** toggle turned on in the instance-level settings. +- Projects that already have the integration configured are not affected, but can choose to use the inherited settings at any time. + +When you make further changes to the instance defaults: + +- They are immediately applied to all projects that have the integration set to use instance-level default settings. +- They are immediately applied to newer projects, created since you last saved defaults for the integration. + - If your instance-level default setting has the **Enable integration** toggle turned on, the integration is automatically enabled for all such projects. +- Projects with custom settings selected for the integration are not immediately affected and may choose to use the latest instance-level defaults at any time. + +It is only possible to inherit the complete settings for an integration. Per-field inheritance is [planned](https://gitlab.com/groups/gitlab-org/-/epics/2137). This would allow instance administrators to update settings inherited by projects without enabling the integration on all non-configured projects by default. + +## Use instance-level default settings for a project integration + +1. Navigate to **Project > Settings > Integrations**. +1. Choose the integration you want to enable or update. +1. From the drop-down, select **Use default settings**. +1. Ensure the toggle is set to **Enable integration**. +1. Click **Save changes**. + +## Use custom settings for a project integration + +1. Navigate to **Project > Settings > Integrations**. +1. Choose the integration you want to enable or update. +1. From the drop-down, select **Use custom settings**. +1. Ensure the toggle is set to **Enable integration** and enter all required settings. +1. Click **Save changes**. diff --git a/doc/user/admin_area/settings/push_event_activities_limit.md b/doc/user/admin_area/settings/push_event_activities_limit.md index bbc5ed04202..cf23ea12ef4 100644 --- a/doc/user/admin_area/settings/push_event_activities_limit.md +++ b/doc/user/admin_area/settings/push_event_activities_limit.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" type: reference --- diff --git a/doc/user/admin_area/settings/visibility_and_access_controls.md b/doc/user/admin_area/settings/visibility_and_access_controls.md index 92eeb6a04b7..e5c7947399d 100644 --- a/doc/user/admin_area/settings/visibility_and_access_controls.md +++ b/doc/user/admin_area/settings/visibility_and_access_controls.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" type: reference --- @@ -64,7 +67,7 @@ To ensure only admin users can delete projects: 1. Check the **Default project deletion protection** checkbox. 1. Click **Save changes**. -## Default deletion adjourned period **(PREMIUM ONLY)** +## Default deletion delay **(PREMIUM ONLY)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/32935) in GitLab 12.6. @@ -85,7 +88,7 @@ To change this period: 1. Select the desired option. 1. Click **Save changes**. -### Override default deletion adjourned period +### Override default deletion delayed period Alternatively, projects that are marked for removal can be deleted immediately. To do so: diff --git a/doc/user/analytics/img/merge_request_analytics_v13_3.png b/doc/user/analytics/img/merge_request_analytics_v13_3.png Binary files differnew file mode 100644 index 00000000000..f90f3625a51 --- /dev/null +++ b/doc/user/analytics/img/merge_request_analytics_v13_3.png diff --git a/doc/user/analytics/img/new_value_stream_v13_3.png b/doc/user/analytics/img/new_value_stream_v13_3.png Binary files differnew file mode 100644 index 00000000000..4284b8ab194 --- /dev/null +++ b/doc/user/analytics/img/new_value_stream_v13_3.png diff --git a/doc/user/analytics/img/repository_analytics_v13_0.png b/doc/user/analytics/img/repository_analytics_v13_0.png Binary files differindex b70b63a6239..dd943b4f44d 100644 --- a/doc/user/analytics/img/repository_analytics_v13_0.png +++ b/doc/user/analytics/img/repository_analytics_v13_0.png diff --git a/doc/user/analytics/img/vsa_filter_bar_v13.3.png b/doc/user/analytics/img/vsa_filter_bar_v13.3.png Binary files differnew file mode 100644 index 00000000000..71e59892434 --- /dev/null +++ b/doc/user/analytics/img/vsa_filter_bar_v13.3.png diff --git a/doc/user/analytics/img/vsa_time_metrics_v13_0.png b/doc/user/analytics/img/vsa_time_metrics_v13_0.png Binary files differindex 073976fd740..799a81584a0 100644 --- a/doc/user/analytics/img/vsa_time_metrics_v13_0.png +++ b/doc/user/analytics/img/vsa_time_metrics_v13_0.png diff --git a/doc/user/analytics/index.md b/doc/user/analytics/index.md index 18f6d79ef23..47852cb0498 100644 --- a/doc/user/analytics/index.md +++ b/doc/user/analytics/index.md @@ -24,11 +24,11 @@ The following analytics features are available at the group level: - [Contribution](../group/contribution_analytics/index.md). **(STARTER)** - [Insights](../group/insights/index.md). **(ULTIMATE)** -- [Issues](../group/issues_analytics/index.md). **(PREMIUM)** +- [Issue](../group/issues_analytics/index.md). **(PREMIUM)** - [Productivity](productivity_analytics.md), enabled with the `productivity_analytics` - [feature flag](../../development/feature_flags/development.md#enabling-a-feature-flag-in-development). **(PREMIUM)** + [feature flag](../../development/feature_flags/development.md#enabling-a-feature-flag-locally-in-development). **(PREMIUM)** - [Value Stream](value_stream_analytics.md), enabled with the `cycle_analytics` - [feature flag](../../development/feature_flags/development.md#enabling-a-feature-flag-in-development). **(PREMIUM)** + [feature flag](../../development/feature_flags/development.md#enabling-a-feature-flag-locally-in-development). **(PREMIUM)** ## Project-level analytics @@ -37,7 +37,8 @@ The following analytics features are available at the project level: - [CI/CD](../../ci/pipelines/index.md#pipeline-success-and-duration-charts). **(STARTER)** - [Code Review](code_review_analytics.md). **(STARTER)** - [Insights](../group/insights/index.md). **(ULTIMATE)** -- [Issues](../group/issues_analytics/index.md). **(PREMIUM)** +- [Issue](../group/issues_analytics/index.md). **(PREMIUM)** +- [Merge Request](merge_request_analytics.md). **(STARTER)** - [Repository](repository_analytics.md). - [Value Stream](value_stream_analytics.md), enabled with the `cycle_analytics` - [feature flag](../../development/feature_flags/development.md#enabling-a-feature-flag-in-development). **(STARTER)** + [feature flag](../../development/feature_flags/development.md#enabling-a-feature-flag-locally-in-development). **(STARTER)** diff --git a/doc/user/analytics/merge_request_analytics.md b/doc/user/analytics/merge_request_analytics.md new file mode 100644 index 00000000000..01295ae888b --- /dev/null +++ b/doc/user/analytics/merge_request_analytics.md @@ -0,0 +1,42 @@ +--- +description: "Merge Request Analytics help you understand the efficiency of your code review process, and the productivity of your team." # Up to ~200 chars long. They will be displayed in Google Search snippets. It may help to write the page intro first, and then reuse it here. +stage: Manage +group: Analytics +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + + +# Merge Request Analytics **(STARTER)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/229045) in [GitLab Starter](https://about.gitlab.com/pricing/) 13.3. + +Merge Request Analytics helps you understand the efficiency of your code review process, and the productivity of your team. + +## Overview + +Merge Request Analytics displays information about all accepted merge requests. + +The Throughput chart shows the number of completed merge requests, by month. Merge request throughput is +a common measure of productivity in software engineering. Although imperfect, the average throughput can +be a meaningful benchmark of your team's overall productivity. + +To access Merge Request Analytics, from your project's menu, go to **Analytics > Merge Request**. + + + +## Use cases + +This feature is designed for [development team leaders](https://about.gitlab.com/handbook/marketing/product-marketing/roles-personas/#delaney-development-team-lead) +and others who want to understand broad patterns in code review and productivity. + +You can use Merge Request Analytics to expose when your team is most and least productive, and +identify improvements that might substantially accelerate your development cycle. + +Merge Request Analytics could be used when: + +- You want to know if you were more productive this month than last month, or 12 months ago. +- You want to drill into low- or high-productivity months to understand the work that took place. + +## Permissions + +- On [Starter or Bronze tier](https://about.gitlab.com/pricing/) and above. diff --git a/doc/user/analytics/value_stream_analytics.md b/doc/user/analytics/value_stream_analytics.md index b11bae98af3..9114b6de6bc 100644 --- a/doc/user/analytics/value_stream_analytics.md +++ b/doc/user/analytics/value_stream_analytics.md @@ -48,7 +48,30 @@ There are seven stages that are tracked as part of the Value Stream Analytics ca - **Total** (Total) - Total lifecycle time. That is, the velocity of the project or team. [Previously known](https://gitlab.com/gitlab-org/gitlab/-/issues/38317) as **Production**. -## Date ranges +## Filter the analytics data + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/13216) in GitLab 13.3 + +GitLab provides the ability to filter analytics based on the following parameters: + +- Milestones (Group level) +- Labels (Group level) +- Author +- Assignees + +To filter results: + +1. Select a group. +1. Click on the filter bar. +1. Select a parameter to filter by. +1. Select a value from the autocompleted results, or type to refine the results. + + + +NOTE: **Note:** +Filtering is available only for group-level Value Stream Analytics. + +### Date ranges > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/13216) in GitLab 12.4. @@ -165,7 +188,7 @@ A few notes: cycles, calculate their median time and the result is what the dashboard of Value Stream Analytics is showing. -## Customizable Value Stream Analytics +## Customizable Value Stream Analytics **(PREMIUM)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12196) in GitLab 12.9. @@ -173,8 +196,7 @@ The default stages are designed to work straight out of the box, but they might all teams. Different teams use different approaches to building software, so some teams might want to customize their Value Stream Analytics. -GitLab allows users to hide default stages and create custom stages that align better to their -development workflow. +GitLab allows users to create multiple value streams, hide default stages and create custom stages that align better to their development workflow. NOTE: **Note:** Customizability is [only available for group-level](https://gitlab.com/gitlab-org/gitlab/-/issues/35823#note_272558950) Value Stream Analytics. @@ -272,6 +294,34 @@ To recover a default stage that was previously hidden: 1. In the top right corner open the **Recover hidden stage** dropdown. 1. Select a stage. +### Creating a value stream + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/221202) in GitLab 13.3 + +A default value stream is readily available for each group. You can create additional value streams based on the different areas of work that you would like to measure. + +Once created, a new value stream includes the [seven stages](#overview) that follow +[GitLab workflow](../../topics/gitlab_flow.md) +best practices. You can customize this flow by adding, hiding or re-ordering stages. + +To create a value stream: + +1. Navigate to your group's **Analytics > Value Stream**. +1. Click the Value stream dropdown and select **Create new Value Stream** +1. Fill in a name for the new Value Stream +1. Click the **Create Value Stream** button. + + + +### Disabling custom value streams + +Custom value streams are enabled by default. If you have a self-managed instance, an +administrator can open a Rails console and disable them with the following command: + +```ruby +Feature.disable(:value_stream_analytics_create_multiple_value_streams) +``` + ## Days to completion chart > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/21631) in GitLab 12.6. diff --git a/doc/user/application_security/configuration/index.md b/doc/user/application_security/configuration/index.md index 229a8572206..1195d07d7b7 100644 --- a/doc/user/application_security/configuration/index.md +++ b/doc/user/application_security/configuration/index.md @@ -9,28 +9,21 @@ info: To determine the technical writer assigned to the Stage/Group associated w > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/20711) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.6. -## Overview +The Security Configuration page displays the configuration state of each security feature in the +current project. The page uses the project's latest default branch [CI pipeline](../../../ci/pipelines/index.md) +to determine each feature's configuration state. If a job with the expected security report artifact +exists in the pipeline, the feature is considered enabled. -The security configuration page displays the configuration state of each of the security -features and can be accessed through a project's sidebar nav. - - - -The page uses the project's latest default branch [CI pipeline](../../../ci/pipelines/index.md) to determine the configuration -state of each feature. If a job with the expected security report artifact exists in the pipeline, -the feature is considered configured. +You can only enable SAST from the Security Configuration page. Documentation links are included for +the other features. For details about configuring SAST, see [Configure SAST in the UI](../sast/index.md#configure-sast-in-the-ui). NOTE: **Note:** If the latest pipeline used [Auto DevOps](../../../topics/autodevops/index.md), -all security features will be configured by default. +all security features are configured by default. -## Limitations +## View Security Configuration -It is not yet possible to enable or disable most features using the -configuration page. However, instructions on how to enable or disable a feature -can be found through the links next to each feature on that page. +To view a project's security configuration: -If a project does not have an existing CI configuration, then the SAST feature -can be enabled by clicking on the "Enable with Merge Request" button under the -"Manage" column. Future work will expand this to editing _existing_ CI -configurations, and to other security features. +1. Go to the project's home page. +1. In the left sidebar, go to **Security & Configuration** > **Configuration**. diff --git a/doc/user/application_security/container_scanning/index.md b/doc/user/application_security/container_scanning/index.md index 7bc8b62825c..6b7086ddc71 100644 --- a/doc/user/application_security/container_scanning/index.md +++ b/doc/user/application_security/container_scanning/index.md @@ -156,25 +156,25 @@ variables: Container Scanning can be [configured](#customizing-the-container-scanning-settings) using environment variables. -| Environment Variable | Description | Default | -| ------ | ------ | ------ | -| `SECURE_ANALYZERS_PREFIX` | Set the Docker registry base address from which to download the analyzer. | `"registry.gitlab.com/gitlab-org/security-products/analyzers"` | -| `KLAR_TRACE` | Set to true to enable more verbose output from klar. | `"false"` | -| `CLAIR_TRACE` | Set to true to enable more verbose output from the clair server process. | `"false"` | -| `DOCKER_USER` | Username for accessing a Docker registry requiring authentication. | `$CI_REGISTRY_USER` | -| `DOCKER_PASSWORD` | Password for accessing a Docker registry requiring authentication. | `$CI_REGISTRY_PASSWORD` | -| `CLAIR_OUTPUT` | Severity level threshold. Vulnerabilities with severity level higher than or equal to this threshold will be outputted. Supported levels are `Unknown`, `Negligible`, `Low`, `Medium`, `High`, `Critical` and `Defcon1`. | `Unknown` | -| `REGISTRY_INSECURE` | Allow [Klar](https://github.com/optiopay/klar) to access insecure registries (HTTP only). Should only be set to `true` when testing the image locally. | `"false"` | -| `DOCKER_INSECURE` | Allow [Klar](https://github.com/optiopay/klar) to access secure Docker registries using HTTPS with bad (or self-signed) SSL certificates. | `"false"` | -| `CLAIR_VULNERABILITIES_DB_URL` | (**DEPRECATED - use `CLAIR_DB_CONNECTION_STRING` instead**) This variable is explicitly set in the [services section](https://gitlab.com/gitlab-org/gitlab/-/blob/898c5da43504eba87b749625da50098d345b60d6/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml#L23) of the `Container-Scanning.gitlab-ci.yml` file and defaults to `clair-vulnerabilities-db`. This value represents the address that the [PostgreSQL server hosting the vulnerabilities definitions](https://hub.docker.com/r/arminc/clair-db) is running on and **shouldn't be changed** unless you're running the image locally as described in the [Running the standalone Container Scanning Tool](#running-the-standalone-container-scanning-tool) section. | `clair-vulnerabilities-db` | -| `CLAIR_DB_CONNECTION_STRING` | This variable represents the [connection string](https://www.postgresql.org/docs/9.3/libpq-connect.html#AEN39692) to the [PostgreSQL server hosting the vulnerabilities definitions](https://hub.docker.com/r/arminc/clair-db) database and **shouldn't be changed** unless you're running the image locally as described in the [Running the standalone Container Scanning Tool](#running-the-standalone-container-scanning-tool) section. The host value for the connection string must match the [alias](https://gitlab.com/gitlab-org/gitlab/-/blob/898c5da43504eba87b749625da50098d345b60d6/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml#L23) value of the `Container-Scanning.gitlab-ci.yml` template file, which defaults to `clair-vulnerabilities-db`. | `postgresql://postgres:password@clair-vulnerabilities-db:5432/postgres?sslmode=disable&statement_timeout=60000` | -| `CI_APPLICATION_REPOSITORY` | Docker repository URL for the image to be scanned. | `$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG` | -| `CI_APPLICATION_TAG` | Docker repository tag for the image to be scanned. | `$CI_COMMIT_SHA` | -| `CLAIR_DB_IMAGE` | The Docker image name and tag for the [PostgreSQL server hosting the vulnerabilities definitions](https://hub.docker.com/r/arminc/clair-db). It can be useful to override this value with a specific version, for example, to provide a consistent set of vulnerabilities for integration testing purposes, or to refer to a locally hosted vulnerabilities database for an on-premise offline installation. | `arminc/clair-db:latest` | -| `CLAIR_DB_IMAGE_TAG` | (**DEPRECATED - use `CLAIR_DB_IMAGE` instead**) The Docker image tag for the [PostgreSQL server hosting the vulnerabilities definitions](https://hub.docker.com/r/arminc/clair-db). It can be useful to override this value with a specific version, for example, to provide a consistent set of vulnerabilities for integration testing purposes. | `latest` | -| `DOCKERFILE_PATH` | The path to the `Dockerfile` to be used for generating remediations. By default, the scanner will look for a file named `Dockerfile` in the root directory of the project, so this variable should only be configured if your `Dockerfile` is in a non-standard location, such as a subdirectory. See [Solutions for vulnerabilities](#solutions-for-vulnerabilities-auto-remediation) for more details. | `Dockerfile` | -| `ADDITIONAL_CA_CERT_BUNDLE` | Bundle of CA certs that you want to trust. | "" | -| `SECURE_LOG_LEVEL` | The log levels available are: `fatal`, `error`, `warn`, `info`, `debug` | `info` | +| Environment Variable | Default | Description | +| -------------------- | ----------- | ------- | +| `SECURE_ANALYZERS_PREFIX` | `"registry.gitlab.com/gitlab-org/security-products/analyzers"` | Set the Docker registry base address from which to download the analyzer. | +| `KLAR_TRACE` | `"false"` | Set to true to enable more verbose output from klar. | +| `CLAIR_TRACE` | `"false"` | Set to true to enable more verbose output from the clair server process. | +| `DOCKER_USER` | `$CI_REGISTRY_USER` | Username for accessing a Docker registry requiring authentication. | +| `DOCKER_PASSWORD` | `$CI_REGISTRY_PASSWORD` | Password for accessing a Docker registry requiring authentication. | +| `CLAIR_OUTPUT` | `Unknown` | Severity level threshold. Vulnerabilities with severity level higher than or equal to this threshold are outputted. Supported levels are `Unknown`, `Negligible`, `Low`, `Medium`, `High`, `Critical` and `Defcon1`. | +| `REGISTRY_INSECURE` | `"false"` | Allow [Klar](https://github.com/optiopay/klar) to access insecure registries (HTTP only). Should only be set to `true` when testing the image locally. | +| `DOCKER_INSECURE` | `"false"` | Allow [Klar](https://github.com/optiopay/klar) to access secure Docker registries using HTTPS with bad (or self-signed) SSL certificates. | +| `CLAIR_VULNERABILITIES_DB_URL` | `clair-vulnerabilities-db` | (**DEPRECATED - use `CLAIR_DB_CONNECTION_STRING` instead**) This variable is explicitly set in the [services section](https://gitlab.com/gitlab-org/gitlab/-/blob/898c5da43504eba87b749625da50098d345b60d6/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml#L23) of the `Container-Scanning.gitlab-ci.yml` file and defaults to `clair-vulnerabilities-db`. This value represents the address that the [PostgreSQL server hosting the vulnerabilities definitions](https://hub.docker.com/r/arminc/clair-db) is running on and **shouldn't be changed** unless you're running the image locally as described in the [Running the standalone Container Scanning Tool](#running-the-standalone-container-scanning-tool) section. | +| `CLAIR_DB_CONNECTION_STRING` | `postgresql://postgres:password@clair-vulnerabilities-db:5432/postgres?sslmode=disable&statement_timeout=60000` | This variable represents the [connection string](https://www.postgresql.org/docs/9.3/libpq-connect.html#AEN39692) to the [PostgreSQL server hosting the vulnerabilities definitions](https://hub.docker.com/r/arminc/clair-db) database and **shouldn't be changed** unless you're running the image locally as described in the [Running the standalone Container Scanning Tool](#running-the-standalone-container-scanning-tool) section. The host value for the connection string must match the [alias](https://gitlab.com/gitlab-org/gitlab/-/blob/898c5da43504eba87b749625da50098d345b60d6/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml#L23) value of the `Container-Scanning.gitlab-ci.yml` template file, which defaults to `clair-vulnerabilities-db`. | +| `CI_APPLICATION_REPOSITORY` | `$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG` | Docker repository URL for the image to be scanned. | +| `CI_APPLICATION_TAG` | `$CI_COMMIT_SHA` | Docker repository tag for the image to be scanned. | +| `CLAIR_DB_IMAGE` | `arminc/clair-db:latest` | The Docker image name and tag for the [PostgreSQL server hosting the vulnerabilities definitions](https://hub.docker.com/r/arminc/clair-db). It can be useful to override this value with a specific version, for example, to provide a consistent set of vulnerabilities for integration testing purposes, or to refer to a locally hosted vulnerabilities database for an on-premise offline installation. | +| `CLAIR_DB_IMAGE_TAG` | `latest` | (**DEPRECATED - use `CLAIR_DB_IMAGE` instead**) The Docker image tag for the [PostgreSQL server hosting the vulnerabilities definitions](https://hub.docker.com/r/arminc/clair-db). It can be useful to override this value with a specific version, for example, to provide a consistent set of vulnerabilities for integration testing purposes. | +| `DOCKERFILE_PATH` | `Dockerfile` | The path to the `Dockerfile` to be used for generating remediations. By default, the scanner looks for a file named `Dockerfile` in the root directory of the project, so this variable should only be configured if your `Dockerfile` is in a non-standard location, such as a subdirectory. See [Solutions for vulnerabilities](#solutions-for-vulnerabilities-auto-remediation) for more details. | +| `ADDITIONAL_CA_CERT_BUNDLE` | `""` | Bundle of CA certs that you want to trust. | +| `SECURE_LOG_LEVEL` | `info` | Set the minimum logging level. Messages of this logging level or higher are output. From highest to lowest severity, the logging levels are: `fatal`, `error`, `warn`, `info`, `debug`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/10880) in GitLab 13.1. | ### Overriding the Container Scanning template @@ -291,7 +291,7 @@ build_latest_vulnerabilities: - docker push $CI_REGISTRY/namespace/clair-vulnerabilities-db ``` -The above template will work for a GitLab Docker registry running on a local installation, however, if you're using a non-GitLab Docker registry, you'll need to change the `$CI_REGISTRY` value and the `docker login` credentials to match the details of your local registry. +The above template works for a GitLab Docker registry running on a local installation, however, if you're using a non-GitLab Docker registry, you'll need to change the `$CI_REGISTRY` value and the `docker login` credentials to match the details of your local registry. ## Running the standalone Container Scanning Tool diff --git a/doc/user/application_security/coverage_fuzzing/index.md b/doc/user/application_security/coverage_fuzzing/index.md index 85da7d85506..1672e9fbb25 100644 --- a/doc/user/application_security/coverage_fuzzing/index.md +++ b/doc/user/application_security/coverage_fuzzing/index.md @@ -7,8 +7,6 @@ type: reference, howto # Coverage Guided Fuzz Testing **(ULTIMATE)** -> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3226) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.2 as an [Alpha feature](https://about.gitlab.com/handbook/product/gitlab-the-product/#alpha). - GitLab allows you to add coverage-guided fuzz testing to your pipelines. This helps you discover bugs and potential security issues that other QA processes may miss. Coverage-guided fuzzing sends random inputs to an instrumented version of your application in an effort to cause unexpected @@ -16,17 +14,19 @@ behavior, such as a crash. Such behavior indicates a bug that you should address We recommend that you use fuzz testing in addition to the other security scanners in [GitLab Secure](../index.md) and your own test processes. If you're using [GitLab CI/CD](../../../ci/README.md), -you can run your coverage guided fuzz tests as part your CI/CD workflow. You can take advantage of -Coverage Guided Fuzzing by including the CI job in your existing `.gitlab-ci.yml` file. +you can run your coverage-guided fuzz tests as part your CI/CD workflow. You can take advantage of +coverage-guided fuzzing by including the CI job in your existing `.gitlab-ci.yml` file. ## Supported fuzzing engines and languages -GitLab supports these languages through the fuzzing engine listed for each. We currently provide a Docker image for apps written in Go, but you can test the other languages below by providing a Docker image with the fuzz engine to run your app. +GitLab supports these languages through the fuzzing engine listed for each. We currently provide a +Docker image for apps written in Go, but you can test the other languages below by providing a +Docker image with the fuzz engine to run your app. -| Language | Fuzzing Engine | Example | -|----------|---------------------------------------------------------------------------|---------| -| C/C++ | [libFuzzer](https://llvm.org/docs/LibFuzzer.html) | | -| GoLang | [go-fuzz (libFuzzer support)](https://github.com/dvyukov/go-fuzz) | | +| Language | Fuzzing Engine | Example | +|----------|----------------|---------| +| C/C++ | [libFuzzer](https://llvm.org/docs/LibFuzzer.html) | [c-cpp-example](https://gitlab.com/gitlab-org/security-products/demos/c-cpp-fuzzing-example) | +| GoLang | [go-fuzz (libFuzzer support)](https://github.com/dvyukov/go-fuzz) | [go-fuzzing-example](https://gitlab.com/gitlab-org/security-products/demos/go-fuzzing-example) | | Rust | [cargo-fuzz (libFuzzer support)](https://github.com/rust-fuzz/cargo-fuzz) | | ## Configuration @@ -49,6 +49,14 @@ targets. Each fuzz target **must** have a separate job. For example, the [go-fuzzing-example project](https://gitlab.com/gitlab-org/security-products/demos/go-fuzzing-example) contains one job that extends `.fuzz_base` for its single fuzz target. +Note that the hidden job `.fuzz_base` uses several YAML keys that you must not override in your own +job. If you include these keys in your own job, you must copy their original content. These keys +are: + +- `before_script` +- `artifacts` +- `rules` + The `my_fuzz_target` job (the separate job for your fuzz target) does the following: - Extends `.fuzz_base`. @@ -59,8 +67,8 @@ The `my_fuzz_target` job (the separate job for your fuzz target) does the follow The `gitlab-cov-fuzz` is a command-line tool that runs the instrumented application. It parses and analyzes the exception information that the fuzzer outputs. It also downloads the [corpus](#glossary) -and crash events from previous pipelines automatically. This helps your fuzz targets build on the progress of -previous fuzzing jobs. The parsed crash events and data are written to +and crash events from previous pipelines automatically. This helps your fuzz targets build on the +progress of previous fuzzing jobs. The parsed crash events and data are written to `gl-coverage-fuzzing-report.json`. ### Artifacts @@ -84,7 +92,7 @@ There are two types of jobs: Here's our current suggestion for configuring your fuzz target's timeout: -- Set `COVERAGE_FUZZING_BRANCH` to the branch where you want to run long-running (async) fuzzing +- Set `COVFUZZ_BRANCH` to the branch where you want to run long-running (async) fuzzing jobs. This is `master` by default. - Use regression or short-running fuzzing jobs for other branches or merge requests. @@ -99,7 +107,54 @@ any option available in the underlying fuzzing engine. | Environment variable | Description | |---------------------------|--------------------------------------------------------------------| -| `COVERAGE_FUZZING_BRANCH` | The branch for long-running fuzzing jobs. The default is `master`. | +| `COVFUZZ_BRANCH` | The branch for long-running fuzzing jobs. The default is `master`. | +| `COVFUZZ_SEED_CORPUS` | Path to a seed corpus directory. The default is empty. | +| `COVFUZZ_URL_PREFIX` | Path to the `gitlab-cov-fuzz` repository cloned for use with an offline environment. You should only change this when using an offline environment. The default value is `https://gitlab.com/gitlab-org/security-products/analyzers/gitlab-cov-fuzz/-/raw`. | + +The files in the seed corpus (`COVFUZZ_SEED_CORPUS`), if provided, aren't updated unless you commit new +files to your Git repository. There's usually no need to frequently update the seed corpus. As part +of the GitLab artifacts system, GitLab saves in a corpus directory the new test cases that every run +generates. In any subsequent runs, GitLab also reuses the generated corpus together with the seed +corpus. + +### Reports JSON format + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/220062) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.3 as an [Alpha feature](https://about.gitlab.com/handbook/product/gitlab-the-product/#alpha). + +The `gitlab-cov-fuzz` tool emits a JSON report file. For more information, see the +[schema for this report](https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/blob/master/dist/coverage-fuzzing-report-format.json). + +You can download the JSON report file from the CI pipelines page. For more information, see +[Downloading artifacts](../../../ci/pipelines/job_artifacts.md#downloading-artifacts). + +Here's an example coverage fuzzing report: + +```json-doc +{ + "version": "v1.0.8", + "regression": false, + "exit_code": -1, + "vulnerabilities": [ + { + "category": "coverage_fuzzing", + "message": "Heap-buffer-overflow\nREAD 1", + "description": "Heap-buffer-overflow\nREAD 1", + "severity": "Critical", + "stacktrace_snippet": "INFO: Seed: 3415817494\nINFO: Loaded 1 modules (7 inline 8-bit counters): 7 [0x10eee2470, 0x10eee2477), \nINFO: Loaded 1 PC tables (7 PCs): 7 [0x10eee2478,0x10eee24e8), \nINFO: 5 files found in corpus\nINFO: -max_len is not provided; libFuzzer will not generate inputs larger than 4096 bytes\nINFO: seed corpus: files: 5 min: 1b max: 4b total: 14b rss: 26Mb\n#6\tINITED cov: 7 ft: 7 corp: 5/14b exec/s: 0 rss: 26Mb\n=================================================================\n==43405==ERROR: AddressSanitizer: heap-buffer-overflow on address 0x602000001573 at pc 0x00010eea205a bp 0x7ffee0d5e090 sp 0x7ffee0d5e088\nREAD of size 1 at 0x602000001573 thread T0\n #0 0x10eea2059 in FuzzMe(unsigned char const*, unsigned long) fuzz_me.cc:9\n #1 0x10eea20ba in LLVMFuzzerTestOneInput fuzz_me.cc:13\n #2 0x10eebe020 in fuzzer::Fuzzer::ExecuteCallback(unsigned char const*, unsigned long) FuzzerLoop.cpp:556\n #3 0x10eebd765 in fuzzer::Fuzzer::RunOne(unsigned char const*, unsigned long, bool, fuzzer::InputInfo*, bool*) FuzzerLoop.cpp:470\n #4 0x10eebf966 in fuzzer::Fuzzer::MutateAndTestOne() FuzzerLoop.cpp:698\n #5 0x10eec0665 in fuzzer::Fuzzer::Loop(std::__1::vector\u003cfuzzer::SizedFile, fuzzer::fuzzer_allocator\u003cfuzzer::SizedFile\u003e \u003e\u0026) FuzzerLoop.cpp:830\n #6 0x10eead0cd in fuzzer::FuzzerDriver(int*, char***, int (*)(unsigned char const*, unsigned long)) FuzzerDriver.cpp:829\n #7 0x10eedaf82 in main FuzzerMain.cpp:19\n #8 0x7fff684fecc8 in start+0x0 (libdyld.dylib:x86_64+0x1acc8)\n\n0x602000001573 is located 0 bytes to the right of 3-byte region [0x602000001570,0x602000001573)\nallocated by thread T0 here:\n #0 0x10ef92cfd in wrap__Znam+0x7d (libclang_rt.asan_osx_dynamic.dylib:x86_64+0x50cfd)\n #1 0x10eebdf31 in fuzzer::Fuzzer::ExecuteCallback(unsigned char const*, unsigned long) FuzzerLoop.cpp:541\n #2 0x10eebd765 in fuzzer::Fuzzer::RunOne(unsigned char const*, unsigned long, bool, fuzzer::InputInfo*, bool*) FuzzerLoop.cpp:470\n #3 0x10eebf966 in fuzzer::Fuzzer::MutateAndTestOne() FuzzerLoop.cpp:698\n #4 0x10eec0665 in fuzzer::Fuzzer::Loop(std::__1::vector\u003cfuzzer::SizedFile, fuzzer::fuzzer_allocator\u003cfuzzer::SizedFile\u003e \u003e\u0026) FuzzerLoop.cpp:830\n #5 0x10eead0cd in fuzzer::FuzzerDriver(int*, char***, int (*)(unsigned char const*, unsigned long)) FuzzerDriver.cpp:829\n #6 0x10eedaf82 in main FuzzerMain.cpp:19\n #7 0x7fff684fecc8 in start+0x0 (libdyld.dylib:x86_64+0x1acc8)\n\nSUMMARY: AddressSanitizer: heap-buffer-overflow fuzz_me.cc:9 in FuzzMe(unsigned char const*, unsigned long)\nShadow bytes around the buggy address:\n 0x1c0400000250: fa fa fd fa fa fa fd fa fa fa fd fa fa fa fd fa\n 0x1c0400000260: fa fa fd fa fa fa fd fa fa fa fd fa fa fa fd fa\n 0x1c0400000270: fa fa fd fa fa fa fd fa fa fa fd fa fa fa fd fa\n 0x1c0400000280: fa fa fd fa fa fa fd fa fa fa fd fa fa fa fd fa\n 0x1c0400000290: fa fa fd fa fa fa fd fa fa fa fd fa fa fa fd fa\n=\u003e0x1c04000002a0: fa fa fd fa fa fa fd fa fa fa fd fa fa fa[03]fa\n 0x1c04000002b0: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa\n 0x1c04000002c0: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa\n 0x1c04000002d0: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa\n 0x1c04000002e0: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa\n 0x1c04000002f0: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa\nShadow byte legend (one shadow byte represents 8 application bytes):\n Addressable: 00\n Partially addressable: 01 02 03 04 05 06 07 \n Heap left redzone: fa\n Freed heap region: fd\n Stack left redzone: f1\n Stack mid redzone: f2\n Stack right redzone: f3\n Stack after return: f5\n Stack use after scope: f8\n Global redzone: f9\n Global init order: f6\n Poisoned by user: f7\n Container overflow: fc\n Array cookie: ac\n Intra object redzone: bb\n ASan internal: fe\n Left alloca redzone: ca\n Right alloca redzone: cb\n Shadow gap: cc\n==43405==ABORTING\nMS: 1 EraseBytes-; base unit: de3a753d4f1def197604865d76dba888d6aefc71\n0x46,0x55,0x5a,\nFUZ\nartifact_prefix='./crashes/'; Test unit written to ./crashes/crash-0eb8e4ed029b774d80f2b66408203801cb982a60\nBase64: RlVa\nstat::number_of_executed_units: 122\nstat::average_exec_per_sec: 0\nstat::new_units_added: 0\nstat::slowest_unit_time_sec: 0\nstat::peak_rss_mb: 28", + "scanner": { + "id": "libFuzzer", + "name": "libFuzzer" + }, + "location": { + "crash_address": "0x602000001573", + "crash_state": "FuzzMe\nstart\nstart+0x0\n\n", + "crash_type": "Heap-buffer-overflow\nREAD 1" + }, + "tool": "libFuzzer" + } + ] +} +``` ### Additional Configuration @@ -107,6 +162,17 @@ The `gitlab-cov-fuzz` command passes all arguments it receives to the underlying can therefore use all the options available in that fuzzing engine. For more information on these options, see the underlying fuzzing engine's documentation. +### Offline Environment + +To use coverage fuzzing in an offline environment, follow these steps: + +1. Clone [`gitlab-cov-fuzz`](https://gitlab.com/gitlab-org/security-products/analyzers/gitlab-cov-fuzz) + to a private repository that your offline GitLab instance can access. + +1. For each fuzzing step, set `COVFUZZ_URL_PREFIX` to `${NEW_URL_GITLAB_COV_FUZ}/-/raw`, where + `NEW_URL_GITLAB_COV_FUZ` is the URL of the private `gitlab-cov-fuzz` clone that you set up in the + first step. + ### Glossary - Seed corpus: The set of test cases given as initial input to the fuzz target. This usually speeds diff --git a/doc/user/application_security/dast/img/dast_on_demand_v13_2.png b/doc/user/application_security/dast/img/dast_on_demand_v13_2.png Binary files differindex 8a733c27be1..045221d713c 100644 --- a/doc/user/application_security/dast/img/dast_on_demand_v13_2.png +++ b/doc/user/application_security/dast/img/dast_on_demand_v13_2.png diff --git a/doc/user/application_security/dast/index.md b/doc/user/application_security/dast/index.md index d68928d858b..b2020d48d38 100644 --- a/doc/user/application_security/dast/index.md +++ b/doc/user/application_security/dast/index.md @@ -48,16 +48,16 @@ uses the popular open source tool [OWASP Zed Attack Proxy](https://www.zaproxy.o to perform an analysis on your running web application. By default, DAST executes [ZAP Baseline Scan](https://www.zaproxy.org/docs/docker/baseline-scan/) -and performs passive scanning only. It won't actively attack your application. +and performs passive scanning only. It doesn't actively attack your application. However, DAST can be [configured](#full-scan) to also perform an *active scan*: attack your application and produce a more extensive security report. It can be very useful combined with [Review Apps](../../../ci/review_apps/index.md). NOTE: **Note:** A pipeline may consist of multiple jobs, including SAST and DAST scanning. If any -job fails to finish for any reason, the security dashboard won't show DAST scanner +job fails to finish for any reason, the security dashboard doesn't show DAST scanner output. For example, if the DAST job finishes but the SAST job fails, the security -dashboard won't show DAST results. The analyzer will output an +dashboard doesn't show DAST results. The analyzer outputs an [exit code](../../../development/integrations/secure.md#exit-code) on failure. ## Use cases @@ -112,7 +112,7 @@ always take the latest DAST artifact available. Behind the scenes, the [GitLab DAST Docker image](https://gitlab.com/gitlab-org/security-products/dast) is used to run the tests on the specified URL and scan it for possible vulnerabilities. -By default, the DAST template will use the latest major version of the DAST Docker +By default, the DAST template uses the latest major version of the DAST Docker image. Using the `DAST_VERSION` variable, you can choose how DAST updates: - Automatically update DAST with new features and fixes by pinning to a major version (such as `1`). @@ -163,7 +163,7 @@ headers whose values you want masked. For details on how to mask headers, see It's also possible to authenticate the user before performing the DAST checks. -Create masked variables to pass the credentials that DAST will use. +Create masked variables to pass the credentials that DAST uses. To create masked variables for the username and password, see [Create a custom variable in the UI](../../../ci/variables/README.md#create-a-custom-variable-in-the-ui). Note that the key of the username variable must be `DAST_USERNAME` and the key of the password variable must be `DAST_PASSWORD`. @@ -182,7 +182,7 @@ variables: DAST_AUTH_EXCLUDE_URLS: http://example.com/sign-out,http://example.com/sign-out-2 # optional, URLs to skip during the authenticated scan; comma-separated, no spaces in between ``` -The results will be saved as a +The results are saved as a [DAST report artifact](../../../ci/pipelines/job_artifacts.md#artifactsreportsdast-ultimate) that you can later download and analyze. Due to implementation limitations, we always take the latest DAST artifact available. @@ -227,10 +227,10 @@ variables: Since ZAP full scan actively attacks the target application, DAST sends a ping to the target (normally defined in `DAST_WEBSITE` or `environment_url.txt`) beforehand. -- If `DAST_FULL_SCAN_DOMAIN_VALIDATION_REQUIRED` is `false` or unset, the scan will - proceed unless the response to the ping includes a `Gitlab-DAST-Permission` +- If `DAST_FULL_SCAN_DOMAIN_VALIDATION_REQUIRED` is `false` or unset, the scan + proceeds unless the response to the ping includes a `Gitlab-DAST-Permission` header with a value of `deny`. -- If `DAST_FULL_SCAN_DOMAIN_VALIDATION_REQUIRED` is `true`, the scan will exit +- If `DAST_FULL_SCAN_DOMAIN_VALIDATION_REQUIRED` is `true`, the scan exits unless the response to the ping includes a `Gitlab-DAST-Permission` header with a value of `allow`. @@ -434,7 +434,7 @@ variables: ``` Because the template is [evaluated before](../../../ci/yaml/README.md#include) the pipeline -configuration, the last mention of the variable will take precedence. +configuration, the last mention of the variable takes precedence. ### Available variables @@ -445,24 +445,24 @@ DAST can be [configured](#customizing-the-dast-settings) using environment varia | `SECURE_ANALYZERS_PREFIX` | URL | Set the Docker registry base address from which to download the analyzer. | | `DAST_WEBSITE` | URL | The URL of the website to scan. `DAST_API_SPECIFICATION` must be specified if this is omitted. | | `DAST_API_SPECIFICATION` | URL or string | The API specification to import. The specification can be hosted at a URL, or the name of a file present in the `/zap/wrk` directory. `DAST_WEBSITE` must be specified if this is omitted. | -| `DAST_AUTH_URL` | URL | The URL of the page containing the sign-in HTML form on the target website. `DAST_USERNAME` and `DAST_PASSWORD` will be submitted with the login form to create an authenticated scan. Not supported for API scans. | +| `DAST_AUTH_URL` | URL | The URL of the page containing the sign-in HTML form on the target website. `DAST_USERNAME` and `DAST_PASSWORD` are submitted with the login form to create an authenticated scan. Not supported for API scans. | | `DAST_USERNAME` | string | The username to authenticate to in the website. | | `DAST_PASSWORD` | string | The password to authenticate to in the website. | | `DAST_USERNAME_FIELD` | string | The name of username field at the sign-in HTML form. | | `DAST_PASSWORD_FIELD` | string | The name of password field at the sign-in HTML form. | -| `DAST_MASK_HTTP_HEADERS` | string | Comma-separated list of request and response headers to be masked (introduced in GitLab 13.1). Must contain **all** headers to be masked. Refer to [list of headers that are masked by default](#hide-sensitive-information). | +| `DAST_MASK_HTTP_HEADERS` | string | Comma-separated list of request and response headers to be masked (GitLab 13.1). Must contain **all** headers to be masked. Refer to [list of headers that are masked by default](#hide-sensitive-information). | | `DAST_AUTH_EXCLUDE_URLS` | URLs | The URLs to skip during the authenticated scan; comma-separated, no spaces in between. Not supported for API scans. | | `DAST_FULL_SCAN_ENABLED` | boolean | Set to `true` to run a [ZAP Full Scan](https://github.com/zaproxy/zaproxy/wiki/ZAP-Full-Scan) instead of a [ZAP Baseline Scan](https://github.com/zaproxy/zaproxy/wiki/ZAP-Baseline-Scan). Default: `false` | | `DAST_FULL_SCAN_DOMAIN_VALIDATION_REQUIRED` | boolean | Set to `true` to require [domain validation](#domain-validation) when running DAST full scans. Not supported for API scans. Default: `false` | | `DAST_AUTO_UPDATE_ADDONS` | boolean | ZAP add-ons are pinned to specific versions in the DAST Docker image. Set to `true` to download the latest versions when the scan starts. Default: `false` | | `DAST_API_HOST_OVERRIDE` | string | Used to override domains defined in API specification files. Example: `example.com:8080` | | `DAST_EXCLUDE_RULES` | string | Set to a comma-separated list of Vulnerability Rule IDs to exclude them from running during the scan. Rule IDs are numbers and can be found from the DAST log or on the [ZAP project](https://github.com/zaproxy/zaproxy/blob/develop/docs/scanners.md). For example, `HTTP Parameter Override` has a rule ID of `10026`. **Note:** In earlier versions of GitLab the excluded rules were executed but alerts they generated were supressed. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/118641) in GitLab 12.10. | -| `DAST_REQUEST_HEADERS` | string | Set to a comma-separated list of request header names and values. Headers will be added to every request made by DAST. For example, `Cache-control: no-cache,User-Agent: DAST/1.0` | +| `DAST_REQUEST_HEADERS` | string | Set to a comma-separated list of request header names and values. Headers are added to every request made by DAST. For example, `Cache-control: no-cache,User-Agent: DAST/1.0` | | `DAST_DEBUG` | boolean | Enable debug message output. Default: `false` | | `DAST_SPIDER_MINS` | number | The maximum duration of the spider scan in minutes. Set to `0` for unlimited. Default: One minute, or unlimited when the scan is a full scan. | -| `DAST_HTML_REPORT` | string | The file name of the HTML report written at the end of a scan. | -| `DAST_MARKDOWN_REPORT` | string | The file name of the Markdown report written at the end of a scan. | -| `DAST_XML_REPORT` | string | The file name of the XML report written at the end of a scan. | +| `DAST_HTML_REPORT` | string | The filename of the HTML report written at the end of a scan. | +| `DAST_MARKDOWN_REPORT` | string | The filename of the Markdown report written at the end of a scan. | +| `DAST_XML_REPORT` | string | The filename of the XML report written at the end of a scan. | | `DAST_INCLUDE_ALPHA_VULNERABILITIES` | boolean | Set to `true` to include alpha passive and active scan rules. Default: `false` | | `DAST_USE_AJAX_SPIDER` | boolean | Set to `true` to use the AJAX spider in addition to the traditional spider, useful for crawling sites that require JavaScript. Default: `false` | | `DAST_ZAP_CLI_OPTIONS` | string | ZAP server command-line options. For example, `-Xmx3072m` would set the Java maximum memory allocation pool size. | @@ -472,7 +472,7 @@ DAST can be [configured](#customizing-the-dast-settings) using environment varia Not all DAST configuration is available via environment variables. To find out all possible options, run the following configuration. -Available command-line options will be printed to the job log: +Available command-line options are printed to the job log: ```yaml include: @@ -526,7 +526,7 @@ A DAST job has two executing processes: - A series of scripts that start, control and stop the ZAP server. Debug mode of the scripts can be enabled by using the `DAST_DEBUG` environment variable. This can help when troubleshooting the job, -and will output statements indicating what percentage of the scan is complete. +and outputs statements indicating what percentage of the scan is complete. For details on using variables, see [Overriding the DAST template](#customizing-the-dast-settings). Debug mode of the ZAP server can be enabled using the `DAST_ZAP_LOG_CONFIGURATION` environment variable. @@ -603,24 +603,76 @@ Alternatively, you can use the variable `SECURE_ANALYZERS_PREFIX` to override th ## On-Demand Scans > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/218465) in GitLab 13.2. -> - It's deployed behind a feature flag, disabled by default. -> - It's disabled on GitLab.com. +> - [Improved](https://gitlab.com/gitlab-org/gitlab/-/issues/218465) in GitLab 13.3. +> - It's deployed behind a feature flag, enabled by default. +> - It's enabled on GitLab.com. > - It's able to be enabled or disabled per-project. > - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enable-or-disable-on-demand-scans). -Passive DAST scans may be run on demand against a target website, outside the DevOps lifecycle. These scans will -always be associated with the default or `master` branch of your project and the results can be seen in the project dashboard. +You can run a passive DAST scan against a target website, outside the DevOps lifecycle. These scans +are always associated with the default branch of your project and the results are available in the +project dashboard. + +### Site profile + +An on-demand scan requires a site profile, which includes a profile name and target URL. The profile +name allows you to describe the site to be scanned. The target URL specifies the URL against which +the DAST scan is run. + +### Run an on-demand scan + +NOTE: **Note:** +You must have permission to run an on-demand DAST scan against a protected branch. +The default branch is automatically protected. For more details, see [Pipeline security on protected branches](../../../ci/pipelines/index.md#pipeline-security-on-protected-branches). + +Running an on-demand scan requires an existing site profile. If a site profile for the target URL +doesn't exist, first [create a site profile](#create-a-site-profile). An on-demand DAST scan has +a fixed timeout of 60 seconds. + +- Navigate to your project's home page, then click **On-demand Scans** in the left sidebar. +- Click **Create new DAST scan**. +- Select a site profile from the profiles dropdown. +- Click **Run scan**. + +#### Create a site profile - +- Navigate to your project's home page, then click **On-demand Scans** in the left sidebar. +- Click **Create new DAST scan**. +- Click **New Site Profile**. +- Type in a unique **Profile name** and **Target URL** then click **Save profile**. -### Enable or disable On-Demand Scans +#### Delete a site profile + +- Navigate to your project's home page, then click **On-demand Scans** in the left sidebar. +- Click **Create new DAST scan**. +- Click **Delete** in the matching site profile's row. + +### Enable or disable On-demand Scans and site profiles + +On-demand Scans with site profiles is enabled by default. You can disable On-demand Scans +instance-wide, or disable it for specific projects if you prefer. DAST site profiles are not +available if the On-demand Scans feature is disabled. + +Use of On-demand Scans with site profiles requires **both** the following feature flags enabled: + +- security_on_demand_scans_feature_flag +- security_on_demand_scans_site_profiles_feature_flag -On-Demand Scans is under development and not ready for production use. It is -deployed behind a feature flag that is **disabled by default**. [GitLab administrators with access to the GitLab Rails console](../../../administration/feature_flags.md) -can enable it for your instance. On-Demand Scans can be enabled or disabled per-project +can disable or enable the feature flags. + +#### Enable or disable On-demand Scans + +To disable On-demand Scans: + +```ruby +# Instance-wide +Feature.disable(:security_on_demand_scans_feature_flag) +# or by project +Feature.disable(:security_on_demand_scans_feature_flag, Project.find(<project id>)) +``` -To enable it: +To enable On-demand Scans: ```ruby # Instance-wide @@ -629,13 +681,29 @@ Feature.enable(:security_on_demand_scans_feature_flag) Feature.enable(:security_on_demand_scans_feature_flag, Project.find(<project id>)) ``` -To disable it: +#### Enable or disable site profiles + +The Site Profiles feature is enabled instance-wide by default. You can disable it instance-wide, or disable it +for specific projects if you prefer. +[GitLab administrators with access to the GitLab Rails console](../../../administration/feature_flags.md) +can disable or enable the feature flag. + +To disable Site Profiles: ```ruby # Instance-wide -Feature.disable(:security_on_demand_scans_feature_flag) +Feature.disable(:security_on_demand_scans_site_profiles_feature_flag) # or by project -Feature.disable(:security_on_demand_scans_feature_flag, Project.find(<project id>)) +Feature.disable(:security_on_demand_scans_site_profiles_feature_flag, Project.find(<project id>)) +``` + +To enable Site Profiles: + +```ruby +# Instance-wide +Feature.enable(:security_on_demand_scans_site_profiles_feature_flag) +# or by project +Feature.enable(:security_on_demand_scans_site_profiles_feature_flag, Project.find(<project id>)) ``` ## Reports @@ -719,7 +787,7 @@ For more information about the vulnerabilities database update, check the ## Optimizing DAST -By default, DAST will download all artifacts defined by previous jobs in the pipeline. If +By default, DAST downloads all artifacts defined by previous jobs in the pipeline. If your DAST job does not rely on `environment_url.txt` to define the URL under test or any other files created in previous jobs, we recommend you don't download artifacts. To avoid downloading artifacts, add the following to your `gitlab-ci.yml` file: diff --git a/doc/user/application_security/dependency_scanning/analyzers.md b/doc/user/application_security/dependency_scanning/analyzers.md index ca2b212ffc3..d41f9441464 100644 --- a/doc/user/application_security/dependency_scanning/analyzers.md +++ b/doc/user/application_security/dependency_scanning/analyzers.md @@ -7,7 +7,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Dependency Scanning Analyzers **(ULTIMATE)** -Dependency Scanning relies on underlying third party tools that are wrapped into +Dependency Scanning relies on underlying third-party tools that are wrapped into what we call "Analyzers". An analyzer is a [dedicated project](https://gitlab.com/gitlab-org/security-products/analyzers) that wraps a particular tool to: @@ -26,7 +26,7 @@ Dependency Scanning supports the following official analyzers: - [`gemnasium-python`](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-python) - [`retire.js`](https://gitlab.com/gitlab-org/security-products/analyzers/retire.js) -The analyzers are published as Docker images that Dependency Scanning will use +The analyzers are published as Docker images, which Dependency Scanning uses to launch dedicated containers for each analysis. Dependency Scanning is pre-configured with a set of **default images** that are @@ -70,12 +70,12 @@ variables: DS_DEFAULT_ANALYZERS: "bundler-audit,gemnasium" ``` -`bundler-audit` runs first. When merging the reports, Dependency Scanning will -remove the duplicates and will keep the `bundler-audit` entries. +`bundler-audit` runs first. When merging the reports, Dependency Scanning +removes the duplicates and keeps the `bundler-audit` entries. ### Disabling default analyzers -Setting `DS_DEFAULT_ANALYZERS` to an empty string will disable all the official +Setting `DS_DEFAULT_ANALYZERS` to an empty string disables all the official default analyzers. In `.gitlab-ci.yml` define: ```yaml @@ -158,8 +158,8 @@ The following table lists the data available for each official analyzer. | Credits | ✓ | 𐄂 | 𐄂 | - ✓ => we have that data -- ⚠ => we have that data but it's partially reliable, or we need to extract that data from unstructured content -- 𐄂 => we don't have that data or it would need to develop specific or inefficient/unreliable logic to obtain it. +- ⚠ => we have that data, but it's partially reliable, or we need to extract that data from unstructured content +- 𐄂 => we don't have that data, or it would need to develop specific or inefficient/unreliable logic to obtain it. -The values provided by these tools are heterogeneous so they are sometimes +The values provided by these tools are heterogeneous, so they are sometimes normalized into common values (e.g., `severity`, `confidence`, etc). diff --git a/doc/user/application_security/dependency_scanning/index.md b/doc/user/application_security/dependency_scanning/index.md index 57b4fae3230..6b14f93735b 100644 --- a/doc/user/application_security/dependency_scanning/index.md +++ b/doc/user/application_security/dependency_scanning/index.md @@ -9,9 +9,9 @@ info: To determine the technical writer assigned to the Stage/Group associated w > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/5105) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 10.7. -Dependency Scanning helps to automatically find security vulnerabilities in your dependencies +Dependency Scanning helps to find security vulnerabilities in your dependencies automatically while you're developing and testing your applications, such as when your -application is using an external (open source) library which is known to be vulnerable. +application is using an external (open source) library that is known to be vulnerable. ## Overview @@ -60,6 +60,7 @@ The following languages and dependency managers are supported: | Language (package managers) | Supported files | Scan tool(s) | |----------------------------- | --------------- | ------------ | +| C# .NET ([NuGet](https://www.nuget.org/) 4.9+) | [`packages.lock.json`](https://docs.microsoft.com/en-us/nuget/consume-packages/package-references-in-project-files#enabling-lock-file) | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | | Java ([Gradle](https://gradle.org/), [Maven](https://maven.apache.org/)) | `build.gradle`, `build.gradle.kts`, `pom.xml` | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | | JavaScript ([npm](https://www.npmjs.com/), [yarn](https://classic.yarnpkg.com/en/)) | `package-lock.json`, `npm-shrinkwrap.json`, `yarn.lock` | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium), [Retire.js](https://retirejs.github.io/retire.js/) | | Go ([Golang](https://golang.org/)) | `go.sum` | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | @@ -84,7 +85,7 @@ The [Security Scanner Integration](../../../development/integrations/secure.md) To enable Dependency Scanning for GitLab 11.9 and later, you must [include](../../../ci/yaml/README.md#includetemplate) the [`Dependency-Scanning.gitlab-ci.yml` template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml) -that's provided as a part of your GitLab installation. +that is provided as a part of your GitLab installation. For GitLab versions earlier than 11.9, you can copy and use the job as defined that template. @@ -95,9 +96,9 @@ include: - template: Dependency-Scanning.gitlab-ci.yml ``` -The included template will create Dependency Scanning jobs in your CI/CD -pipeline and scan your project's source code for possible vulnerabilities. -The results will be saved as a +The included template creates Dependency Scanning jobs in your CI/CD +pipeline and scans your project's source code for possible vulnerabilities. +The results are saved as a [Dependency Scanning report artifact](../../../ci/pipelines/job_artifacts.md#artifactsreportsdependency_scanning-ultimate) that you can later download and analyze. Due to implementation limitations, we always take the latest Dependency Scanning artifact available. @@ -117,7 +118,7 @@ variables: ``` Because template is [evaluated before](../../../ci/yaml/README.md#include) the pipeline -configuration, the last mention of the variable will take precedence. +configuration, the last mention of the variable takes precedence. ### Overriding Dependency Scanning jobs @@ -155,7 +156,7 @@ The following variables allow configuration of global dependency scanning settin | `DS_DISABLE_DIND` | Disable Docker-in-Docker and run analyzers [individually](#enabling-docker-in-docker). This variable is `true` by default. | | `ADDITIONAL_CA_CERT_BUNDLE` | Bundle of CA certs to trust. | | `DS_EXCLUDED_PATHS` | Exclude vulnerabilities from output based on the paths. A comma-separated list of patterns. Patterns can be globs, or file or folder paths (for example, `doc,spec`). Parent directories also match patterns. Default: `"spec, test, tests, tmp"` | -| `SECURE_LOG_LEVEL` | Default log level is `info`, you can set it to any of the following strings: `fatal`, `error`, `warn`, `info`, `debug`. | +| `SECURE_LOG_LEVEL` | Set the minimum logging level. Messages of this logging level or higher are output. From highest to lowest severity, the logging levels are: `fatal`, `error`, `warn`, `info`, `debug`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/10880) in GitLab 13.1. Default: `info` | #### Configuring Docker-in-Docker orchestrator @@ -186,10 +187,10 @@ The following variables are used for configuring specific analyzers (used for a | `DS_PIP_VERSION` | `gemnasium-python` | | Force the install of a specific pip version (example: `"19.3"`), otherwise the pip installed in the Docker image is used. ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12811) in GitLab 12.7) | | `DS_PIP_DEPENDENCY_PATH` | `gemnasium-python` | | Path to load Python pip dependencies from. ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12412) in GitLab 12.2) | | `DS_PYTHON_VERSION` | `retire.js` | | Version of Python. If set to 2, dependencies are installed using Python 2.7 instead of Python 3.6. ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12296) in GitLab 12.1)| -| `DS_JAVA_VERSION` | `gemnasium-maven` | `11` | Version of Java. Available versions: `8`, `11`, `13`, `14`. Maven and Gradle will use the Java version specified by this value. | -| `MAVEN_CLI_OPTS` | `gemnasium-maven` | `"-DskipTests --batch-mode"` | List of command line arguments that will be passed to `maven` by the analyzer. See an example for [using private repositories](../index.md#using-private-maven-repos). | -| `GRADLE_CLI_OPTS` | `gemnasium-maven` | | List of command line arguments that will be passed to `gradle` by the analyzer. | -| `SBT_CLI_OPTS` | `gemnasium-maven` | | List of command-line arguments that the analyzer will pass to `sbt`. | +| `DS_JAVA_VERSION` | `gemnasium-maven` | `11` | Version of Java. Available versions: `8`, `11`, `13`, `14`. Maven and Gradle use the Java version specified by this value. | +| `MAVEN_CLI_OPTS` | `gemnasium-maven` | `"-DskipTests --batch-mode"` | List of command line arguments that are passed to `maven` by the analyzer. See an example for [using private repositories](../index.md#using-private-maven-repos). | +| `GRADLE_CLI_OPTS` | `gemnasium-maven` | | List of command line arguments that are passed to `gradle` by the analyzer. | +| `SBT_CLI_OPTS` | `gemnasium-maven` | | List of command-line arguments that the analyzer passes to `sbt`. | | `BUNDLER_AUDIT_UPDATE_DISABLED` | `bundler-audit` | `"false"` | Disable automatic updates for the `bundler-audit` analyzer. Useful if you're running Dependency Scanning in an offline, air-gapped environment.| | `BUNDLER_AUDIT_ADVISORY_DB_URL` | `bundler-audit` | `https://github.com/rubysec/ruby-advisory-db` | URL of the advisory database used by bundler-audit. | | `BUNDLER_AUDIT_ADVISORY_DB_REF_NAME` | `bundler-audit` | `master` | Git ref for the advisory database specified by `BUNDLER_AUDIT_ADVISORY_DB_URL`. | @@ -310,7 +311,7 @@ Here's an example Dependency Scanning report: "category": "dependency_scanning", "name": "Authentication bypass via incorrect DOM traversal and canonicalization", "message": "Authentication bypass via incorrect DOM traversal and canonicalization in saml2-js", - "description": "Some XML DOM traversal and canonicalization APIs may be inconsistent in handling of comments within XML nodes. Incorrect use of these APIs by some SAML libraries results in incorrect parsing of the inner text of XML nodes such that any inner text after the comment is lost prior to cryptographically signing the SAML message. Text after the comment therefore has no impact on the signature on the SAML message.\r\n\r\nA remote attacker can modify SAML content for a SAML service provider without invalidating the cryptographic signature, which may allow attackers to bypass primary authentication for the affected SAML service provider.", + "description": "Some XML DOM traversal and canonicalization APIs may be inconsistent in handling of comments within XML nodes. Incorrect use of these APIs by some SAML libraries results in incorrect parsing of the inner text of XML nodes such that any inner text after the comment is lost prior to cryptographically signing the SAML message. Text after the comment, therefore, has no impact on the signature on the SAML message.\r\n\r\nA remote attacker can modify SAML content for a SAML service provider without invalidating the cryptographic signature, which may allow attackers to bypass primary authentication for the affected SAML service provider.", "severity": "Unknown", "solution": "Upgrade to fixed version.\r\n", "scanner": { @@ -390,7 +391,9 @@ Here are the requirements for using Dependency Scanning in an offline environmen - Keep Docker-In-Docker disabled (default). - GitLab Runner with the [`docker` or `kubernetes` executor](#requirements). - Docker Container Registry with locally available copies of Dependency Scanning [analyzer](https://gitlab.com/gitlab-org/security-products/analyzers) images. -- Host an offline Git copy of the [gemnasium-db advisory database](https://gitlab.com/gitlab-org/security-products/gemnasium-db/) +- Host an offline Git copy of the [gemnasium-db advisory database](https://gitlab.com/gitlab-org/security-products/gemnasium-db/). + This is required because, in an offline environment, the Gemnasium analyzer can't fetch the latest + advisories from the online repository. - _Only if scanning Ruby projects_: Host an offline Git copy of the [advisory database](https://github.com/rubysec/ruby-advisory-db). - _Only if scanning npm/yarn projects_: Host an offline copy of the [retire.js](https://github.com/RetireJS/retire.js/) [node](https://github.com/RetireJS/retire.js/blob/master/repository/npmrepository.json) and [js](https://github.com/RetireJS/retire.js/blob/master/repository/jsrepository.json) advisory databases. @@ -428,8 +431,10 @@ For details on saving and transporting Docker images as a file, see Docker's doc ### Set Dependency Scanning CI job variables to use local Dependency Scanning analyzers -Add the following configuration to your `.gitlab-ci.yml` file. You must replace -`SECURE_ANALYZERS_PREFIX` to refer to your local Docker container registry: +Add the following configuration to your `.gitlab-ci.yml` file. You must change the value of +`SECURE_ANALYZERS_PREFIX` to refer to your local Docker container registry. You must also change the +value of `GEMNASIUM_DB_REMOTE_URL` to the location of your offline Git copy of the +[gemnasium-db advisory database](https://gitlab.com/gitlab-org/security-products/gemnasium-db/): ```yaml include: diff --git a/doc/user/application_security/img/adding_a_dismissal_reason_v13_0.png b/doc/user/application_security/img/adding_a_dismissal_reason_v13_0.png Binary files differindex 385236d08f2..cb8911b14b1 100644 --- a/doc/user/application_security/img/adding_a_dismissal_reason_v13_0.png +++ b/doc/user/application_security/img/adding_a_dismissal_reason_v13_0.png diff --git a/doc/user/application_security/img/interacting_with_vulnerability_v13_0.png b/doc/user/application_security/img/interacting_with_vulnerability_v13_0.png Binary files differindex 866ad74d42c..19d47712f9e 100644 --- a/doc/user/application_security/img/interacting_with_vulnerability_v13_0.png +++ b/doc/user/application_security/img/interacting_with_vulnerability_v13_0.png diff --git a/doc/user/application_security/img/vulnerability-check_v13_0.png b/doc/user/application_security/img/vulnerability-check_v13_0.png Binary files differindex 536fc4f10f7..9f0bd0f759b 100644 --- a/doc/user/application_security/img/vulnerability-check_v13_0.png +++ b/doc/user/application_security/img/vulnerability-check_v13_0.png diff --git a/doc/user/application_security/img/vulnerability_related_issues_add_button_v13_2.png b/doc/user/application_security/img/vulnerability_related_issues_add_button_v13_2.png Binary files differnew file mode 100644 index 00000000000..10d9effb811 --- /dev/null +++ b/doc/user/application_security/img/vulnerability_related_issues_add_button_v13_2.png diff --git a/doc/user/application_security/img/vulnerability_related_issues_remove_v13_2.gif b/doc/user/application_security/img/vulnerability_related_issues_remove_v13_2.gif Binary files differnew file mode 100644 index 00000000000..22acba5fe1e --- /dev/null +++ b/doc/user/application_security/img/vulnerability_related_issues_remove_v13_2.gif diff --git a/doc/user/application_security/img/vulnerability_related_issues_text_box_tags_v13_2.gif b/doc/user/application_security/img/vulnerability_related_issues_text_box_tags_v13_2.gif Binary files differnew file mode 100644 index 00000000000..562ffe7e329 --- /dev/null +++ b/doc/user/application_security/img/vulnerability_related_issues_text_box_tags_v13_2.gif diff --git a/doc/user/application_security/index.md b/doc/user/application_security/index.md index 3aca4c59423..c003b512808 100644 --- a/doc/user/application_security/index.md +++ b/doc/user/application_security/index.md @@ -45,6 +45,12 @@ To add Container Scanning, follow the steps listed in the [Container Scanning do To further configure any of the other scanners, refer to each scanner's documentation. +### SAST configuration + +You can set up and configure Static Application Security Testing +(SAST) for your project, without opening a text editor. For more details, +see [configure SAST in the UI](sast/index.md#configure-sast-in-the-ui). + ### Override the default registry base address By default, GitLab security scanners use `registry.gitlab.com/gitlab-org/security-products/analyzers` as the @@ -61,9 +67,10 @@ GitLab uses the following tools to scan and report known vulnerabilities found i | [Dependency List](dependency_list/index.md) **(ULTIMATE)** | View your project's dependencies and their known vulnerabilities. | | [Dependency Scanning](dependency_scanning/index.md) **(ULTIMATE)** | Analyze your dependencies for known vulnerabilities. | | [Dynamic Application Security Testing (DAST)](dast/index.md) **(ULTIMATE)** | Analyze running web applications for known vulnerabilities. | -| [Secret Detection](secret_detection/index.md) **(ULTIMATE)** | Analyze Git history for leaked secrets. | +| [Secret Detection](secret_detection/index.md) **(ULTIMATE)** | Analyze Git history for leaked secrets. | | [Security Dashboard](security_dashboard/index.md) **(ULTIMATE)** | View vulnerabilities in all your projects and groups. | -| [Static Application Security Testing (SAST)](sast/index.md) **(ULTIMATE)** | Analyze source code for known vulnerabilities. | +| [Static Application Security Testing (SAST)](sast/index.md) | Analyze source code for known vulnerabilities. | +| [Coverage fuzzing](coverage_fuzzing/index.md) **(ULTIMATE)** | Find unknown bugs and vulnerabilities with coverage-guided fuzzing. | ## Security Scanning with Auto DevOps @@ -118,6 +125,8 @@ information with several options: ### View details of a DAST vulnerability +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/36332) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.1. + Vulnerabilities detected by DAST occur in the live web application. Rectification of these types of vulnerabilities requires specific information. DAST provides the information required to investigate and rectify the underlying cause. @@ -242,6 +251,36 @@ Click this button to create a merge request to apply the solution onto the sourc  +### Managing related issues for a vulnerability + +Issues can be linked to a vulnerability using the related issues block on the vulnerability page. +The relationship is uni-directional. The vulnerability page shows related issues, but the issue page +doesn't show the vulnerability it's related to. An issue can only be related to one vulnerability at +a time. Issues can be linked across groups and projects. + +#### Adding a related issue + +You can link an issue by clicking the **{plus}** button in the **Related Issues** block. + + + +A text box appears that lets you type an issue number or paste an issue link. You can enter multiple +issues at once. Pressing the space bar after each issue number or link converts them to tags that +you can remove by clicking the **{close}** icon to the tag's right. Typing `#` followed by a number +shows an autocomplete menu. Click an issue in the menu to add it as a tag. When you're finished +entering issues, click the **Add** button to link the issues to the vulnerability. Alternatively, +click **Cancel** to exit without linking any issues. + + + +### Removing a related issue + +Click the **{close}** icon to right of an issue to remove it as a related issue. Note that this only +removes it as a related issue of the vulnerability; it doesn't modify or remove the issue itself. +You can link it to the vulnerability again if desired. + + + ## Security approvals in merge requests > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9928) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.2. @@ -272,7 +311,7 @@ To enable Security Approvals, a [project approval rule](../project/merge_request must be created with the case-sensitive name `Vulnerability-Check`. This approval group must be set with the number of approvals required greater than zero. You must have Maintainer or Owner [permissions](../permissions.md#project-members-permissions) to manage approval rules. -1. Navigate to your project's **{settings}** **Settings > General** and expand **Merge request approvals**. +1. Navigate to your project's **Settings > General** and expand **Merge request approvals**. 1. Click **Add approval rule**, or **Edit**. - Add or change the **Rule name** to `Vulnerability-Check` (case sensitive). @@ -282,14 +321,15 @@ Once this group is added to your project, the approval rule is enabled for all m Any code changes cause the approvals required to reset. -An approval is required when a security report: +An approval is required when the latest security report in a merge request: -- Contains a new vulnerability of `high`, `critical`, or `unknown` severity, regardless of dismissal. +- Contains a vulnerability of `high`, `critical`, or `unknown` severity that is not present in the + target branch. Note that approval is still required for dismissed vulnerabilities. - Is not generated during pipeline execution. -An approval is optional when a security report: +An approval is optional when the security report: -- Contains no new vulnerabilities. +- Contains no new vulnerabilities when compared to the target branch. - Contains only new vulnerabilities of `low` or `medium` severity. ## Enabling License Approvals within a project diff --git a/doc/user/application_security/offline_deployments/index.md b/doc/user/application_security/offline_deployments/index.md index 9a2f8768fc0..a5cf93f9448 100644 --- a/doc/user/application_security/offline_deployments/index.md +++ b/doc/user/application_security/offline_deployments/index.md @@ -91,3 +91,126 @@ above. You can find more information at each of the pages below: - [DAST offline directions](../dast/index.md#running-dast-in-an-offline-environment) - [License Compliance offline directions](../../compliance/license_compliance/index.md#running-license-compliance-in-an-offline-environment) - [Dependency Scanning offline directions](../dependency_scanning/index.md#running-dependency-scanning-in-an-offline-environment) + +## Loading Docker images onto your offline host + +To use many GitLab features, including +[security scans](../index.md#working-in-an-offline-environment) +and [Auto DevOps](../../../topics/autodevops/index.md), the GitLab Runner must be able to fetch the +relevant Docker images. + +The process for making these images available without direct access to the public internet +involves downloading the images then packaging and transferring them to the offline host. Here's an +example of such a transfer: + +1. Download Docker images from public internet. +1. Package Docker images as tar archives. +1. Transfer images to offline environment. +1. Load transferred images into offline Docker registry. + +### Using the official GitLab template + +GitLab provides a [vendored template](../../../ci/yaml/README.md#includetemplate) +to ease this process. + +This template should be used in a new, empty project, with a `gitlab-ci.yml` file containing: + +```yaml +include: + - template: Secure-Binaries.gitlab-ci.yml +``` + +The pipeline downloads the Docker images needed for the Security Scanners and saves them as +[job artifacts](../../../ci/pipelines/job_artifacts.md) or pushes them to the [Container Registry](../../packages/container_registry/index.md) +of the project where the pipeline is executed. These archives can be transferred to another location +and [loaded](https://docs.docker.com/engine/reference/commandline/load/) in a Docker daemon. +This method requires a GitLab Runner with access to both `gitlab.com` (including +`registry.gitlab.com`) and the local offline instance. This runner must run in +[privileged mode](https://docs.gitlab.com/runner/executors/docker.html#use-docker-in-docker-with-privileged-mode) +to be able to use the `docker` command inside the jobs. This runner can be installed in a DMZ or on +a bastion, and used only for this specific project. + +#### Scheduling the updates + +By default, this project's pipeline will run only once, when the `.gitlab-ci.yml` is added to the +repo. To update the GitLab security scanners and signatures, it's necessary to run this pipeline +regularly. GitLab provides a way to [schedule pipelines](../../../ci/pipelines/schedules.md). For +example, you can set this up to download and store the Docker images every week. + +Some images can be updated more frequently than others. For example, the [vulnerability database](https://hub.docker.com/r/arminc/clair-db/tags) +for Container Scanning is updated daily. To update this single image, create a new Scheduled +Pipeline that runs daily and set `SECURE_BINARIES_ANALYZERS` to `clair-vulnerabilities-db`. Only +this job will be triggered, and the image will be updated daily and made available in the project +registry. + +#### Using the secure bundle created + +The project using the `Secure-Binaries.gitlab-ci.yml` template should now host all the required +images and resources needed to run GitLab Security features. + +Next, you must tell the offline instance to use these resources instead of the default ones on +GitLab.com. To do so, set the environment variable `SECURE_ANALYZERS_PREFIX` with the URL of the +project [container registry](../../packages/container_registry/index.md). + +You can set this variable in the projects' `.gitlab-ci.yml`, or +in the GitLab UI at the project or group level. See the [GitLab CI/CD environment variables page](../../../ci/variables/README.md#custom-environment-variables) +for more information. + +#### Variables + +The following table shows which variables you can use with the `Secure-Binaries.gitlab-ci.yml` +template: + +| VARIABLE | Description | Default value | +|-------------------------------------------|-----------------------------------------------|-----------------------------------| +| `SECURE_BINARIES_ANALYZERS` | Comma-separated list of analyzers to download | `"bandit, brakeman, gosec, and so on..."` | +| `SECURE_BINARIES_DOWNLOAD_IMAGES` | Used to disable jobs | `"true"` | +| `SECURE_BINARIES_PUSH_IMAGES` | Push files to the project registry | `"true"` | +| `SECURE_BINARIES_SAVE_ARTIFACTS` | Also save image archives as artifacts | `"false"` | +| `SECURE_BINARIES_ANALYZER_VERSION` | Default analyzer version (Docker tag) | `"2"` | + +### Alternate way without the official template + +If it's not possible to follow the above method, the images can be transferred manually instead: + +#### Example image packager script + +```shell +#!/bin/bash +set -ux + +# Specify needed analyzer images +analyzers=${SAST_ANALYZERS:-"bandit eslint gosec"} +gitlab=registry.gitlab.com/gitlab-org/security-products/analyzers/ + +for i in "${analyzers[@]}" +do + tarname="${i}_2.tar" + docker pull $gitlab$i:2 + docker save $gitlab$i:2 -o ./analyzers/${tarname} + chmod +r ./analyzers/${tarname} +done +``` + +#### Example image loader script + +This example loads the images from a bastion host to an offline host. In certain configurations, +physical media may be needed for such a transfer: + +```shell +#!/bin/bash +set -ux + +# Specify needed analyzer images +analyzers=${SAST_ANALYZERS:-"bandit eslint gosec"} +registry=$GITLAB_HOST:4567 + +for i in "${analyzers[@]}" +do + tarname="${i}_2.tar" + scp ./analyzers/${tarname} ${GITLAB_HOST}:~/${tarname} + ssh $GITLAB_HOST "sudo docker load -i ${tarname}" + ssh $GITLAB_HOST "sudo docker tag $(sudo docker images | grep $i | awk '{print $3}') ${registry}/analyzers/${i}:2" + ssh $GITLAB_HOST "sudo docker push ${registry}/analyzers/${i}:2" +done +``` diff --git a/doc/user/application_security/sast/index.md b/doc/user/application_security/sast/index.md index 70d4b513cf9..fd331020719 100644 --- a/doc/user/application_security/sast/index.md +++ b/doc/user/application_security/sast/index.md @@ -5,7 +5,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w type: reference, howto --- -# Static Application Security Testing (SAST) **(ULTIMATE)** +# Static Application Security Testing (SAST) > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/3775) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 10.3. @@ -14,19 +14,11 @@ The whitepaper ["A Seismic Shift in Application Security"](https://about.gitlab. explains how **4 of the top 6 attacks were application based**. Download it to learn how to protect your organization. -## Overview - If you're using [GitLab CI/CD](../../../ci/README.md), you can analyze your source code for known -vulnerabilities using Static Application Security Testing (SAST). - -You can take advantage of SAST by doing one of the following: - -- [Including the SAST template](#configuration) in your existing `.gitlab-ci.yml` file. -- Implicitly using [Auto SAST](../../../topics/autodevops/stages.md#auto-sast-ultimate) provided by - [Auto DevOps](../../../topics/autodevops/index.md). +vulnerabilities using Static Application Security Testing (SAST). GitLab checks the SAST report and +compares the found vulnerabilities between the source and target branches. -GitLab checks the SAST report, compares the found vulnerabilities between the -source and target branches, and shows the information right on the merge request. +Details of the vulnerabilities found are included in the merge request. **(ULTIMATE)**  @@ -40,7 +32,7 @@ The results are sorted by the priority of the vulnerability: 1. Everything else NOTE: **Note:** -A pipeline consists of multiple jobs, including SAST and DAST scanning. If any job fails to finish for any reason, the security dashboard won't show SAST scanner output. For example, if the SAST job finishes but the DAST job fails, the security dashboard won't show SAST results. The analyzer will output an [exit code](../../../development/integrations/secure.md#exit-code) on failure. +A pipeline consists of multiple jobs, including SAST and DAST scanning. If any job fails to finish for any reason, the security dashboard doesn't show SAST scanner output. For example, if the SAST job finishes but the DAST job fails, the security dashboard doesn't show SAST results. The analyzer outputs an [exit code](../../../development/integrations/secure.md#exit-code) on failure. ## Use cases @@ -51,15 +43,15 @@ A pipeline consists of multiple jobs, including SAST and DAST scanning. If any j ## Requirements -To run SAST jobs, by default, you need GitLab Runner with the +To run SAST jobs, by default, you need a GitLab Runner with the [`docker`](https://docs.gitlab.com/runner/executors/docker.html) or [`kubernetes`](https://docs.gitlab.com/runner/install/kubernetes.html) executor. If you're using the shared Runners on GitLab.com, this is enabled by default. -Beginning with GitLab 13.0, Docker privileged mode is necessary only if you've [enabled Docker-in-Docker for SAST](#enabling-docker-in-docker). +Beginning with GitLab 13.0, Docker privileged mode is necessary only if you've [enabled Docker-in-Docker for SAST](#enabling-docker-in-docker-ultimate). CAUTION: **Caution:** -Our SAST jobs currently expect a Linux container type. Windows containers are not yet supported. +Our SAST jobs require a Linux container type. Windows containers are not yet supported. CAUTION: **Caution:** If you use your own Runners, make sure the Docker version installed @@ -69,27 +61,27 @@ is **not** `19.03.0`. See [troubleshooting information](#error-response-from-dae The following table shows which languages, package managers and frameworks are supported and which tools are used. -| Language (package managers) / framework | Scan tool | Introduced in GitLab Version | -|-----------------------------------------------------------------------------|----------------------------------------------------------------------------------------|------------------------------| -| .NET Core | [Security Code Scan](https://security-code-scan.github.io) | 11.0 | -| .NET Framework | [Security Code Scan](https://security-code-scan.github.io) | 13.0 | -| Any | [Gitleaks](https://github.com/zricethezav/gitleaks) and [TruffleHog](https://github.com/dxa4481/truffleHog) | 11.9 | -| Apex (Salesforce) | [PMD](https://pmd.github.io/pmd/index.html) | 12.1 | -| C/C++ | [Flawfinder](https://github.com/david-a-wheeler/flawfinder) | 10.7 | -| Elixir (Phoenix) | [Sobelow](https://github.com/nccgroup/sobelow) | 11.10 | -| Go | [Gosec](https://github.com/securego/gosec) | 10.7 | -| Groovy ([Ant](https://ant.apache.org/), [Gradle](https://gradle.org/), [Maven](https://maven.apache.org/) and [SBT](https://www.scala-sbt.org/)) | [SpotBugs](https://spotbugs.github.io/) with the [find-sec-bugs](https://find-sec-bugs.github.io/) plugin | 11.3 (Gradle) & 11.9 (Ant, Maven, SBT) | -| Helm Charts | [Kubesec](https://github.com/controlplaneio/kubesec) | 13.1 | -| Java ([Ant](https://ant.apache.org/), [Gradle](https://gradle.org/), [Maven](https://maven.apache.org/) and [SBT](https://www.scala-sbt.org/)) | [SpotBugs](https://spotbugs.github.io/) with the [find-sec-bugs](https://find-sec-bugs.github.io/) plugin | 10.6 (Maven), 10.8 (Gradle) & 11.9 (Ant, SBT) | -| JavaScript | [ESLint security plugin](https://github.com/nodesecurity/eslint-plugin-security) | 11.8, moved to [GitLab Core](https://about.gitlab.com/pricing/) in 13.2 | -| Kubernetes manifests | [Kubesec](https://github.com/controlplaneio/kubesec) | 12.6 | -| Node.js | [NodeJsScan](https://github.com/ajinabraham/NodeJsScan) | 11.1 | -| PHP | [phpcs-security-audit](https://github.com/FloeDesignTechnologies/phpcs-security-audit) | 10.8 | -| Python ([pip](https://pip.pypa.io/en/stable/)) | [bandit](https://github.com/PyCQA/bandit) | 10.3 | -| React | [ESLint react plugin](https://github.com/yannickcr/eslint-plugin-react) | 12.5 | -| Ruby on Rails | [brakeman](https://brakemanscanner.org) | 10.3, moved to [GitLab Core](https://about.gitlab.com/pricing/) in 13.1 | -| Scala ([Ant](https://ant.apache.org/), [Gradle](https://gradle.org/), [Maven](https://maven.apache.org/) and [SBT](https://www.scala-sbt.org/)) | [SpotBugs](https://spotbugs.github.io/) with the [find-sec-bugs](https://find-sec-bugs.github.io/) plugin | 11.0 (SBT) & 11.9 (Ant, Gradle, Maven) | -| TypeScript | [ESLint security plugin](https://github.com/nodesecurity/eslint-plugin-security) | 11.9, merged with ESLint in 13.2 | +| Language (package managers) / framework | Scan tool | Introduced in GitLab Version | +|--------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| .NET Core | [Security Code Scan](https://security-code-scan.github.io) | 11.0, [moved](https://gitlab.com/groups/gitlab-org/-/epics/2098) to [GitLab Core](https://about.gitlab.com/pricing/) in 13.3 | +| .NET Framework | [Security Code Scan](https://security-code-scan.github.io) | 13.0, [moved](https://gitlab.com/groups/gitlab-org/-/epics/2098) to [GitLab Core](https://about.gitlab.com/pricing/) in 13.3 | +| Any | [Gitleaks](https://github.com/zricethezav/gitleaks) and [TruffleHog](https://github.com/dxa4481/truffleHog) | 11., [moved](https://gitlab.com/groups/gitlab-org/-/epics/2098) to [GitLab Core](https://about.gitlab.com/pricing/) in 13.3 | +| Apex (Salesforce) | [PMD](https://pmd.github.io/pmd/index.html) | 12.1, [moved](https://gitlab.com/groups/gitlab-org/-/epics/2098) to [GitLab Core](https://about.gitlab.com/pricing/) in 13.3 | +| C/C++ | [Flawfinder](https://github.com/david-a-wheeler/flawfinder) | 10.7, [moved](https://gitlab.com/groups/gitlab-org/-/epics/2098) to [GitLab Core](https://about.gitlab.com/pricing/) in 13.3 | +| Elixir (Phoenix) | [Sobelow](https://github.com/nccgroup/sobelow) | 11.10, [moved](https://gitlab.com/groups/gitlab-org/-/epics/2098) to [GitLab Core](https://about.gitlab.com/pricing/) in 13.3 | +| Go | [Gosec](https://github.com/securego/gosec) | 10.7, [moved](https://gitlab.com/groups/gitlab-org/-/epics/2098) to [GitLab Core](https://about.gitlab.com/pricing/) in 13.3 | +| Groovy ([Ant](https://ant.apache.org/), [Gradle](https://gradle.org/), [Maven](https://maven.apache.org/) and [SBT](https://www.scala-sbt.org/)) | [SpotBugs](https://spotbugs.github.io/) with the [find-sec-bugs](https://find-sec-bugs.github.io/) plugin | 11.3 (Gradle) & 11.9 (Ant, Maven, SBT), [moved](https://gitlab.com/groups/gitlab-org/-/epics/2098) to [GitLab Core](https://about.gitlab.com/pricing/) in 13.3 | +| Helm Charts | [Kubesec](https://github.com/controlplaneio/kubesec) | 13.1, [moved](https://gitlab.com/groups/gitlab-org/-/epics/2098) to [GitLab Core](https://about.gitlab.com/pricing/) in 13.3 | +| Java ([Ant](https://ant.apache.org/), [Gradle](https://gradle.org/), [Maven](https://maven.apache.org/) and [SBT](https://www.scala-sbt.org/)) | [SpotBugs](https://spotbugs.github.io/) with the [find-sec-bugs](https://find-sec-bugs.github.io/) plugin | 10.6 (Maven), 10.8 (Gradle) & 11.9 (Ant, SBT), [moved](https://gitlab.com/groups/gitlab-org/-/epics/2098) to [GitLab Core](https://about.gitlab.com/pricing/) in 13.3 | +| JavaScript | [ESLint security plugin](https://github.com/nodesecurity/eslint-plugin-security) | 11.8, [moved](https://gitlab.com/groups/gitlab-org/-/epics/2098) to [GitLab Core](https://about.gitlab.com/pricing/) in 13.2 | +| Kubernetes manifests | [Kubesec](https://github.com/controlplaneio/kubesec) | 12.6, [moved](https://gitlab.com/groups/gitlab-org/-/epics/2098) to [GitLab Core](https://about.gitlab.com/pricing/) in 13.3 | +| Node.js | [NodeJsScan](https://github.com/ajinabraham/NodeJsScan) | 11.1, [moved](https://gitlab.com/groups/gitlab-org/-/epics/2098) to [GitLab Core](https://about.gitlab.com/pricing/) in 13.3 | +| PHP | [phpcs-security-audit](https://github.com/FloeDesignTechnologies/phpcs-security-audit) | 10.8, [moved](https://gitlab.com/groups/gitlab-org/-/epics/2098) to [GitLab Core](https://about.gitlab.com/pricing/) in 13.3 | +| Python ([pip](https://pip.pypa.io/en/stable/)) | [bandit](https://github.com/PyCQA/bandit) | 10.3, [moved](https://gitlab.com/groups/gitlab-org/-/epics/2098) to [GitLab Core](https://about.gitlab.com/pricing/) in 13.3 | +| React | [ESLint react plugin](https://github.com/yannickcr/eslint-plugin-react) | 12.5, [moved](https://gitlab.com/groups/gitlab-org/-/epics/2098) to [GitLab Core](https://about.gitlab.com/pricing/) in 13.2 | +| Ruby on Rails | [brakeman](https://brakemanscanner.org) | 10.3, [moved](https://gitlab.com/groups/gitlab-org/-/epics/2098) to [GitLab Core](https://about.gitlab.com/pricing/) in 13.1 | +| Scala ([Ant](https://ant.apache.org/), [Gradle](https://gradle.org/), [Maven](https://maven.apache.org/) and [SBT](https://www.scala-sbt.org/)) | [SpotBugs](https://spotbugs.github.io/) with the [find-sec-bugs](https://find-sec-bugs.github.io/) plugin | 11.0 (SBT) & 11.9 (Ant, Gradle, Maven), [moved](https://gitlab.com/groups/gitlab-org/-/epics/2098) to [GitLab Core](https://about.gitlab.com/pricing/) in 13.3 | +| TypeScript | [ESLint security plugin](https://github.com/nodesecurity/eslint-plugin-security) | 11.9, [merged](https://gitlab.com/gitlab-org/gitlab/-/issues/36059) with ESLint in 13.2 | NOTE: **Note:** The Java analyzers can also be used for variants like the @@ -98,11 +90,11 @@ The Java analyzers can also be used for variants like the ### Making SAST analyzers available to all GitLab tiers -All open source (OSS) analyzers are in the process of being reviewed and potentially moved to the GitLab Core tier. Progress can be +All open source (OSS) analyzers have been moved to the GitLab Core tier. Progress can be tracked in the corresponding [epic](https://gitlab.com/groups/gitlab-org/-/epics/2098). -Please note that support for [Docker-in-Docker](#enabling-docker-in-docker) +Please note that support for [Docker-in-Docker](#enabling-docker-in-docker-ultimate) will not be extended to the GitLab Core tier. #### Summary of features per tier @@ -110,14 +102,14 @@ will not be extended to the GitLab Core tier. Different features are available in different [GitLab tiers](https://about.gitlab.com/pricing/), as shown in the following table: -| Capability | In Core | In Ultimate | -|:--------------------------------------------------------------------------|:--------------------|:-------------------| -| [Configure SAST Scanners](#configuration) | **{check-circle}** | **{check-circle}** | -| [Customize SAST Settings](#customizing-the-sast-settings) | **{check-circle}** | **{check-circle}** | -| View [JSON Report](#reports-json-format) | **{check-circle}** | **{check-circle}** | -| [Presentation of JSON Report in Merge Request](#overview) | **{dotted-circle}** | **{check-circle}** | -| [Interaction with Vulnerabilities](#interacting-with-the-vulnerabilities) | **{dotted-circle}** | **{check-circle}** | -| [Access to Security Dashboard](#security-dashboard) | **{dotted-circle}** | **{check-circle}** | +| Capability | In Core | In Ultimate | +|:-----------------------------------------------------------------------------------|:--------------------|:-------------------| +| [Configure SAST Scanners](#configuration) | **{check-circle}** | **{check-circle}** | +| [Customize SAST Settings](#customizing-the-sast-settings) | **{check-circle}** | **{check-circle}** | +| View [JSON Report](#reports-json-format) | **{check-circle}** | **{check-circle}** | +| [Presentation of JSON Report in Merge Request](#overview) | **{dotted-circle}** | **{check-circle}** | +| [Interaction with Vulnerabilities](#interacting-with-the-vulnerabilities-ultimate) | **{dotted-circle}** | **{check-circle}** | +| [Access to Security Dashboard](#security-dashboard-ultimate) | **{dotted-circle}** | **{check-circle}** | ## Contribute your scanner @@ -125,9 +117,14 @@ The [Security Scanner Integration](../../../development/integrations/secure.md) ## Configuration -NOTE: **Note:** -You don't have to configure SAST manually as shown in this section if you're using [Auto SAST](../../../topics/autodevops/stages.md#auto-sast-ultimate) -provided by [Auto DevOps](../../../topics/autodevops/index.md). +To configure SAST for a project you can: + +- Use [Auto SAST](../../../topics/autodevops/stages.md#auto-sast-ultimate) provided by + [Auto DevOps](../../../topics/autodevops/index.md). +- [Configure SAST manually](#configure-sast-manually). +- [Configure SAST using the UI](#configure-sast-in-the-ui) (introduced in GitLab 13.3). + +### Configure SAST manually For GitLab 11.9 and later, to enable SAST you must [include](../../../ci/yaml/README.md#includetemplate) the [`SAST.gitlab-ci.yml` template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml) @@ -141,14 +138,29 @@ include: - template: SAST.gitlab-ci.yml ``` -The included template will create SAST jobs in your CI/CD pipeline and scan +The included template creates SAST jobs in your CI/CD pipeline and scans your project's source code for possible vulnerabilities. -The results will be saved as a +The results are saved as a [SAST report artifact](../../../ci/pipelines/job_artifacts.md#artifactsreportssast-ultimate) that you can later download and analyze. Due to implementation limitations, we always take the latest SAST artifact available. +### Configure SAST in the UI + +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3659) in GitLab Ultimate 13.3. + +For a project that does not have a `.gitlab-ci.yml` file, you can enable SAST with a basic +configuration using the **SAST Configuration** page: + +1. From the project's home page, go to **Security & Configuration** > **Configuration** in the + left sidebar. +1. Click **Enable via Merge Request** on the Static Application Security Testing (SAST) row. +1. Enter the appropriate SAST details into the fields on the page. See [Available variables](#available-variables) + for a description of these variables. +1. Click **Create Merge Request**. +1. Review and merge the merge request. + ### Customizing the SAST settings The SAST settings can be changed through [environment variables](#available-variables) @@ -203,12 +215,12 @@ you can use the `MAVEN_CLI_OPTS` environment variable. Read more on [how to use private Maven repositories](../index.md#using-private-maven-repos). -### Enabling Docker-in-Docker +### Enabling Docker-in-Docker **(ULTIMATE)** If needed, you can enable Docker-in-Docker to restore the SAST behavior that existed prior to GitLab 13.0. Follow these steps to do so: -1. Configure GitLab Runner with Docker-inDocker in [privileged mode](https://docs.gitlab.com/runner/executors/docker.html#use-docker-in-docker-with-privileged-mode). +1. Configure a GitLab Runner with Docker-in-Docker in [privileged mode](https://docs.gitlab.com/runner/executors/docker.html#use-docker-in-docker-with-privileged-mode). 1. Set the variable `SAST_DISABLE_DIND` set to `false`: ```yaml @@ -289,14 +301,16 @@ See [Analyzer settings](#analyzer-settings) for the complete list of available o SAST can be [configured](#customizing-the-sast-settings) using environment variables. -#### Logging Level +#### Logging level -You can control the verbosity of logs by setting the `SECURE_LOG_LEVEL` env var. The default is set to `info`, you can set it to any of the following levels: +To control the verbosity of logs set the `SECURE_LOG_LEVEL` environment variable. Messages of this logging level or higher are output. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/10880) in GitLab 13.1. + +From highest to lowest severity, the logging levels are: - `fatal` - `error` - `warn` -- `info` +- `info` (default) - `debug` #### Custom Certificate Authority @@ -308,12 +322,12 @@ of CA certs that you want to trust within the SAST environment. The following are Docker image-related variables. -| Environment variable | Description | -|------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `SECURE_ANALYZERS_PREFIX` | Override the name of the Docker registry providing the default images (proxy). Read more about [customizing analyzers](analyzers.md). | -| `SAST_ANALYZER_IMAGE_TAG` | **DEPRECATED:** Override the Docker tag of the default images. Read more about [customizing analyzers](analyzers.md). | -| `SAST_DEFAULT_ANALYZERS` | Override the names of default images. Read more about [customizing analyzers](analyzers.md). | -| `SAST_DISABLE_DIND` | Disable Docker-in-Docker and run analyzers [individually](#enabling-docker-in-docker). This variable is `true` by default. | +| Environment variable | Description | +|---------------------------|---------------------------------------------------------------------------------------------------------------------------------------| +| `SECURE_ANALYZERS_PREFIX` | Override the name of the Docker registry providing the default images (proxy). Read more about [customizing analyzers](analyzers.md). | +| `SAST_ANALYZER_IMAGE_TAG` | **DEPRECATED:** Override the Docker tag of the default images. Read more about [customizing analyzers](analyzers.md). | +| `SAST_DEFAULT_ANALYZERS` | Override the names of default images. Read more about [customizing analyzers](analyzers.md). | +| `SAST_DISABLE_DIND` | Disable Docker-in-Docker and run analyzers [individually](#enabling-docker-in-docker-ultimate). This variable is `true` by default. | #### Vulnerability filters @@ -322,9 +336,9 @@ Some analyzers make it possible to filter out vulnerabilities under a given thre | Environment variable | Default value | Description | |-------------------------------|--------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `SAST_EXCLUDED_PATHS` | `spec, test, tests, tmp` | Exclude vulnerabilities from output based on the paths. This is a comma-separated list of patterns. Patterns can be globs, or file or folder paths (for example, `doc,spec` ). Parent directories will also match patterns. | -| `SAST_BANDIT_EXCLUDED_PATHS` | | Comma-separated list of paths to exclude from scan. Uses Python's [`fnmatch` syntax](https://docs.python.org/2/library/fnmatch.html); For example: `'*/tests/*, */venv/*'` | +| `SEARCH_MAX_DEPTH` | 4 | Maximum number of directories traversed when searching for source code files. | +| `SAST_BANDIT_EXCLUDED_PATHS` | | Comma-separated list of paths to exclude from scan. Uses Python's [`fnmatch` syntax](https://docs.python.org/2/library/fnmatch.html); For example: `'*/tests/*, */venv/*'` | | `SAST_BRAKEMAN_LEVEL` | 1 | Ignore Brakeman vulnerabilities under given confidence level. Integer, 1=Low 3=High. | -| `SAST_DISABLE_BABEL` | `false` | Disable Babel processing for the NodeJsScan scanner. Set to `true` to disable Babel processing. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/33065) in GitLab 13.2. | | `SAST_FLAWFINDER_LEVEL` | 1 | Ignore Flawfinder vulnerabilities under given risk level. Integer, 0=No risk, 5=High risk. | | `SAST_GITLEAKS_ENTROPY_LEVEL` | 8.0 | Minimum entropy for secret detection. Float, 0.0 = low, 8.0 = high. | | `SAST_GOSEC_LEVEL` | 0 | Ignore Gosec vulnerabilities under given confidence level. Integer, 0=Undefined, 1=Low, 2=Medium, 3=High. | @@ -334,40 +348,40 @@ Some analyzers make it possible to filter out vulnerabilities under a given thre #### Docker-in-Docker orchestrator -The following variables configure the Docker-in-Docker orchestrator, and therefore are only used when the Docker-in-Docker mode is [enabled](#enabling-docker-in-docker). +The following variables configure the Docker-in-Docker orchestrator, and therefore are only used when the Docker-in-Docker mode is [enabled](#enabling-docker-in-docker-ultimate). -| Environment variable | Default value | Description | -|------------------------------------------|---------------|-------------| -| `SAST_ANALYZER_IMAGES` | | Comma-separated list of custom images. Default images are still enabled. Read more about [customizing analyzers](analyzers.md). | -| `SAST_PULL_ANALYZER_IMAGES` | 1 | Pull the images from the Docker registry (set to 0 to disable). Read more about [customizing analyzers](analyzers.md). | -| `SAST_DOCKER_CLIENT_NEGOTIATION_TIMEOUT` | 2m | Time limit for Docker client negotiation. Timeouts are parsed using Go's [`ParseDuration`](https://golang.org/pkg/time/#ParseDuration). Valid time units are `ns`, `us` (or `µs`), `ms`, `s`, `m`, `h`. For example, `300ms`, `1.5h` or `2h45m`. | -| `SAST_PULL_ANALYZER_IMAGE_TIMEOUT` | 5m | Time limit when pulling the image of an analyzer. Timeouts are parsed using Go's [`ParseDuration`](https://golang.org/pkg/time/#ParseDuration). Valid time units are `ns`, `us` (or `µs`), `ms`, `s`, `m`, `h`. For example, `300ms`, `1.5h` or `2h45m`. | -| `SAST_RUN_ANALYZER_TIMEOUT` | 20m | Time limit when running an analyzer. Timeouts are parsed using Go's [`ParseDuration`](https://golang.org/pkg/time/#ParseDuration). Valid time units are `ns`, `us` (or `µs`), `ms`, `s`, `m`, `h`. For example, `300ms`, `1.5h` or `2h45m`.| +| Environment variable | Default value | Description | +|------------------------------------------|---------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `SAST_ANALYZER_IMAGES` | | Comma-separated list of custom images. Default images are still enabled. Read more about [customizing analyzers](analyzers.md). | +| `SAST_PULL_ANALYZER_IMAGES` | 1 | Pull the images from the Docker registry (set to 0 to disable). Read more about [customizing analyzers](analyzers.md). | +| `SAST_DOCKER_CLIENT_NEGOTIATION_TIMEOUT` | 2m | Time limit for Docker client negotiation. Timeouts are parsed using Go's [`ParseDuration`](https://golang.org/pkg/time/#ParseDuration). Valid time units are `ns`, `us` (or `µs`), `ms`, `s`, `m`, `h`. For example, `300ms`, `1.5h` or `2h45m`. | +| `SAST_PULL_ANALYZER_IMAGE_TIMEOUT` | 5m | Time limit when pulling the image of an analyzer. Timeouts are parsed using Go's [`ParseDuration`](https://golang.org/pkg/time/#ParseDuration). Valid time units are `ns`, `us` (or `µs`), `ms`, `s`, `m`, `h`. For example, `300ms`, `1.5h` or `2h45m`. | +| `SAST_RUN_ANALYZER_TIMEOUT` | 20m | Time limit when running an analyzer. Timeouts are parsed using Go's [`ParseDuration`](https://golang.org/pkg/time/#ParseDuration). Valid time units are `ns`, `us` (or `µs`), `ms`, `s`, `m`, `h`. For example, `300ms`, `1.5h` or `2h45m`. | #### Analyzer settings Some analyzers can be customized with environment variables. -| Environment variable | Analyzer | Description | -|---------------------------------------|----------------------|-------------| -| `SCAN_KUBERNETES_MANIFESTS` | Kubesec | Set to `"true"` to scan Kubernetes manifests. | -| `KUBESEC_HELM_CHARTS_PATH` | Kubesec | Optional path to Helm charts that `helm` will use to generate a Kubernetes manifest that `kubesec` will scan. If dependencies are defined, `helm dependency build` should be ran in a `before_script` to fetch the necessary dependencies. | -| `KUBESEC_HELM_OPTIONS` | Kubesec | Additional arguments for the `helm` executable. | -| `COMPILE` | SpotBugs | Set to `false` to disable project compilation and dependency fetching. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/195252) in GitLab 13.1. | -| `ANT_HOME` | SpotBugs | The `ANT_HOME` environment variable. | -| `ANT_PATH` | SpotBugs | Path to the `ant` executable. | -| `GRADLE_PATH` | SpotBugs | Path to the `gradle` executable. | -| `JAVA_OPTS` | SpotBugs | Additional arguments for the `java` executable. | -| `JAVA_PATH` | SpotBugs | Path to the `java` executable. | -| `SAST_JAVA_VERSION` | SpotBugs | Which Java version to use. Supported versions are `8` and `11`. Defaults to `8`. | -| `MAVEN_CLI_OPTS` | SpotBugs | Additional arguments for the `mvn` or `mvnw` executable. | -| `MAVEN_PATH` | SpotBugs | Path to the `mvn` executable. | -| `MAVEN_REPO_PATH` | SpotBugs | Path to the Maven local repository (shortcut for the `maven.repo.local` property). | -| `SBT_PATH` | SpotBugs | Path to the `sbt` executable. | -| `FAIL_NEVER` | SpotBugs | Set to `1` to ignore compilation failure. | -| `SAST_GOSEC_CONFIG` | Gosec | Path to configuration for Gosec (optional). | -| `PHPCS_SECURITY_AUDIT_PHP_EXTENSIONS` | phpcs-security-audit | Comma separated list of additional PHP Extensions. | -| `SEARCH_MAX_DEPTH` | any | Maximum number of directories traversed when searching for source code files. Default: `4`. | +| Environment variable | Analyzer | Description | +|---------------------------------------|----------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `SCAN_KUBERNETES_MANIFESTS` | Kubesec | Set to `"true"` to scan Kubernetes manifests. | +| `KUBESEC_HELM_CHARTS_PATH` | Kubesec | Optional path to Helm charts that `helm` will use to generate a Kubernetes manifest that `kubesec` will scan. If dependencies are defined, `helm dependency build` should be ran in a `before_script` to fetch the necessary dependencies. | +| `KUBESEC_HELM_OPTIONS` | Kubesec | Additional arguments for the `helm` executable. | +| `COMPILE` | SpotBugs | Set to `false` to disable project compilation and dependency fetching. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/195252) in GitLab 13.1. | +| `ANT_HOME` | SpotBugs | The `ANT_HOME` environment variable. | +| `ANT_PATH` | SpotBugs | Path to the `ant` executable. | +| `GRADLE_PATH` | SpotBugs | Path to the `gradle` executable. | +| `JAVA_OPTS` | SpotBugs | Additional arguments for the `java` executable. | +| `JAVA_PATH` | SpotBugs | Path to the `java` executable. | +| `SAST_JAVA_VERSION` | SpotBugs | Which Java version to use. Supported versions are `8` and `11`. Defaults to `8`. | +| `MAVEN_CLI_OPTS` | SpotBugs | Additional arguments for the `mvn` or `mvnw` executable. | +| `MAVEN_PATH` | SpotBugs | Path to the `mvn` executable. | +| `MAVEN_REPO_PATH` | SpotBugs | Path to the Maven local repository (shortcut for the `maven.repo.local` property). | +| `SBT_PATH` | SpotBugs | Path to the `sbt` executable. | +| `FAIL_NEVER` | SpotBugs | Set to `1` to ignore compilation failure. | +| `SAST_GOSEC_CONFIG` | Gosec | Path to configuration for Gosec (optional). | +| `PHPCS_SECURITY_AUDIT_PHP_EXTENSIONS` | phpcs-security-audit | Comma separated list of additional PHP Extensions. | +| `SAST_DISABLE_BABEL` | NodeJsScan | Disable Babel processing for the NodeJsScan scanner. Set to `true` to disable Babel processing. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/33065) in GitLab 13.2. | #### Custom environment variables @@ -387,6 +401,9 @@ analyzer containers: `DOCKER_`, `CI`, `GITLAB_`, `FF_`, `HOME`, `PWD`, `OLDPWD`, The SAST tool emits a JSON report file. For more information, see the [schema for this report](https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/blob/master/dist/sast-report-format.json). +The JSON report file can be downloaded from the CI pipelines page, for more +information see [Downloading artifacts](../../../ci/pipelines/job_artifacts.md). + Here's an example SAST report: ```json-doc @@ -466,13 +483,13 @@ Here's an example SAST report: Learn more about [Secret Detection](../secret_detection). -## Security Dashboard +## Security Dashboard **(ULTIMATE)** The Security Dashboard is a good place to get an overview of all the security vulnerabilities in your groups, projects and pipelines. Read more about the [Security Dashboard](../security_dashboard/index.md). -## Interacting with the vulnerabilities +## Interacting with the vulnerabilities **(ULTIMATE)** Once a vulnerability is found, you can interact with it. Read more on how to [interact with the vulnerabilities](../index.md#interacting-with-the-vulnerabilities). @@ -498,8 +515,9 @@ run successfully. For more information, see [Offline environments](../offline_de To use SAST in an offline environment, you need: - To keep Docker-In-Docker disabled (default). -- GitLab Runner with the [`docker` or `kubernetes` executor](#requirements). -- Docker Container Registry with locally available copies of SAST [analyzer](https://gitlab.com/gitlab-org/security-products/analyzers) images. +- A GitLab Runner with the [`docker` or `kubernetes` executor](#requirements). +- A Docker Container Registry with locally available copies of SAST [analyzer](https://gitlab.com/gitlab-org/security-products/analyzers) images. +- Configure certificate checking of packages (optional). NOTE: **Note:** GitLab Runner has a [default `pull policy` of `always`](https://docs.gitlab.com/runner/executors/docker.html#using-the-always-pull-policy), @@ -556,6 +574,13 @@ variables: The SAST job should now use local copies of the SAST analyzers to scan your code and generate security reports without requiring internet access. +### Configure certificate checking of packages + +If a SAST job invokes a package manager, you must configure its certificate verification. In an +offline environment, certificate verification with an external source isn't possible. Either use a +self-signed certificate or disable certificate verification. Refer to the package manager's +documentation for instructions. + ## Troubleshooting ### `Error response from daemon: error processing tar file: docker-tar: relocation error` diff --git a/doc/user/application_security/secret_detection/index.md b/doc/user/application_security/secret_detection/index.md index ea635212c5d..7daf2f3308b 100644 --- a/doc/user/application_security/secret_detection/index.md +++ b/doc/user/application_security/secret_detection/index.md @@ -19,7 +19,7 @@ malicious users to gain access to resources like deployment environments. GitLab 11.9 includes a new check called Secret Detection. It scans the content of the repository to find API keys and other information that should not be there. -GitLab displays identified secrets as part of the SAST reports visibly in a few places: +GitLab displays identified secrets visibly in a few places: - [Security Dashboard](../security_dashboard/) - Pipelines' **Security** tab @@ -46,6 +46,25 @@ CAUTION: **Caution:** If you use your own Runners, make sure the Docker version installed is **not** `19.03.0`. See [troubleshooting information](../sast#error-response-from-daemon-error-processing-tar-file-docker-tar-relocation-error) for details. +### Making Secret Detection available to all GitLab tiers + +To make Secret Detection available to as many customers as possible, we have enabled it for all GitLab tiers. +However not all features are available on every tier. See the breakdown below for more details. + +#### Summary of features per tier + +Different features are available in different [GitLab tiers](https://about.gitlab.com/pricing/), +as shown in the following table: + +| Capability | In Core | In Ultimate | +|:--------------------------------------------------------------------------|:--------------------|:-------------------| +| [Configure Secret Detection Scanners](#configuration) | **{check-circle}** | **{check-circle}** | +| [Customize Secret Detection Settings](#customizing-settings) | **{check-circle}** | **{check-circle}** | +| View [JSON Report](../sast/index.md#reports-json-format) | **{check-circle}** | **{check-circle}** | +| [Presentation of JSON Report in Merge Request](#overview) | **{dotted-circle}** | **{check-circle}** | +| [Interaction with Vulnerabilities](../vulnerabilities/index.md) | **{dotted-circle}** | **{check-circle}** | +| [Access to Security Dashboard](../security_dashboard/index.md) | **{dotted-circle}** | **{check-circle}** | + ## Configuration NOTE: **Note:** @@ -145,16 +164,19 @@ Secret Detection can be customized by defining available variables: |-------------------------|---------------|-------------| | `SECRET_DETECTION_COMMIT_FROM` | - | The commit a Gitleaks scan starts at. | | `SECRET_DETECTION_COMMIT_TO` | - | The commit a Gitleaks scan ends at. | +| `SECRET_DETECTION_EXCLUDED_PATHS` | "" | Exclude vulnerabilities from output based on the paths. This is a comma-separated list of patterns. Patterns can be globs, or file or folder paths (for example, `doc,spec` ). Parent directories will also match patterns. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/225273) in GitLab 13.3. | | `SECRET_DETECTION_HISTORIC_SCAN` | false | Flag to enable a historic Gitleaks scan. | -### Logging Level +### Logging level + +To control the verbosity of logs set the `SECURE_LOG_LEVEL` environment variable. Messages of this logging level or higher are output. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/10880) in GitLab 13.1. -You can control the verbosity of logs by setting the `SECURE_LOG_LEVEL` env var. The default is set to `info`, you can set it to any of the following levels: +From highest to lowest severity, the logging levels are: - `fatal` - `error` - `warn` -- `info` +- `info` (default) - `debug` ## Full History Secret Scan diff --git a/doc/user/application_security/security_dashboard/img/group_security_dashboard_export_csv_v13_1.png b/doc/user/application_security/security_dashboard/img/group_security_dashboard_export_csv_v13_1.png Binary files differindex d98fb71ae37..8fab4e39175 100644 --- a/doc/user/application_security/security_dashboard/img/group_security_dashboard_export_csv_v13_1.png +++ b/doc/user/application_security/security_dashboard/img/group_security_dashboard_export_csv_v13_1.png diff --git a/doc/user/application_security/security_dashboard/img/group_security_dashboard_v13_2_noNav.png b/doc/user/application_security/security_dashboard/img/group_security_dashboard_v13_2_noNav.png Binary files differdeleted file mode 100644 index d6cfc2de980..00000000000 --- a/doc/user/application_security/security_dashboard/img/group_security_dashboard_v13_2_noNav.png +++ /dev/null diff --git a/doc/user/application_security/security_dashboard/img/group_security_dashboard_v13_3.png b/doc/user/application_security/security_dashboard/img/group_security_dashboard_v13_3.png Binary files differnew file mode 100644 index 00000000000..4d51f57a98d --- /dev/null +++ b/doc/user/application_security/security_dashboard/img/group_security_dashboard_v13_3.png diff --git a/doc/user/application_security/security_dashboard/img/group_vulnerability_report_v13_3.png b/doc/user/application_security/security_dashboard/img/group_vulnerability_report_v13_3.png Binary files differnew file mode 100644 index 00000000000..7b9a48b8738 --- /dev/null +++ b/doc/user/application_security/security_dashboard/img/group_vulnerability_report_v13_3.png diff --git a/doc/user/application_security/security_dashboard/img/pipeline_security_v13_3.gif b/doc/user/application_security/security_dashboard/img/pipeline_security_v13_3.gif Binary files differnew file mode 100644 index 00000000000..29e7168b6ea --- /dev/null +++ b/doc/user/application_security/security_dashboard/img/pipeline_security_v13_3.gif diff --git a/doc/user/application_security/security_dashboard/img/standalone_vulnerability_page_v13_1.png b/doc/user/application_security/security_dashboard/img/vulnerability_page_v13_1.png Binary files differindex 9cf95b197fe..9cf95b197fe 100644 --- a/doc/user/application_security/security_dashboard/img/standalone_vulnerability_page_v13_1.png +++ b/doc/user/application_security/security_dashboard/img/vulnerability_page_v13_1.png diff --git a/doc/user/application_security/security_dashboard/index.md b/doc/user/application_security/security_dashboard/index.md index 9a13d143d1f..b8fcc513cb1 100644 --- a/doc/user/application_security/security_dashboard/index.md +++ b/doc/user/application_security/security_dashboard/index.md @@ -8,24 +8,24 @@ info: To determine the technical writer assigned to the Stage/Group associated w # GitLab Security Dashboard **(ULTIMATE)** The Security Dashboard is a good place to get an overview of all the security -vulnerabilities in your groups, projects and pipelines. +vulnerabilities in your groups, projects, and pipelines. -You can also drill down into a vulnerability and get extra information, see which -project it comes from, the file it's in, and various metadata to help you analyze -the risk. You can also take actions on vulnerabilities by creating an issue for them, -or by dismissing them. +You can also drill down into a vulnerability and get extra information. This includes the project it +comes from, any related file(s), and metadata that helps you analyze the risk it poses. You can also +dismiss a vulnerability or create an issue for it. To benefit from the Security Dashboard you must first configure one of the -[security reports](../index.md). +[security scanners](../index.md). ## Supported reports -The Security Dashboard supports the following reports: +The Security Dashboard displays vulnerabilities detected by scanners such as: - [Container Scanning](../container_scanning/index.md) - [Dynamic Application Security Testing](../dast/index.md) - [Dependency Scanning](../dependency_scanning/index.md) - [Static Application Security Testing](../sast/index.md) +- And others! ## Requirements @@ -43,10 +43,13 @@ To use the instance, group, project, or pipeline security dashboard: At the pipeline level, the Security section displays the vulnerabilities present in the branch of the project the pipeline was run against. -Visit the page for any pipeline which has run any of the [supported reports](#supported-reports). Click the **Security** tab to view the Security findings. -  +Visit the page for any pipeline that ran any of the [supported reports](#supported-reports). To view +the pipeline's security findings, select the **Security** tab when viewing the pipeline. + + + NOTE: **Note:** A pipeline consists of multiple jobs, including SAST and DAST scanning. If any job fails to finish for any reason, the security dashboard will not show SAST scanner output. For example, if the SAST job finishes but the DAST job fails, the security dashboard will not show SAST results. The analyzer will output an [exit code](../../../development/integrations/secure.md#exit-code) on failure. @@ -56,7 +59,8 @@ A pipeline consists of multiple jobs, including SAST and DAST scanning. If any j At the project level, the Security Dashboard displays the vulnerabilities merged into your project's [default branch](../../project/repository/branches/index.md#default-branch). Access it by navigating -to **Security & Compliance > Security Dashboard**. +to **Security & Compliance > Security Dashboard**. By default, the Security Dashboard displays all +detected and confirmed vulnerabilities. The Security Dashboard first displays the total number of vulnerabilities by severity (for example, Critical, High, Medium, Low). Below this, a table displays each vulnerability's status, severity, @@ -67,7 +71,7 @@ You can filter the vulnerabilities by: - Status - Severity -- Report type +- Scanner You can also dismiss vulnerabilities in the table: @@ -82,31 +86,21 @@ You can also dismiss vulnerabilities in the table: The group Security Dashboard gives an overview of the vulnerabilities in the default branches of the projects in a group and its subgroups. Access it by navigating to **Security > Security Dashboard** -for your group. +for your group. By default, the Security Dashboard displays all detected and confirmed +vulnerabilities. NOTE: **Note:** The Security Dashboard only shows projects with [security reports](#supported-reports) enabled in a group. - - -You can filter which vulnerabilities the Security Dashboard displays by: - -- Status -- Severity -- Report type -- Project - -A table lists the vulnerabilities, sorted by severity. The table shows each vulnerability's status, -severity, and description. Clicking a vulnerability takes you to its [Vulnerability Details](../vulnerabilities) -page to view more information about that vulnerability. + -Next to the list is a timeline chart that shows how many open +There is a timeline chart that shows how many open vulnerabilities your projects had at various points in time. You can filter among 30, 60, and 90 days, with the default being 90. Hover over the chart to get more details about the open vulnerabilities at a specific time. -Below the timeline chart is a list of projects, grouped and sorted by the severity of the vulnerability found: +Next to the timeline chart is a list of projects, grouped and sorted by the severity of the vulnerability found: - F: 1 or more "critical" - D: 1 or more "high" or "unknown" @@ -117,7 +111,7 @@ Below the timeline chart is a list of projects, grouped and sorted by the severi Projects with no vulnerability tests configured will not appear in the list. Additionally, dismissed vulnerabilities are not included either. -Read more on how to [interact with the vulnerabilities](../index.md#interacting-with-the-vulnerabilities). +Navigate to the group's [Vulnerability Report](#vulnerability-list) to view the vulnerabilities found. ## Instance Security Dashboard @@ -195,10 +189,21 @@ to configure daily security scans. Each dashboard's vulnerability list contains vulnerabilities from the latest scans that were merged into the default branch. -Click any vulnerability in the table to see more information on that vulnerability. To create an -issue associated with the vulnerability, click the **Create Issue** button. - + + +You can filter which vulnerabilities the Security Dashboard displays by: + +- Status +- Severity +- Scanner +- Project + +Clicking any vulnerability in the table takes you to its +[Vulnerability Details](../vulnerabilities) page to see more information on that vulnerability. +To create an issue associated with the vulnerability, click the **Create Issue** button. + + Once you create the issue, the vulnerability list contains a link to the issue and an icon whose color indicates the issue's status (green for open issues, blue for closed issues). @@ -216,3 +221,5 @@ questions that you know someone might ask. Each scenario can be a third-level heading, e.g. `### Getting error message X`. If you have none to add when creating a doc, leave this section in place but commented out to help encourage others to add to it in the future. --> + +Read more on how to [interact with the vulnerabilities](../index.md#interacting-with-the-vulnerabilities). diff --git a/doc/user/application_security/threat_monitoring/index.md b/doc/user/application_security/threat_monitoring/index.md index a6738677454..c916cdbfe7c 100644 --- a/doc/user/application_security/threat_monitoring/index.md +++ b/doc/user/application_security/threat_monitoring/index.md @@ -99,6 +99,11 @@ deployment platform. Changes performed outside of this tab are reflected upon refresh. Enforcement status changes are deployed directly to a deployment namespace of the selected environment. +By default, the network policy list contains predefined policies in a +disabled state. Once enabled,a predefined policy deploys to the +selected environment's deployment platform and you can manage it like +the regular policies. + NOTE: **Note:** If you're using [Auto DevOps](../../../topics/autodevops/index.md) and change a policy in this section, your `auto-deploy-values.yaml` file diff --git a/doc/user/application_security/vulnerabilities/img/standalone_vulnerability_page_merge_request_button_dropdown_v13_1.png b/doc/user/application_security/vulnerabilities/img/standalone_vulnerability_page_merge_request_button_dropdown_v13_1.png Binary files differdeleted file mode 100644 index 2063762d3eb..00000000000 --- a/doc/user/application_security/vulnerabilities/img/standalone_vulnerability_page_merge_request_button_dropdown_v13_1.png +++ /dev/null diff --git a/doc/user/application_security/vulnerabilities/img/standalone_vulnerability_page_merge_request_button_v13_1.png b/doc/user/application_security/vulnerabilities/img/standalone_vulnerability_page_merge_request_button_v13_1.png Binary files differdeleted file mode 100644 index ee4e97bcafe..00000000000 --- a/doc/user/application_security/vulnerabilities/img/standalone_vulnerability_page_merge_request_button_v13_1.png +++ /dev/null diff --git a/doc/user/application_security/vulnerabilities/img/standalone_vulnerability_page_v13_1.png b/doc/user/application_security/vulnerabilities/img/standalone_vulnerability_page_v13_1.png Binary files differdeleted file mode 100644 index e0e0fdb6f6e..00000000000 --- a/doc/user/application_security/vulnerabilities/img/standalone_vulnerability_page_v13_1.png +++ /dev/null diff --git a/doc/user/application_security/vulnerabilities/img/standalone_vulnerability_page_download_patch_button_v13_1.png b/doc/user/application_security/vulnerabilities/img/vulnerability_page_download_patch_button_v13_1.png Binary files differindex b925c342a11..b925c342a11 100644 --- a/doc/user/application_security/vulnerabilities/img/standalone_vulnerability_page_download_patch_button_v13_1.png +++ b/doc/user/application_security/vulnerabilities/img/vulnerability_page_download_patch_button_v13_1.png diff --git a/doc/user/application_security/vulnerabilities/img/vulnerability_page_merge_request_button_dropdown_v13_1.png b/doc/user/application_security/vulnerabilities/img/vulnerability_page_merge_request_button_dropdown_v13_1.png Binary files differnew file mode 100644 index 00000000000..05ca74c3d5c --- /dev/null +++ b/doc/user/application_security/vulnerabilities/img/vulnerability_page_merge_request_button_dropdown_v13_1.png diff --git a/doc/user/application_security/vulnerabilities/img/vulnerability_page_merge_request_button_v13_1.png b/doc/user/application_security/vulnerabilities/img/vulnerability_page_merge_request_button_v13_1.png Binary files differnew file mode 100644 index 00000000000..a3034a7db04 --- /dev/null +++ b/doc/user/application_security/vulnerabilities/img/vulnerability_page_merge_request_button_v13_1.png diff --git a/doc/user/application_security/vulnerabilities/img/vulnerability_page_v13_1.png b/doc/user/application_security/vulnerabilities/img/vulnerability_page_v13_1.png Binary files differnew file mode 100644 index 00000000000..30a7195e1ab --- /dev/null +++ b/doc/user/application_security/vulnerabilities/img/vulnerability_page_v13_1.png diff --git a/doc/user/application_security/vulnerabilities/index.md b/doc/user/application_security/vulnerabilities/index.md index d5cce6434d8..ffec4bf336d 100644 --- a/doc/user/application_security/vulnerabilities/index.md +++ b/doc/user/application_security/vulnerabilities/index.md @@ -5,16 +5,16 @@ group: Threat Insights info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# Standalone Vulnerability pages +# Vulnerability Pages > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/13561) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.0. Each security vulnerability in the [Security Dashboard](../security_dashboard/index.md#project-security-dashboard) has its own standalone page. - + -On the standalone vulnerability page, you can interact with the vulnerability in +On the vulnerability page, you can interact with the vulnerability in several different ways: - [Change the Vulnerability Status](#changing-vulnerability-status) - You can change the @@ -57,7 +57,7 @@ generates for you. GitLab supports the following scanners: When an automatic solution is available, the button in the header will show "Resolve with merge request": - + Selecting the button will create a merge request with the automatic solution. @@ -66,8 +66,8 @@ Selecting the button will create a merge request with the automatic solution. To manually apply the patch that was generated by GitLab for a vulnerability, select the dropdown arrow on the "Resolve with merge request" button, then select the "Download patch to resolve" option: - + This will change the button text to "Download patch to resolve". Click on it to download the patch: - + diff --git a/doc/user/asciidoc.md b/doc/user/asciidoc.md index 512a98d567b..4bd15c7922d 100644 --- a/doc/user/asciidoc.md +++ b/doc/user/asciidoc.md @@ -1,3 +1,10 @@ +--- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference, howto +--- + # AsciiDoc GitLab uses the [Asciidoctor](https://asciidoctor.org) gem to convert AsciiDoc content to HTML5. @@ -364,6 +371,89 @@ comment - content which is not included in the output document |=== ``` +### Colors + +It’s possible to have color written in `HEX`, `RGB`, or `HSL` format rendered with a color indicator. +Supported formats (named colors are not supported): + +- HEX: `` `#RGB[A]` `` or `` `#RRGGBB[AA]` `` +- RGB: `` `RGB[A](R, G, B[, A])` `` +- HSL: `` `HSL[A](H, S, L[, A])` `` + +Color written inside backticks will be followed by a color "chip": + +```plaintext +- `#F00` +- `#F00A` +- `#FF0000` +- `#FF0000AA` +- `RGB(0,255,0)` +- `RGB(0%,100%,0%)` +- `RGBA(0,255,0,0.3)` +- `HSL(540,70%,50%)` +- `HSLA(540,70%,50%,0.3)` +``` + +### STEM + +To activate equation and formula support, +set the `stem` attribute in the document's header to `latexmath`. +Equations and formulas will be rendered using [KaTeX](https://katex.org/): + +```plaintext +:stem: latexmath + +latexmath:[C = \alpha + \beta Y^{\gamma} + \epsilon] + +[stem] +++++ +sqrt(4) = 2 +++++ + +A matrix can be written as stem:[[[a,b\],[c,d\]\]((n),(k))]. +``` + +### Diagrams and flowcharts + +It's possible to generate diagrams and flowcharts from text in GitLab using +[Mermaid](https://mermaidjs.github.io/) or [PlantUML](https://plantuml.com). + +#### Mermaid + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/31818) in GitLab 13.3. + +Visit the [official page](https://mermaidjs.github.io/) for more details. +If you're new to using Mermaid or need help identifying issues in your Mermaid code, +the [Mermaid Live Editor](https://mermaid-js.github.io/mermaid-live-editor/) is a helpful tool +for creating and resolving issues within Mermaid diagrams. + +In order to generate a diagram or flowchart, you should write your text inside the `mermaid` block: + +```plaintext +[mermaid] +---- +graph LR + A[Square Rect] -- Link text --> B((Circle)) + A --> C(Round Rect) + B --> D{Rhombus} + C --> D +---- +``` + +#### PlantUML + +To make PlantUML available in GitLab, a GitLab administrator needs to enable it first. +Read more in [PlantUML & GitLab](../administration/integration/plantuml.md). + +Once enabled, you should write your text inside the `plantuml` block: + +```plaintext +[plantuml] +---- +Bob -> Alice : hello +---- +``` + ### Multimedia ```plaintext diff --git a/doc/user/clusters/applications.md b/doc/user/clusters/applications.md index 507ba25850d..3b04c7aac18 100644 --- a/doc/user/clusters/applications.md +++ b/doc/user/clusters/applications.md @@ -28,9 +28,9 @@ This namespace: To see a list of available applications to install. For a: - [Project-level cluster](../project/clusters/index.md), navigate to your project's - **{cloud-gear}** **Operations > Kubernetes**. + **Operations > Kubernetes**. - [Group-level cluster](../group/clusters/index.md), navigate to your group's - **{cloud-gear}** **Kubernetes** page. + **Kubernetes** page. NOTE: **Note:** As of GitLab 11.6, Helm will be upgraded to the latest version supported @@ -69,47 +69,23 @@ can lead to confusion during deployments. > - Introduced in GitLab 10.2 for project-level clusters. > - Introduced in GitLab 11.6 for group-level clusters. -> - A local Tiller option was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/209736) in GitLab 13.2 behind a feature flag, enabled by default. -> - The feature flag for local Tiller is enabled on GitLab.com. +> - [Uses a local Tiller](https://gitlab.com/gitlab-org/gitlab/-/issues/209736) since GitLab 13.2. [Helm](https://helm.sh/docs/) is a package manager for Kubernetes and is used to install the GitLab-managed apps. GitLab runs each `helm` command in a pod within the `gitlab-managed-apps` namespace inside the cluster. -As of GitLab 13.2, the integration uses a local -[Tiller](https://v2.helm.sh/docs/glossary/#tiller) by default. When using a -local Tiller, the Helm application does not need to be installed and will not -be shown in the list of applications. +GitLab's integration uses Helm 2 with a local +[Tiller](https://v2.helm.sh/docs/glossary/#tiller) server for managing +applications. Prior to [GitLab 13.2](https://gitlab.com/gitlab-org/gitlab/-/issues/209736), +GitLab used an in-cluster Tiller server in the `gitlab-managed-apps` +namespace. This server can now be safely removed. NOTE: **Note:** GitLab's Helm integration does not support installing applications behind a proxy, but a [workaround](../../topics/autodevops/index.md#install-applications-behind-a-proxy) is available. -### Enable or disable local Tiller **(CORE ONLY)** - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/209736) in GitLab 13.2 -> - The option to disable local Tiller is [planned for removal](https://gitlab.com/gitlab-org/gitlab/-/issues/209736) in GitLab 13.3 - -Local Tiller is under development, but is ready for production use. It is -deployed behind a feature flag that is **enabled by default**. -[GitLab administrators with access to the GitLab Rails console](../../administration/feature_flags.md) -can enable it for your instance. - -To enable it: - -```ruby -# Instance-wide -Feature.enable(:managed_apps_local_tiller) -``` - -To disable it: - -```ruby -# Instance-wide -Feature.disable(:managed_apps_local_tiller) -``` - ### cert-manager > Introduced in GitLab 11.6 for project- and group-level clusters. @@ -311,7 +287,7 @@ This feature: For example: ```shell - kubectl logs -n gitlab-managed-apps $(kubectl get pod -n gitlab-managed-apps -l app=nginx-ingress,component=controller --no-headers=true -o custom-columns=:metadata.name) modsecurity-log -f + kubectl -n gitlab-managed-apps logs -l app=nginx-ingress,component=controller -c modsecurity-log -f ``` To enable WAF, switch its respective toggle to the enabled position when installing or updating [Ingress application](#ingress). @@ -343,7 +319,7 @@ To help you tune your WAF rules, you can globally set your WAF to either To change your WAF's mode: 1. [Install ModSecurity](../../topics/web_application_firewall/quick_start_guide.md) if you have not already done so. -1. Navigate to **{cloud-gear}** **Operations > Kubernetes**. +1. Navigate to **Operations > Kubernetes**. 1. In **Applications**, scroll to **Ingress**. 1. Under **Global default**, select your desired mode. 1. Click **Save changes**. @@ -535,7 +511,7 @@ To enable log shipping: 1. Ensure your cluster contains at least 3 nodes of instance types larger than `f1-micro`, `g1-small`, or `n1-standard-1`. -1. Navigate to **{cloud-gear}** **Operations > Kubernetes**. +1. Navigate to **Operations > Kubernetes**. 1. In **Kubernetes Cluster**, select a cluster. 1. In the **Applications** section, find **Elastic Stack** and click **Install**. @@ -547,7 +523,7 @@ file. NOTE: **Note:** The chart deploys 3 identical Elasticsearch pods which can't be colocated, and each -require 1 CPU and 2 GB of RAM, making them incompatible with clusters containing +requires 1 CPU and 2 GB of RAM, making them incompatible with clusters containing fewer than 3 nodes or consisting of `f1-micro`, `g1-small`, `n1-standard-1`, or `*-highcpu-2` instance types. @@ -601,7 +577,7 @@ your data. Fluentd sends logs in syslog format. To enable Fluentd: -1. Navigate to **{cloud-gear}** **Operations > Kubernetes** and click +1. Navigate to **Operations > Kubernetes** and click **Applications**. You will be prompted to enter a host, port and protocol where the WAF logs will be sent to via syslog. 1. Provide the host domain name or URL in **SIEM Hostname**. @@ -719,7 +695,7 @@ for the available configuration options. NOTE: **Note:** Support for installing the Ingress managed application is provided by the GitLab Configure group. -If you run into unknown issues, please [open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new) and ping at least 2 people from the [Configure group](https://about.gitlab.com/handbook/product/categories/#configure-group). +If you run into unknown issues, please [open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new) and ping at least 2 people from the [Configure group](https://about.gitlab.com/handbook/product/product-categories/#configure-group). ### Install cert-manager using GitLab CI/CD @@ -760,7 +736,7 @@ available configuration options. NOTE: **Note:** Support for installing the Cert Manager managed application is provided by the GitLab Configure group. -If you run into unknown issues, please [open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new) and ping at least 2 people from the [Configure group](https://about.gitlab.com/handbook/product/categories/#configure-group). +If you run into unknown issues, please [open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new) and ping at least 2 people from the [Configure group](https://about.gitlab.com/handbook/product/product-categories/#configure-group). ### Install Sentry using GitLab CI/CD @@ -951,17 +927,15 @@ For an overview, see the [Container Network Security Demo for GitLab 12.8](https Enable Cilium in the `.gitlab/managed-apps/config.yaml` file to install it: ```yaml -# possible values are gke, eks or you can leave it blank +# possible values are gke or eks clusterType: gke cilium: installed: true ``` -The `clusterType` variable enables the recommended Helm variables for -a corresponding cluster type. The default value is blank. You can -check the recommended variables for each cluster type in the official -documentation: +The `clusterType` variable enables the recommended Helm variables for a corresponding cluster type. +You can check the recommended variables for each cluster type in the official documentation: - [Google GKE](https://docs.cilium.io/en/stable/gettingstarted/k8s-install-gke/#deploy-cilium) - [AWS EKS](https://docs.cilium.io/en/stable/gettingstarted/k8s-install-eks/#deploy-cilium) @@ -972,6 +946,11 @@ management project. Refer to the [Cilium chart](https://github.com/cilium/cilium/tree/master/install/kubernetes/cilium) for the available configuration options. +You can check Cilium's installation status on the cluster management page: + +- [Project-level cluster](../project/clusters/index.md): Navigate to your project's **Operations > Kubernetes** page. +- [Group-level cluster](../group/clusters/index.md): Navigate to your group's **Kubernetes** page. + CAUTION: **Caution:** Installation and removal of the Cilium requires a **manual** [restart](https://docs.cilium.io/en/stable/gettingstarted/k8s-install-gke/#restart-unmanaged-pods) @@ -1004,7 +983,7 @@ The Cilium monitor log for traffic is logged out by the `cilium-monitor` sidecar container. You can check these logs with the following command: ```shell -kubectl -n gitlab-managed-apps logs cilium-XXXX cilium-monitor +kubectl -n gitlab-managed-apps logs -l k8s-app=cilium -c cilium-monitor ``` You can disable the monitor log in `.gitlab/managed-apps/cilium/values.yaml`: @@ -1127,7 +1106,7 @@ falco: You can check these logs with the following command: ```shell -kubectl logs -l app=falco -n gitlab-managed-apps +kubectl -n gitlab-managed-apps logs -l app=falco ``` NOTE: **Note:** @@ -1183,7 +1162,7 @@ below are examples and should be replaced with settings specific to your environ ui: enabled: true server: - # Disable the built in data storage volume as it's not safe for Hight Availability mode + # Disable the built in data storage volume as it's not safe for High Availability mode dataStorage: enabled: false # Enable High Availability Mode @@ -1210,9 +1189,9 @@ server: } ``` -Once you have successfully installed Vault, you will need to [initialize the Vault](https://learn.hashicorp.com/vault/getting-started/deploy#initializing-the-vault) +Once you have successfully installed Vault, you will need to [initialize the Vault](https://learn.hashicorp.com/tutorials/vault/getting-started-deploy#initializing-the-vault) and obtain the initial root token. You will need access to your Kubernetes cluster that Vault has been deployed into in order to do this. -To initialise the Vault, get a shell to one of the Vault pods running inside Kubernetes (typically this is done by using the `kubectl` command line tool). +To initialize the Vault, get a shell to one of the Vault pods running inside Kubernetes (typically this is done by using the `kubectl` command line tool). Once you have a shell into the pod, run the `vault operator init` command: ```shell @@ -1278,7 +1257,7 @@ available configuration options. NOTE: **Note:** Support for installing the JupyterHub managed application is provided by the GitLab Configure group. -If you run into unknown issues, please [open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new) and ping at least 2 people from the [Configure group](https://about.gitlab.com/handbook/product/categories/#configure-group). +If you run into unknown issues, please [open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new) and ping at least 2 people from the [Configure group](https://about.gitlab.com/handbook/product/product-categories/#configure-group). ### Install Elastic Stack using GitLab CI/CD @@ -1393,7 +1372,7 @@ If you plan to use GitLab Serverless capabilities, be sure to set an A record wi NOTE: **Note:** Support for installing the Knative managed application is provided by the GitLab Configure group. -If you run into unknown issues, please [open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new) and ping at least 2 people from the [Configure group](https://about.gitlab.com/handbook/product/categories/#configure-group). +If you run into unknown issues, please [open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new) and ping at least 2 people from the [Configure group](https://about.gitlab.com/handbook/product/product-categories/#configure-group). #### Knative Metrics @@ -1587,7 +1566,7 @@ To avoid installation errors: If you're using a managed cluster on AWS EKS, and you are not able to install some of the managed apps, consider checking the logs. -You can check the logs by running following commands: +You can check the logs by running the following commands: ```shell kubectl get pods --all-namespaces diff --git a/doc/user/clusters/management_project.md b/doc/user/clusters/management_project.md index 892d2bce184..f7241444a6b 100644 --- a/doc/user/clusters/management_project.md +++ b/doc/user/clusters/management_project.md @@ -73,10 +73,10 @@ configure cluster: name: production ``` -### Setting the environment scope **(PREMIUM)** +### Setting the environment scope [Environment -scopes](../project/clusters/index.md#setting-the-environment-scope-premium) +scopes](../project/clusters/index.md#setting-the-environment-scope) are usable when associating multiple clusters to the same management project. diff --git a/doc/user/compliance/compliance_dashboard/img/compliance_dashboard_v13_2.png b/doc/user/compliance/compliance_dashboard/img/compliance_dashboard_v13_2.png Binary files differdeleted file mode 100644 index e1edfcdd024..00000000000 --- a/doc/user/compliance/compliance_dashboard/img/compliance_dashboard_v13_2.png +++ /dev/null diff --git a/doc/user/compliance/compliance_dashboard/img/compliance_dashboard_v13_3_1.png b/doc/user/compliance/compliance_dashboard/img/compliance_dashboard_v13_3_1.png Binary files differnew file mode 100644 index 00000000000..a06f8812b41 --- /dev/null +++ b/doc/user/compliance/compliance_dashboard/img/compliance_dashboard_v13_3_1.png diff --git a/doc/user/compliance/compliance_dashboard/img/failed_icon_v13_3.png b/doc/user/compliance/compliance_dashboard/img/failed_icon_v13_3.png Binary files differnew file mode 100644 index 00000000000..c3f386c9dee --- /dev/null +++ b/doc/user/compliance/compliance_dashboard/img/failed_icon_v13_3.png diff --git a/doc/user/compliance/compliance_dashboard/img/success_icon_v13_3.png b/doc/user/compliance/compliance_dashboard/img/success_icon_v13_3.png Binary files differnew file mode 100644 index 00000000000..ea6ca924f81 --- /dev/null +++ b/doc/user/compliance/compliance_dashboard/img/success_icon_v13_3.png diff --git a/doc/user/compliance/compliance_dashboard/img/warning_icon_v13_3.png b/doc/user/compliance/compliance_dashboard/img/warning_icon_v13_3.png Binary files differnew file mode 100644 index 00000000000..168a7021948 --- /dev/null +++ b/doc/user/compliance/compliance_dashboard/img/warning_icon_v13_3.png diff --git a/doc/user/compliance/compliance_dashboard/index.md b/doc/user/compliance/compliance_dashboard/index.md index e7db73e25d9..5c05725d95b 100644 --- a/doc/user/compliance/compliance_dashboard/index.md +++ b/doc/user/compliance/compliance_dashboard/index.md @@ -17,7 +17,10 @@ for merging into production. To access the Compliance Dashboard for a group, navigate to **{shield}** **Security & Compliance > Compliance** on the group's menu. - + + +NOTE: **Note:** +The Compliance Dashboard shows only the latest MR on each project. ## Use cases @@ -34,3 +37,40 @@ You can use the dashboard to: - On [GitLab Ultimate](https://about.gitlab.com/pricing/) tier. - By **Administrators** and **Group Owners**. + +## Approval status and separation of duties + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/217939) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.3. + +We support a separation of duties policy between users who create and approve Merge Requests. +The approval status column can help you identify violations of this policy. +Our criteria for the separation of duties is as follows: + +- [A Merge Request author is **not** allowed to approve their Merge Request](../../project/merge_requests/merge_request_approvals.md#allowing-merge-request-authors-to-approve-their-own-merge-requests) +- [A Merge Request committer is **not** allowed to approve a Merge Request they have added commits to](../../project/merge_requests/merge_request_approvals.md#prevent-approval-of-merge-requests-by-their-committers) +- [The minimum number of approvals required to merge a Merge Request is **at least** two](../../project/merge_requests/merge_request_approvals.md#approval-rules) + +The "Approval status" column shows you, at a glance, whether a Merge Request is complying with the above. +This column has four states: + +| State | Description | +|:------|:------------| +| Empty | The Merge Request approval status is unknown | +|  | The Merge Request **does not** comply with any of the above criteria | +|  | The Merge Request complies with **some** of the above criteria | +|  | The Merge Request complies with **all** of the above criteria | + +If you do not see the success icon in your Compliance dashboard; please review the above criteria for the Merge Requests +project to make sure it complies with the separation of duties described above. + +## Chain of Custody report + +The Chain of Custody report allows customers to export a list of merge commits within the group. +The data provides a comprehensive view with respect to merge commits. It includes the merge commit SHA, +merge request author, merge request ID, merge user, pipeline ID, group name, project name, and merge request approvers. + +To download the Chain of Custody report, navigate to **{shield}** **Security & Compliance > Compliance** on the group's menu and click **List of all merge commits** + +NOTE: **Note:** +The Chain of Custody report download is a CSV file, with a maximum size of 15 MB. +The remaining records are truncated when this limit is reached. diff --git a/doc/user/compliance/license_compliance/img/denied_licenses_v13_3.png b/doc/user/compliance/license_compliance/img/denied_licenses_v13_3.png Binary files differnew file mode 100644 index 00000000000..aa3deb0c154 --- /dev/null +++ b/doc/user/compliance/license_compliance/img/denied_licenses_v13_3.png diff --git a/doc/user/compliance/license_compliance/img/license_compliance_add_license_v13_0.png b/doc/user/compliance/license_compliance/img/license_compliance_add_license_v13_0.png Binary files differindex 992c08edcd3..1366c569f17 100644 --- a/doc/user/compliance/license_compliance/img/license_compliance_add_license_v13_0.png +++ b/doc/user/compliance/license_compliance/img/license_compliance_add_license_v13_0.png diff --git a/doc/user/compliance/license_compliance/img/license_compliance_decision_v13_0.png b/doc/user/compliance/license_compliance/img/license_compliance_decision_v13_0.png Binary files differindex d6c6142c0e7..42bf8bd1ed5 100644 --- a/doc/user/compliance/license_compliance/img/license_compliance_decision_v13_0.png +++ b/doc/user/compliance/license_compliance/img/license_compliance_decision_v13_0.png diff --git a/doc/user/compliance/license_compliance/img/license_compliance_pipeline_tab_v13_0.png b/doc/user/compliance/license_compliance/img/license_compliance_pipeline_tab_v13_0.png Binary files differindex 9ae59e2b96b..49c66832f00 100644 --- a/doc/user/compliance/license_compliance/img/license_compliance_pipeline_tab_v13_0.png +++ b/doc/user/compliance/license_compliance/img/license_compliance_pipeline_tab_v13_0.png diff --git a/doc/user/compliance/license_compliance/img/license_compliance_search_v13_0.png b/doc/user/compliance/license_compliance/img/license_compliance_search_v13_0.png Binary files differindex 8ee55003768..5a4216dd645 100644 --- a/doc/user/compliance/license_compliance/img/license_compliance_search_v13_0.png +++ b/doc/user/compliance/license_compliance/img/license_compliance_search_v13_0.png diff --git a/doc/user/compliance/license_compliance/img/license_compliance_settings_v13_0.png b/doc/user/compliance/license_compliance/img/license_compliance_settings_v13_0.png Binary files differindex 52b26abd9c5..91f1eec2a23 100644 --- a/doc/user/compliance/license_compliance/img/license_compliance_settings_v13_0.png +++ b/doc/user/compliance/license_compliance/img/license_compliance_settings_v13_0.png diff --git a/doc/user/compliance/license_compliance/img/license_compliance_v13_0.png b/doc/user/compliance/license_compliance/img/license_compliance_v13_0.png Binary files differindex dc227bf05ef..20ed30a21e7 100644 --- a/doc/user/compliance/license_compliance/img/license_compliance_v13_0.png +++ b/doc/user/compliance/license_compliance/img/license_compliance_v13_0.png diff --git a/doc/user/compliance/license_compliance/img/license_list_v13_0.png b/doc/user/compliance/license_compliance/img/license_list_v13_0.png Binary files differindex 3964c837c6a..3c15d4fc99a 100644 --- a/doc/user/compliance/license_compliance/img/license_list_v13_0.png +++ b/doc/user/compliance/license_compliance/img/license_list_v13_0.png diff --git a/doc/user/compliance/license_compliance/img/policies_v13_0.png b/doc/user/compliance/license_compliance/img/policies_v13_0.png Binary files differindex 4712d2b7aba..4918a0e6b62 100644 --- a/doc/user/compliance/license_compliance/img/policies_v13_0.png +++ b/doc/user/compliance/license_compliance/img/policies_v13_0.png diff --git a/doc/user/compliance/license_compliance/index.md b/doc/user/compliance/license_compliance/index.md index fb287fb2bf6..47f14b93d29 100644 --- a/doc/user/compliance/license_compliance/index.md +++ b/doc/user/compliance/license_compliance/index.md @@ -9,10 +9,10 @@ info: To determine the technical writer assigned to the Stage/Group associated w > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/5483) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.0. -## Overview - -If you are using [GitLab CI/CD](../../../ci/README.md), you can search your project dependencies for their licenses -using License Compliance. +If you're using [GitLab CI/CD](../../../ci/README.md), you can use License Compliance to search your +project's dependencies for their licenses. You can then decide whether to allow or deny the use of +each license. For example, if your application uses an external (open source) library whose license +is incompatible with yours, then you can deny the use of that license. You can take advantage of License Compliance by either [including the job](#configuration) in your existing `.gitlab-ci.yml` file or by implicitly using @@ -24,7 +24,9 @@ source and target branches, and shows the information right on the merge request Denied licenses will be clearly visible with an `x` red icon next to them as well as new licenses which need a decision from you. In addition, you can [manually allow or deny](#policies) -licenses in your project's license compliance policy section. +licenses in your project's license compliance policy section. If GitLab detects a denied license +in a new commit, GitLab blocks any merge requests containing that commit and instructs the developer +to remove the license. NOTE: **Note:** If the license compliance report doesn't have anything to compare to, no information @@ -48,29 +50,23 @@ You can view and modify existing policies from the [policies](#policies) tab.  -## Use cases - -It helps you find what licenses your project uses in its dependencies, and decide for each of then -whether to allow it or forbid it. For example, your application is using an external (open source) -library whose license is incompatible with yours. - ## Supported languages and package managers The following languages and package managers are supported. -| Language | Package managers | Scan Tool | -|------------|-------------------------------------------------------------------|----------------------------------------------------------| -| JavaScript | [Bower](https://bower.io/), [npm](https://www.npmjs.com/) |[License Finder](https://github.com/pivotal/LicenseFinder)| -| Go | [Godep](https://github.com/tools/godep), [go mod](https://github.com/golang/go/wiki/Modules) |[License Finder](https://github.com/pivotal/LicenseFinder)| -| Java | [Gradle](https://gradle.org/), [Maven](https://maven.apache.org/) |[License Finder](https://github.com/pivotal/LicenseFinder)| -| .NET | [Nuget](https://www.nuget.org/) (.NET Framework is supported via the [mono project](https://www.mono-project.com/). Windows specific dependencies are not supported at this time.) |[License Finder](https://github.com/pivotal/LicenseFinder)| -| Python | [pip](https://pip.pypa.io/en/stable/) (Python is supported through [requirements.txt](https://pip.pypa.io/en/stable/user_guide/#requirements-files) and [Pipfile.lock](https://github.com/pypa/pipfile#pipfilelock).) |[License Finder](https://github.com/pivotal/LicenseFinder)| -| Ruby | [gem](https://rubygems.org/) |[License Finder](https://github.com/pivotal/LicenseFinder)| -| Objective-C, Swift | [Carthage](https://github.com/Carthage/Carthage) |[License Finder](https://github.com/pivotal/LicenseFinder)| +| Language | Package managers | Notes | Scan Tool | +|------------|------------------|-------|-----------| +| JavaScript | [Bower](https://bower.io/), [npm](https://www.npmjs.com/) | | [License Finder](https://github.com/pivotal/LicenseFinder) | +| Go | [Godep](https://github.com/tools/godep), [go mod](https://github.com/golang/go/wiki/Modules) | | [License Finder](https://github.com/pivotal/LicenseFinder) | +| Java | [Gradle](https://gradle.org/), [Maven](https://maven.apache.org/) | | [License Finder](https://github.com/pivotal/LicenseFinder) | +| .NET | [Nuget](https://www.nuget.org/) | The .NET Framework is supported via the [mono project](https://www.mono-project.com/). There are, however, some limitations. The scanner doesn't support Windows-specific dependencies and doesn't report dependencies of your project's listed dependencies. Also, the scanner always marks detected licenses for all dependencies as `unknown`. | [License Finder](https://github.com/pivotal/LicenseFinder) | +| Python | [pip](https://pip.pypa.io/en/stable/) | Python is supported through [requirements.txt](https://pip.pypa.io/en/stable/user_guide/#requirements-files) and [Pipfile.lock](https://github.com/pypa/pipfile#pipfilelock). | [License Finder](https://github.com/pivotal/LicenseFinder) | +| Ruby | [gem](https://rubygems.org/) | | [License Finder](https://github.com/pivotal/LicenseFinder)| +| Objective-C, Swift | [Carthage](https://github.com/Carthage/Carthage) | | [License Finder](https://github.com/pivotal/LicenseFinder) | NOTE: **Note:** - Java 8 and Gradle 1.x projects are not supported. +The minimum supported version of Maven is 3.2.5. ### Experimental support @@ -79,15 +75,15 @@ which means that the reported licenses might be incomplete or inaccurate. | Language | Package managers | Scan Tool | |------------|-------------------------------------------------------------------|----------------------------------------------------------| -| JavaScript | [yarn](https://yarnpkg.com/)|[License Finder](https://github.com/pivotal/LicenseFinder)| +| JavaScript | [Yarn](https://yarnpkg.com/)|[License Finder](https://github.com/pivotal/LicenseFinder)| | Go | go get, gvt, glide, dep, trash, govendor |[License Finder](https://github.com/pivotal/LicenseFinder)| -| Erlang | [rebar](https://www.rebar3.org/) |[License Finder](https://github.com/pivotal/LicenseFinder)| +| Erlang | [Rebar](https://www.rebar3.org/) |[License Finder](https://github.com/pivotal/LicenseFinder)| | Objective-C, Swift | [CocoaPods](https://cocoapods.org/) v0.39 and below |[License Finder](https://github.com/pivotal/LicenseFinder)| -| Elixir | [mix](https://elixir-lang.org/getting-started/mix-otp/introduction-to-mix.html) |[License Finder](https://github.com/pivotal/LicenseFinder)| -| C++/C | [conan](https://conan.io/) |[License Finder](https://github.com/pivotal/LicenseFinder)| +| Elixir | [Mix](https://elixir-lang.org/getting-started/mix-otp/introduction-to-mix.html) |[License Finder](https://github.com/pivotal/LicenseFinder)| +| C++/C | [Conan](https://conan.io/) |[License Finder](https://github.com/pivotal/LicenseFinder)| | Scala | [sbt](https://www.scala-sbt.org/) |[License Finder](https://github.com/pivotal/LicenseFinder)| -| Rust | [cargo](https://crates.io) |[License Finder](https://github.com/pivotal/LicenseFinder)| -| PHP | [composer](https://getcomposer.org/) |[License Finder](https://github.com/pivotal/LicenseFinder)| +| Rust | [Cargo](https://crates.io) |[License Finder](https://github.com/pivotal/LicenseFinder)| +| PHP | [Composer](https://getcomposer.org/) |[License Finder](https://github.com/pivotal/LicenseFinder)| ## Requirements @@ -106,24 +102,19 @@ For older versions of GitLab from 11.9 to 12.7, you must For GitLab versions earlier than 11.9, you can copy and use the job as defined that template. -NOTE: **Note:** -GitLab 13.0 removes the `License-Management.gitlab-ci.yml` template. -Use `License-Scanning.gitlab-ci.yml` instead. - Add the following to your `.gitlab-ci.yml` file: ```yaml include: - - template: License-Scanning.gitlab-ci.yml + - template: Security/License-Scanning.gitlab-ci.yml ``` -The included template will create a `license_scanning` job in your CI/CD pipeline -and scan your dependencies to find their licenses. +The included template creates a `license_scanning` job in your CI/CD pipeline and scans your +dependencies to find their licenses. NOTE: **Note:** -Before GitLab 12.8, the `license_scanning` job was named `license_management`. -GitLab 13.0 removes the `license_management` job, -so you're advised to migrate to the `license_scanning` job and used the new +Before GitLab 12.8, the `license_scanning` job was named `license_management`. GitLab 13.0 removes +the `license_management` job, so you must migrate to the `license_scanning` job and use the new `License-Scanning.gitlab-ci.yml` template. The results will be saved as a @@ -175,7 +166,7 @@ For example: ```yaml include: - - template: License-Scanning.gitlab-ci.yml + - template: Security/License-Scanning.gitlab-ci.yml variables: LICENSE_MANAGEMENT_SETUP_CMD: sh my-custom-install-script.sh @@ -196,7 +187,7 @@ after the template inclusion and specify any additional keys under it. For examp ```yaml include: - - template: License-Scanning.gitlab-ci.yml + - template: Security/License-Scanning.gitlab-ci.yml license_scanning: variables: @@ -211,7 +202,7 @@ Feel free to use it for the customization of Maven execution. For example: ```yaml include: - - template: License-Scanning.gitlab-ci.yml + - template: Security/License-Scanning.gitlab-ci.yml license_scanning: variables: @@ -239,7 +230,7 @@ or internally trusted certificate. For example: ```yaml include: - - template: License-Scanning.gitlab-ci.yml + - template: Security/License-Scanning.gitlab-ci.yml license_scanning: variables: @@ -262,7 +253,7 @@ by setting the `LM_PYTHON_VERSION` environment variable to `2`. ```yaml include: - - template: License-Scanning.gitlab-ci.yml + - template: Security/License-Scanning.gitlab-ci.yml license_scanning: variables: @@ -282,7 +273,7 @@ to inject a custom [`pip.conf`](https://pip.pypa.io/en/stable/user_guide/#config ```yaml include: - - template: License-Scanning.gitlab-ci.yml + - template: Security/License-Scanning.gitlab-ci.yml license_scanning: variables: @@ -339,13 +330,13 @@ strict-ssl = false ### Configuring Yarn projects -You can configure Yarn projects by using a [`.yarnrc.yml`](https://yarnpkg.com/configuration/yarnrc) +You can configure Yarn projects by using a [`.yarnrc.yml`](https://yarnpkg.com/configuration/yarnrc/) file. #### Using private Yarn registries If you have a private Yarn registry you can use the -[`npmRegistryServer`](https://yarnpkg.com/configuration/yarnrc#npmRegistryServer) +[`npmRegistryServer`](https://yarnpkg.com/configuration/yarnrc/#npmRegistryServer) setting to specify its location. For example: @@ -385,6 +376,8 @@ You can supply a custom root certificate to complete TLS verification by using t specifying a `ca` setting in a [`.bowerrc`](https://bower.io/docs/config/#bowerrc-specification) file. +### Configuring Bundler projects + #### Using private Bundler registries If you have a private Bundler registry you can use the @@ -405,6 +398,63 @@ specifying a [`BUNDLE_SSL_CA_CERT`](https://bundler.io/v2.0/man/bundle-config.1. [environment variable](../../../ci/variables/README.md#custom-environment-variables) in the job definition. +### Configuring Cargo projects + +#### Using private Cargo registries + +If you have a private Cargo registry you can use the +[`registries`](https://doc.rust-lang.org/cargo/reference/registries.html) +setting to specify its location. + +For example: + +```toml +[registries] +my-registry = { index = "https://my-intranet:8080/git/index" } +``` + +#### Custom root certificates for Cargo + +To supply a custom root certificate to complete TLS verification, do one of the following: + +- Use the `ADDITIONAL_CA_CERT_BUNDLE` [environment variable](#available-variables). +- Specify a [`CARGO_HTTP_CAINFO`](https://doc.rust-lang.org/cargo/reference/environment-variables.html) + [environment variable](../../../ci/variables/README.md#custom-environment-variables) + in the job definition. + +### Configuring Composer projects + +#### Using private Composer registries + +If you have a private Composer registry you can use the +[`repositories`](https://getcomposer.org/doc/05-repositories.md) +setting to specify its location. + +For example: + +```json +{ + "repositories": [ + { "packagist.org": false }, + { + "type": "composer", + "url": "https://composer.example.com" + } + ], + "require": { + "monolog/monolog": "1.0.*" + } +} +``` + +#### Custom root certificates for Composer + +You can supply a custom root certificate to complete TLS verification by using the +`ADDITIONAL_CA_CERT_BUNDLE` [environment variable](#available-variables), or by +specifying a [`COMPOSER_CAFILE`](https://getcomposer.org/doc/03-cli.md#composer-cafile) +[environment variable](../../../ci/variables/README.md#custom-environment-variables) +in the job definition. + ### Configuring Conan projects You can configure [Conan](https://conan.io/) projects by adding a `.conan` directory to your @@ -503,7 +553,7 @@ environment variable. For example: ```yaml include: - - template: License-Scanning.gitlab-ci.yml + - template: Security/License-Scanning.gitlab-ci.yml license_scanning: variables: @@ -560,7 +610,7 @@ Should be changed to: ```yaml include: - - template: License-Scanning.gitlab-ci.yml + - template: Security/License-Scanning.gitlab-ci.yml license_scanning: artifacts: @@ -625,7 +675,7 @@ the License Compliance Docker image hosted on your local Docker container regist ```yaml include: - - template: License-Scanning.gitlab-ci.yml + - template: Security/License-Scanning.gitlab-ci.yml license_scanning: image: @@ -645,6 +695,16 @@ Additional configuration may be needed for connecting to [private Python repositories](#using-private-python-repos), and [private Yarn registries](#using-private-yarn-registries). +### SPDX license list name matching + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/212388) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.3. + +Prior to GitLab 13.3, offline environments required an exact name match for [project policies](#policies). +In GitLab 13.3 and later, GitLab matches the name of [project policies](#policies) +with identifiers from the [SPDX license list](https://spdx.org/licenses/). +A local copy of the SPDX license list is distributed with the GitLab instance. If needed, the GitLab +instance's administrator can manually update it with a [Rake task](../../../raketasks/spdx.md). + Exact name matches are required for [project policies](#policies) when running in an offline environment ([see related issue](https://gitlab.com/gitlab-org/gitlab/-/issues/212388)). @@ -675,10 +735,16 @@ in your project's sidebar, and you'll see the licenses displayed, where: > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/22465) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.9. -The **Policies** tab allows you to see your project's software license policies -and the associated classifications for each. +Policies allow you to specify licenses that are `allowed` or `denied` in a project. If a `denied` +license is newly committed it will disallow a merge request and instruct the developer to remove it. +Note, the merge request will not be able to be merged until the `denied` license is removed. +You may add a [`License-Check` approval rule](#enabling-license-approvals-within-a-project), +which enables a designated approver that can approve and then merge a merge request with `denied` license. + + -Policies can be configured by maintainers of the project. +The **Policies** tab in the project's license compliance section displays your project's license +policies. Project maintainers can specify policies in this section.   @@ -742,7 +808,7 @@ project's `.gitlab-ci.yml` file. ```yaml include: - - template: License-Scanning.gitlab-ci.yml + - template: Security/License-Scanning.gitlab-ci.yml license_scanning: variables: diff --git a/doc/user/discussions/img/quickly_assign_commenter_v13_1.png b/doc/user/discussions/img/quickly_assign_commenter_v13_1.png Binary files differindex e19a3ed4f75..7f8ce62fe88 100644 --- a/doc/user/discussions/img/quickly_assign_commenter_v13_1.png +++ b/doc/user/discussions/img/quickly_assign_commenter_v13_1.png diff --git a/doc/user/discussions/img/resolve_thread_button.png b/doc/user/discussions/img/resolve_thread_button.png Binary files differdeleted file mode 100644 index ca0a3e50550..00000000000 --- a/doc/user/discussions/img/resolve_thread_button.png +++ /dev/null diff --git a/doc/user/discussions/img/resolve_thread_button_v13_3.png b/doc/user/discussions/img/resolve_thread_button_v13_3.png Binary files differnew file mode 100644 index 00000000000..1d7b10ce25d --- /dev/null +++ b/doc/user/discussions/img/resolve_thread_button_v13_3.png diff --git a/doc/user/discussions/index.md b/doc/user/discussions/index.md index 599f46b2c40..f39d0d6c217 100644 --- a/doc/user/discussions/index.md +++ b/doc/user/discussions/index.md @@ -1,7 +1,8 @@ --- -stage: Plan -group: Project Management -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference, howto --- # Threads @@ -89,15 +90,20 @@ When a link of a commit reference is found in a thread inside a merge request, it will be automatically converted to a link in the context of the current merge request. -### Jumping between unresolved threads +### Jumping between unresolved threads (deprecated) + +> - [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/199718) in GitLab 13.3. +> - This button's removal is behind a feature flag enabled by default. +> - For GitLab self-managed instances, GitLab administrators with access to the + [GitLab Rails console](../../administration/feature_flags.md) can opt to disable it by running + `Feature.disable(:hide_jump_to_next_unresolved_in_threads)` (for the instance) or + `Feature.disable(:hide_jump_to_next_unresolved_in_threads, Project.find(<project id>))` + (per project.) **(CORE ONLY)** When a merge request has a large number of comments it can be difficult to track what remains unresolved. You can jump between unresolved threads with the Jump button next to the Reply field on a thread. -You can also jump to the next unresolved thread from the button next to the -resolved threads tracker. - You can also use keyboard shortcuts to navigate among threads: - Use <kbd>n</kbd> to jump to the next unresolved thread. @@ -110,7 +116,7 @@ You can also use keyboard shortcuts to navigate among threads: You can mark a thread as resolved by clicking the **Resolve thread** button at the bottom of the thread. - + Alternatively, you can mark each comment as resolved individually. @@ -242,7 +248,7 @@ After you click on the image, a comment form will be displayed that would be the of your thread. Once you save your comment, you will see a new badge displayed on top of your image. This badge represents your thread. ->**Note:** +NOTE: **Note:** This thread badge is typically associated with a number that is only used as a visual reference for each thread. In the merge request thread tab, this badge will be indicated with a comment icon since each thread will render a new diff --git a/doc/user/gitlab_com/index.md b/doc/user/gitlab_com/index.md index bcaba97cab7..aa9e6715335 100644 --- a/doc/user/gitlab_com/index.md +++ b/doc/user/gitlab_com/index.md @@ -29,7 +29,10 @@ gitlab.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAA ## Mail configuration GitLab.com sends emails from the `mg.gitlab.com` domain via [Mailgun](https://www.mailgun.com/) and has -its own dedicated IP address (`198.61.254.240`). +its own dedicated IP address (`192.237.158.143`). + +NOTE: **Note:** +The IP address for `mg.gitlab.com` is subject to change at any time. ## Backups @@ -81,6 +84,7 @@ Below are the current settings regarding [GitLab CI/CD](../../ci/README.md). | Artifacts [expiry time](../../ci/yaml/README.md#artifactsexpire_in) | From June 22, 2020, deleted after 30 days unless otherwise specified (artifacts created before that date have no expiry). | deleted after 30 days unless otherwise specified | | Scheduled Pipeline Cron | `*/5 * * * *` | `19 * * * *` | | [Max jobs in active pipelines](../../administration/instance_limits.md#number-of-jobs-in-active-pipelines) | `500` for Free tier, unlimited otherwise | Unlimited +| [Max CI/CD subscriptions to a project](../../administration/instance_limits.md#number-of-cicd-subscriptions-to-a-project) | `2` | Unlimited | | [Max pipeline schedules in projects](../../administration/instance_limits.md#number-of-pipeline-schedules) | `10` for Free tier, `50` for all paid tiers | Unlimited | | [Max number of instance level variables](../../administration/instance_limits.md#number-of-instance-level-variables) | `25` | `25` | | [Scheduled Job Archival](../../user/admin_area/settings/continuous_integration.md#archive-jobs-core-only) | 3 months | Never | @@ -112,12 +116,13 @@ All our runners are deployed into Google Cloud Platform (GCP) - any IP based firewall can be configured by looking up all [IP address ranges or CIDR blocks for GCP](https://cloud.google.com/compute/docs/faq#where_can_i_find_product_name_short_ip_ranges). -## Maximum number of webhooks +## Webhooks A limit of: - 100 webhooks applies to projects. - 50 webhooks applies to groups. **(BRONZE ONLY)** +- Payload is limited to 25MB ## Shared Runners @@ -212,10 +217,6 @@ sentry_dsn = "X" [runners.machine] IdleCount = 50 IdleTime = 3600 - OffPeakPeriods = ["* * * * * sat,sun *"] - OffPeakTimezone = "UTC" - OffPeakIdleCount = 15 - OffPeakIdleTime = 3600 MaxBuilds = 1 # For security reasons we delete the VM after job has finished so it's not reused. MachineName = "srm-%s" MachineDriver = "google" @@ -233,6 +234,16 @@ sentry_dsn = "X" "engine-opt=fixed-cidr-v6=fc00::/7", "google-operation-backoff-initial-interval=2" # Custom flag from forked docker-machine, for more information check https://github.com/docker/machine/pull/4600 ] + [[runners.machine.autoscaling]] + Periods = ["* * * * * sat,sun *"] + Timezone = "UTC" + IdleCount = 70 + IdleTime = 3600 + [[runners.machine.autoscaling]] + Periods = ["* 30-59 3 * * * *", "* 0-30 4 * * * *"] + Timezone = "UTC" + IdleCount = 700 + IdleTime = 3600 [runners.cache] Type = "gcs" Shared = true diff --git a/doc/user/group/clusters/index.md b/doc/user/group/clusters/index.md index 89e0c4898fb..e61b24f84f6 100644 --- a/doc/user/group/clusters/index.md +++ b/doc/user/group/clusters/index.md @@ -84,7 +84,7 @@ your cluster, which can cause deployment jobs to fail. To clear the cache: -1. Navigate to your group’s **{cloud-gear}** **Kubernetes** page, +1. Navigate to your group’s **Kubernetes** page, and select your cluster. 1. Expand the **Advanced settings** section. 1. Click **Clear cluster cache**. diff --git a/doc/user/group/custom_project_templates.md b/doc/user/group/custom_project_templates.md index ebeacda24c6..fd8d966fbe1 100644 --- a/doc/user/group/custom_project_templates.md +++ b/doc/user/group/custom_project_templates.md @@ -9,8 +9,49 @@ info: To determine the technical writer assigned to the Stage/Group associated w > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/6861) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.6. -When you create a new [project](../project/index.md), creating it based on custom project templates is -a convenient option. +Custom project templates are useful for organizations that need to create many similar types of [projects](../project/index.md) and want to start from the same jumping-off point. + +## Setting up Group-level Project Templates + +To use a custom project template for a new project you need to: + +1. [Create a 'templates' subgroup](subgroups/index.md). +1. [Add repositories (projects) to the that new subgroup](index.md#add-projects-to-a-group), as your templates. +1. Edit your group's settings to look to your 'templates' subgroup for templates: + 1. In the left-hand menu, click **{settings}** **Settings > General**. + + NOTE: **Note:** + If you don't have access to the group's settings, you may not have sufficient privileges (for example, you may need developer or higher permissions). + + 1. Scroll to **Custom project templates** and click **Expand**. If no **Custom project templates** section displays, make sure you've created a subgroup, and added a project (repository) to it. + 1. Select the 'templates' subgroup. + +### Example structure + +Here is a sample group/project structure for a hypothetical "Acme Co" for project templates: + +```txt +# GitLab instance and group +gitlab.com/acmeco/ + # Subgroups + internal + tools + # Subgroup for handling project templates + websites + templates + # Project templates + client-site-django + client-site-gatsby + client-site-hTML + + # Other projects + client-site-a + client-site-b + client-site-c + ... +``` + +### Adjust Settings Users can configure a GitLab group that serves as template source under a group's **Settings > General > Custom project templates**. diff --git a/doc/user/group/epics/img/epic_activity_sort_order_v13_2.png b/doc/user/group/epics/img/epic_activity_sort_order_v13_2.png Binary files differindex c7e1448fea9..ce85efe3e67 100644 --- a/doc/user/group/epics/img/epic_activity_sort_order_v13_2.png +++ b/doc/user/group/epics/img/epic_activity_sort_order_v13_2.png diff --git a/doc/user/group/epics/img/issue_list_v13_1.png b/doc/user/group/epics/img/issue_list_v13_1.png Binary files differindex f08f774e986..ed0b4842bfe 100644 --- a/doc/user/group/epics/img/issue_list_v13_1.png +++ b/doc/user/group/epics/img/issue_list_v13_1.png diff --git a/doc/user/group/epics/img/new_epic_form_v13.2.png b/doc/user/group/epics/img/new_epic_form_v13.2.png Binary files differindex 3d24763d105..ac1450ae111 100644 --- a/doc/user/group/epics/img/new_epic_form_v13.2.png +++ b/doc/user/group/epics/img/new_epic_form_v13.2.png diff --git a/doc/user/group/epics/img/new_epic_from_groups_v13.2.png b/doc/user/group/epics/img/new_epic_from_groups_v13.2.png Binary files differindex 85bc4255595..bb75605af60 100644 --- a/doc/user/group/epics/img/new_epic_from_groups_v13.2.png +++ b/doc/user/group/epics/img/new_epic_from_groups_v13.2.png diff --git a/doc/user/group/epics/index.md b/doc/user/group/epics/index.md index a2b04e2d7fe..04b57d13828 100644 --- a/doc/user/group/epics/index.md +++ b/doc/user/group/epics/index.md @@ -20,6 +20,10 @@ An epic's page contains the following tabs: shown in a tree view. - Click the chevron (**>**) next to a parent epic to reveal the child epics and issues. - Hover over the total counts to see a breakdown of open and closed items. + + NOTE: **Note:** + The number provided here includes all epics associated with this project. The number includes epics for which users may not currently have permission. + - **Roadmap**: a roadmap view of child epics which have start and due dates.  @@ -65,7 +69,8 @@ to add an issue to an epic, reorder issues, move issues between epics, or promot ## Issue health status in Epic tree **(ULTIMATE)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/199184) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.10. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/199184) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.10. +> - The health status of a closed issue [will be hidden](https://gitlab.com/gitlab-org/gitlab/-/issues/220867) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.3 or later. You can report on and quickly respond to the health of individual issues and epics by setting a red, amber, or green [health status on an issue](../../project/issues/index.md#health-status-ultimate), @@ -175,7 +180,7 @@ Once you write your comment, you can either: ### Activity sort order -> [Introduced](https://https://gitlab.com/gitlab-org/gitlab/-/issues/214364) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.2. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/214364) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.2. You can reverse the default order and interact with the activity feed sorted by most recent items at the top. Your preference is saved via local storage and automatically applied to every issue diff --git a/doc/user/group/epics/manage_epics.md b/doc/user/group/epics/manage_epics.md index 4f9bb0e24fb..aaa5d3a3034 100644 --- a/doc/user/group/epics/manage_epics.md +++ b/doc/user/group/epics/manage_epics.md @@ -151,9 +151,18 @@ The sort option and order is saved and used wherever you browse epics, including > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/213068) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.0 behind a feature flag, disabled by default. > - [Became enabled by default](https://gitlab.com/gitlab-org/gitlab/-/issues/224513) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.2. +> - You can [use the Confidentiality option in the epic sidebar](https://gitlab.com/gitlab-org/gitlab/-/issues/197340) in GitLab [Premium](https://about.gitlab.com/pricing/) 13.3 and later. -When you're creating an epic, you can make it confidential by selecting the **Make this epic -confidential** checkbox. +If you're working on items that contain private information, you can make an epic confidential. + +NOTE: **Note:** +A confidential epic can only contain confidential issues and confidential child epics. + +To make an epic confidential: + +- **When creating an epic:** select the checkbox **Make this epic confidential**. +- **In an existing epic:** in the epic's sidebar, select **Edit** next to **Confidentiality** then + select **Turn on**. ### Disable confidential epics **(PREMIUM ONLY)** @@ -271,6 +280,16 @@ The following issue metadata will be copied to the epic: - Upvotes/downvotes. - Participants. - Group labels that the issue already has. +- Parent epic. **(ULTIMATE)** + +### Use an epic template for repeating issues + +You can create a spreadsheet template to manage a pattern of consistently repeating issues. + +<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> +For an introduction to epic templates, see [GitLab Epics and Epic Template Tip](https://www.youtube.com/watch?v=D74xKFNw8vg). + +For more on epic templates, see [Epic Templates - Repeatable sets of issues](https://about.gitlab.com/handbook/marketing/product-marketing/getting-started/104/). ## Manage multi-level child epics **(ULTIMATE)** diff --git a/doc/user/group/img/add_new_members.png b/doc/user/group/img/add_new_members.png Binary files differindex 6d43e309e84..8bd9e2374bc 100644 --- a/doc/user/group/img/add_new_members.png +++ b/doc/user/group/img/add_new_members.png diff --git a/doc/user/group/img/ldap_sync_cn_v13_1.png b/doc/user/group/img/ldap_sync_cn_v13_1.png Binary files differindex 1b0a20b7e64..279b3192328 100644 --- a/doc/user/group/img/ldap_sync_cn_v13_1.png +++ b/doc/user/group/img/ldap_sync_cn_v13_1.png diff --git a/doc/user/group/img/ldap_sync_filter_v13_1.png b/doc/user/group/img/ldap_sync_filter_v13_1.png Binary files differindex 3fc1f28becd..655bed58d46 100644 --- a/doc/user/group/img/ldap_sync_filter_v13_1.png +++ b/doc/user/group/img/ldap_sync_filter_v13_1.png diff --git a/doc/user/group/img/manual_permissions_v13_1.png b/doc/user/group/img/manual_permissions_v13_1.png Binary files differindex 504e6ab3a42..0ada9a4839c 100644 --- a/doc/user/group/img/manual_permissions_v13_1.png +++ b/doc/user/group/img/manual_permissions_v13_1.png diff --git a/doc/user/group/index.md b/doc/user/group/index.md index 5ba0680127c..22ad311ab4f 100644 --- a/doc/user/group/index.md +++ b/doc/user/group/index.md @@ -219,7 +219,7 @@ By default, every group inherits the branch protection set at the global level. To change this setting for a specific group: -1. Go to the group's **{settings}** **Settings > General** page. +1. Go to the group's **Settings > General** page. 1. Expand the **Permissions, LFS, 2FA** section. 1. Select the desired option in the **Default branch protection** dropdown list. 1. Click **Save changes**. @@ -278,7 +278,7 @@ The group details view also shows the number of the following items created in t - Issues. - Members. -These Group Activity Analytics can be enabled with the `group_activity_analytics` [feature flag](../../development/feature_flags/development.md#enabling-a-feature-flag-in-development). +These Group Activity Analytics can be enabled with the `group_activity_analytics` [feature flag](../../development/feature_flags/development.md#enabling-a-feature-flag-locally-in-development).  @@ -334,7 +334,7 @@ To share a given group, for example, 'Frontend' with another group, for example, All the members of the 'Engineering' group will have been added to 'Frontend'. -## Manage group memberships via LDAP +## Manage group memberships via LDAP **(STARTER ONLY)** Group syncing allows LDAP groups to be mapped to GitLab groups. This provides more control over per-group user management. To configure group syncing edit the `group_base` **DN** (`'OU=Global Groups,OU=GitLab INT,DC=GitLab,DC=org'`). This **OU** contains all groups that will be associated with GitLab groups. @@ -426,6 +426,7 @@ When transferring groups, note: - You must update your local repositories to point to the new location. - If the immediate parent group's visibility is lower than the group's current visibility, visibility levels for subgroups and projects will change to match the new parent group's visibility. - Only explicit group membership is transferred, not inherited membership. If the group's owners have only inherited membership, this leaves the group without an owner. In this case, the user transferring the group becomes the group's owner. +- Transfers will fail if [packages](../packages/index.md) exist in any of the projects within the group, or in any of its subgroups. ## Group settings @@ -442,7 +443,7 @@ access further configurations for your group. #### Changing a group's path -Changing a group's path can have unintended side effects. Read +Changing a group's path (group URL) can have unintended side effects. Read [how redirects will behave](../project/index.md#redirects-when-changing-repository-paths) before proceeding. @@ -450,12 +451,12 @@ If you are vacating the path so it can be claimed by another group or user, you may need to rename the group too, since both names and paths must be unique. -To change your group path: +To change your group path (group URL): 1. Navigate to your group's **Settings > General** page. 1. Expand the **Path, transfer, remove** section. -1. Enter a new name under **Change group path**. -1. Click **Change group path**. +1. Enter a new name under **Change group URL**. +1. Click **Change group URL**. CAUTION: **Caution:** It is currently not possible to rename a namespace if it contains a @@ -471,7 +472,7 @@ username, you can create a new group and transfer projects to it. To remove a group and its contents: -1. Navigate to your group's **{settings}** **Settings > General** page. +1. Navigate to your group's **Settings > General** page. 1. Expand the **Path, transfer, remove** section. 1. In the Remove group section, click the **Remove group** button. 1. Confirm the action when asked to. @@ -479,7 +480,7 @@ To remove a group and its contents: This action either: - Removes the group, and also queues a background job to delete all projects in that group. -- Since [GitLab 12.8](https://gitlab.com/gitlab-org/gitlab/-/issues/33257), on [Premium or Silver](https://about.gitlab.com/pricing/premium/) or higher tiers, marks a group for deletion. The deletion will happen 7 days later by default, but this can be changed in the [instance settings](../admin_area/settings/visibility_and_access_controls.md#default-deletion-adjourned-period-premium-only). +- Since [GitLab 12.8](https://gitlab.com/gitlab-org/gitlab/-/issues/33257), on [Premium or Silver](https://about.gitlab.com/pricing/premium/) or higher tiers, marks a group for deletion. The deletion will happen 7 days later by default, but this can be changed in the [instance settings](../admin_area/settings/visibility_and_access_controls.md#default-deletion-delay-premium-only). ### Restore a group **(PREMIUM)** @@ -487,7 +488,7 @@ This action either: To restore a group that is marked for deletion: -1. Navigate to your group's **{settings}** **Settings > General** page. +1. Navigate to your group's **Settings > General** page. 1. Expand the **Path, transfer, remove** section. 1. In the Restore group section, click the **Restore group** button. @@ -659,7 +660,7 @@ Optionally, on [Premium or Silver](https://about.gitlab.com/pricing/) or higher you can configure the projects within a group to be deleted after a delayed interval. During this interval period, the projects will be in a read-only state and can be restored, if required. -The interval period defaults to 7 days, and can be modified by an admin in the [instance settings](../admin_area/settings/visibility_and_access_controls.md#default-deletion-adjourned-period-premium-only). +The interval period defaults to 7 days, and can be modified by an admin in the [instance settings](../admin_area/settings/visibility_and_access_controls.md#default-deletion-delay-premium-only). To enable delayed deletion of projects: @@ -667,6 +668,23 @@ To enable delayed deletion of projects: 1. Expand the **Permissions, LFS, 2FA** section, and check **Enable delayed project removal**. 1. Click **Save changes**. +#### Prevent project forking outside group **(PREMIUM)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/216987) in GitLab 13.3. + +By default, projects within a group can be forked. +Optionally, on [Premium or Silver](https://about.gitlab.com/pricing/) or higher tiers, +you can prevent the projects within a group from being forked outside of the current top-level group. + +Previously this setting was available only for groups enforcing group managed account. This setting will be +removed from SAML setting page and migrated to group setting, but in the interim period of changes both of those settings will be taken into consideration, if even one is set to `true` then it will be assumed group does not allow forking projects outside. + +To enable prevent project forking: + +1. Navigate to the top-level group's **Settings > General** page. +1. Expand the **Permissions, LFS, 2FA** section, and check **Prevent project forking outside current group**. +1. Click **Save changes**. + ### Advanced settings - **Projects**: View all projects within that group, add members to each project, @@ -725,9 +743,9 @@ With [GitLab Contribution Analytics](contribution_analytics/index.md), you have an overview of the contributions (pushes, merge requests, and issues) performed by your group members. -## Issues analytics **(PREMIUM)** +## Issue analytics **(PREMIUM)** -With [GitLab Issues Analytics](issues_analytics/index.md), you can see a bar chart of the number of issues created each month in your groups. +With [GitLab Issue Analytics](issues_analytics/index.md), you can see a bar chart of the number of issues created each month in your groups. ## Dependency Proxy **(PREMIUM)** diff --git a/doc/user/group/issues_analytics/img/issues_table_v13_1.png b/doc/user/group/issues_analytics/img/issues_table_v13_1.png Binary files differindex 0f94f1ba2c5..3e8a729a884 100644 --- a/doc/user/group/issues_analytics/img/issues_table_v13_1.png +++ b/doc/user/group/issues_analytics/img/issues_table_v13_1.png diff --git a/doc/user/group/issues_analytics/index.md b/doc/user/group/issues_analytics/index.md index 01cee3d09b8..69512f90fca 100644 --- a/doc/user/group/issues_analytics/index.md +++ b/doc/user/group/issues_analytics/index.md @@ -5,16 +5,16 @@ group: Analytics info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# Issues Analytics **(PREMIUM)** +# Issue Analytics **(PREMIUM)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7478) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.5. > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/196561) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.9 at the project level. -Issues Analytics is a bar graph which illustrates the number of issues created each month. +Issue Analytics is a bar graph which illustrates the number of issues created each month. The default timespan is 13 months, which includes the current month, and the 12 months prior. -To access the chart, navigate to your group or project sidebar and select **{chart}** **Analytics > Issues Analytics**. +To access the chart, navigate to your group or project sidebar and select **{chart}** **Analytics > Issue Analytics**. Hover over each bar to see the total number of issues. diff --git a/doc/user/group/saml_sso/group_managed_accounts.md b/doc/user/group/saml_sso/group_managed_accounts.md index e317573d89d..126970ebbb6 100644 --- a/doc/user/group/saml_sso/group_managed_accounts.md +++ b/doc/user/group/saml_sso/group_managed_accounts.md @@ -7,13 +7,16 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Group Managed Accounts **(PREMIUM)** -CAUTION: **Warning:** -This is a [Closed Beta](https://about.gitlab.com/handbook/product/#closed-beta) feature. +CAUTION: **Caution:** +This [Closed Beta](https://about.gitlab.com/handbook/product/#closed-beta) feature is being re-evaluated in favor of a different +[identity model](https://gitlab.com/gitlab-org/gitlab/-/issues/218631) that does not require separate accounts. +We recommend that group administrators who haven't yet implemented this feature wait for +the new solution. > - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/709) in GitLab 12.1. > - It's deployed behind a feature flag, disabled by default. -When [SSO for Groups](index.md) is being enforced, groups can enable an additional level of protection by enforcing the creation of dedicated user accounts to access the group. +When [SSO for Groups](index.md) is enforced, groups can enable an additional level of protection by enforcing the creation of dedicated user accounts to access the group. With group-managed accounts enabled, users are required to create a new, dedicated user linked to the group. The notification email address associated with the user is locked to the email address received from the configured identity provider. @@ -21,8 +24,8 @@ Without group-managed accounts, users can link their SAML identity with any exis When this option is enabled: -- All users in the group will be required to log in via the SSO URL associated with the group. -- After the group-managed account has been created, group activity will require the use of this user account. +- All users in the group are required to log in via the SSO URL associated with the group. +- After the group-managed account has been created, group activity requires the use of this user account. - Users can't share a project in the group outside the top-level group (also applies to forked projects). Upon successful authentication, GitLab prompts the user with options, based on the email address received from the configured identity provider: @@ -30,10 +33,10 @@ Upon successful authentication, GitLab prompts the user with options, based on t - To create a unique account with the newly received email address. - If the received email address matches one of the user's verified GitLab email addresses, the option to convert the existing account to a group-managed account. ([Introduced in GitLab 12.9](https://gitlab.com/gitlab-org/gitlab/-/issues/13481).) -Since use of the group-managed account requires the use of SSO, users of group-managed accounts will lose access to these accounts when they are no longer able to authenticate with the connected identity provider. In the case of an offboarded employee who has been removed from your identity provider: +Since use of the group-managed account requires the use of SSO, users of group-managed accounts lose access to these accounts when they are no longer able to authenticate with the connected identity provider. In the case of an offboarded employee who has been removed from your identity provider: -- The user will be unable to access the group (their credentials will no longer work on the identity provider when prompted to SSO). -- Contributions in the group (e.g. issues, merge requests) will remain intact. +- The user is unable to access the group (their credentials no longer work on the identity provider when prompted to use SSO). +- Contributions in the group (for example, issues and merge requests) remains intact. ## Assertions @@ -49,7 +52,7 @@ assertions to be able to create a user. ## Feature flag **(PREMIUM ONLY)** -Currently the group-managed accounts feature is behind a feature flag: `group_managed_accounts`. The flag is disabled by default. +The group-managed accounts feature is behind a feature flag: `group_managed_accounts`. The flag is disabled by default. To activate the feature, ask a GitLab administrator with Rails console access to run: ```ruby @@ -101,16 +104,16 @@ Since personal access tokens are the only token needed for programmatic access t ### Setting a limit -Only a GitLab administrator or an owner of a Group-managed account can set a limit. Leaving it empty means that the [instance level restrictions](../../admin_area/settings/account_and_limit_settings.md#limiting-lifetime-of-personal-access-tokens-ultimate-only) on the lifetime of personal access tokens will apply. +Only a GitLab administrator or an owner of a group-managed account can set a limit. When this field is left empty, the [instance-level restriction](../../admin_area/settings/account_and_limit_settings.md#limiting-lifetime-of-personal-access-tokens-ultimate-only) on the lifetime of personal access tokens apply. To set a limit on how long personal access tokens are valid for users in a group managed account: -1. Navigate to the **{settings}** **Settings > General** page in your group's sidebar. +1. Navigate to the **Settings > General** page in your group's sidebar. 1. Expand the **Permissions, LFS, 2FA** section. 1. Fill in the **Maximum allowable lifetime for personal access tokens (days)** field. 1. Click **Save changes**. -Once a lifetime for personal access tokens is set, GitLab will: +Once a lifetime for personal access tokens is set: -- Apply the lifetime for new personal access tokens, and require users managed by the group to set an expiration date that is no later than the allowed lifetime. +- GitLab applies the lifetime for new personal access tokens and requires users managed by the group to set an expiration date that's no later than the allowed lifetime. - After three hours, revoke old tokens with no expiration date or with a lifetime longer than the allowed lifetime. Three hours is given to allow administrators/group owner to change the allowed lifetime, or remove it, before revocation takes place. diff --git a/doc/user/group/saml_sso/img/group_saml_settings.png b/doc/user/group/saml_sso/img/group_saml_settings.png Binary files differdeleted file mode 100644 index 487be4faeba..00000000000 --- a/doc/user/group/saml_sso/img/group_saml_settings.png +++ /dev/null diff --git a/doc/user/group/saml_sso/img/group_saml_settings_v13_3.png b/doc/user/group/saml_sso/img/group_saml_settings_v13_3.png Binary files differnew file mode 100644 index 00000000000..f6450c7c1ba --- /dev/null +++ b/doc/user/group/saml_sso/img/group_saml_settings_v13_3.png diff --git a/doc/user/group/saml_sso/img/scim_token.png b/doc/user/group/saml_sso/img/scim_token.png Binary files differdeleted file mode 100644 index bf9347716e9..00000000000 --- a/doc/user/group/saml_sso/img/scim_token.png +++ /dev/null diff --git a/doc/user/group/saml_sso/img/scim_token_v13_3.png b/doc/user/group/saml_sso/img/scim_token_v13_3.png Binary files differnew file mode 100644 index 00000000000..e98f755718a --- /dev/null +++ b/doc/user/group/saml_sso/img/scim_token_v13_3.png diff --git a/doc/user/group/saml_sso/index.md b/doc/user/group/saml_sso/index.md index afd676cf897..f0d0fbff158 100644 --- a/doc/user/group/saml_sso/index.md +++ b/doc/user/group/saml_sso/index.md @@ -18,6 +18,8 @@ If you follow our guidance to automate user provisioning using [SCIM](scim_setup User synchronization of SAML SSO groups is supported through [SCIM](scim_setup.md). SCIM supports adding and removing users from the GitLab group. For example, if you remove a user from the SCIM app, SCIM removes that same user from the GitLab group. +SAML SSO is not supported at the subgroup level, + ## Configuring your Identity Provider 1. Navigate to the group and click **Settings > SAML SSO**. @@ -63,10 +65,11 @@ Once you've set up your identity provider to work with GitLab, you'll need to co 1. Navigate to the group's **Settings > SAML SSO**. 1. Find the SSO URL from your Identity Provider and enter it the **Identity provider single sign-on URL** field. 1. Find and enter the fingerprint for the SAML token signing certificate in the **Certificate** field. +1. Select the access level to be applied to newly added users in the **Default membership role** field. The default access level is 'Guest'. 1. Click the **Enable SAML authentication for this group** toggle switch. 1. Click the **Save changes** button. - + NOTE: **Note:** Please note that the certificate [fingerprint algorithm](#additional-providers-and-setup-options) must be in SHA1. When configuring the identity provider, use a secure signature algorithm. @@ -79,6 +82,7 @@ Please note that the certificate [fingerprint algorithm](#additional-providers-a With this option enabled, users must go through your group's GitLab single sign-on URL. They may also be added via SCIM, if configured. Users cannot be added manually, and may only access project/group resources via the UI by signing in through the SSO URL. However, users will not be prompted to sign in through SSO on each visit. GitLab will check whether a user has authenticated through SSO, and will only prompt the user to sign in via SSO if the session has expired. +You can see more information about how long a session is valid in our [user profile documentation](../../profile/#why-do-i-keep-getting-signed-out). We intend to add a similar SSO requirement for [Git and API activity](https://gitlab.com/gitlab-org/gitlab/-/issues/9152). @@ -94,7 +98,7 @@ GitLab is unable to provide support for IdPs that are not listed here. | Provider | Documentation | |----------|---------------| | ADFS (Active Directory Federation Services) | [Create a Relying Party Trust](https://docs.microsoft.com/en-us/windows-server/identity/ad-fs/operations/create-a-relying-party-trust) | -| Azure | [Configuring single sign-on to applications](https://docs.microsoft.com/en-us/azure/active-directory/manage-apps/configure-single-sign-on-non-gallery-applications) | +| Azure | [Configuring single sign-on to applications](https://docs.microsoft.com/en-us/azure/active-directory/manage-apps/view-applications-portal) | | Okta | [Setting up a SAML application in Okta](https://developer.okta.com/docs/guides/build-sso-integration/saml2/overview/) | | OneLogin | [Use the OneLogin SAML Test Connector](https://onelogin.service-now.com/support?id=kb_article&sys_id=93f95543db109700d5505eea4b96198f) | @@ -160,7 +164,7 @@ For more information, see our [discussion on providers](#providers). Your identity provider may have relevant documentation. It may be generic SAML documentation, or specifically targeted for GitLab. Examples: -- [Auth0](https://auth0.com/docs/protocols/saml/saml-idp-generic) +- [Auth0](https://auth0.com/docs/protocols/saml-configuration-options/configure-auth0-as-saml-identity-provider) - [G Suite](https://support.google.com/a/answer/6087519?hl=en) - [JumpCloud](https://support.jumpcloud.com/support/s/article/single-sign-on-sso-with-gitlab-2019-08-21-10-36-47) - [PingOne by Ping Identity](https://docs.pingidentity.com/bundle/pingone/page/xsh1564020480660-1.html) @@ -216,7 +220,9 @@ On subsequent visits, you should be able to go [sign in to GitLab.com with SAML] ### Role -The first time you sign in, GitLab adds you to the top-level parent group with the Guest role. Existing members with appropriate privileges can promote that new user. +Starting from [GitLab 13.3](https://gitlab.com/gitlab-org/gitlab/-/issues/214523), group owners can set a 'Default membership role' other than 'Guest'. To do so, [configure the SAML SSO for the group](#configuring-gitlab). That role becomes the starting access level of all users added to the group. + +Existing members with appropriate privileges can promote or demote users, as needed. If a user is already a member of the group, linking the SAML identity does not change their role. @@ -268,7 +274,7 @@ Group SAML on a self-managed instance is limited when compared to the recommende [instance-wide SAML](../../../integration/saml.md). The recommended solution allows you to take advantage of: - [LDAP compatibility](../../../administration/auth/ldap/index.md). -- [LDAP Group Sync](../index.md#manage-group-memberships-via-ldap). +- [LDAP Group Sync](../index.md#manage-group-memberships-via-ldap-starter-only). - [Required groups](../../../integration/saml.md#required-groups-starter-only). - [Admin groups](../../../integration/saml.md#admin-groups-starter-only). - [Auditor groups](../../../integration/saml.md#auditor-groups-starter-only). @@ -297,6 +303,10 @@ Group SAML on a self-managed instance is limited when compared to the recommende - { name: 'group_saml' } ``` +## Passwords for users created via SAML SSO for Groups + +The [Generated passwords for users created through integrated authentication](../../../security/passwords_for_integrated_authentication_methods.md) guide provides an overview of how GitLab generates and sets passwords for users created via SAML SSO for Groups. + ## Troubleshooting This section contains possible solutions for problems you might encounter. diff --git a/doc/user/group/saml_sso/scim_setup.md b/doc/user/group/saml_sso/scim_setup.md index 13e9d623e2c..9a2bd2e8806 100644 --- a/doc/user/group/saml_sso/scim_setup.md +++ b/doc/user/group/saml_sso/scim_setup.md @@ -40,7 +40,7 @@ Once [Group Single Sign-On](index.md) has been configured, we can: 1. Click on the **Generate a SCIM token** button. 1. Save the token and URL so they can be used in the next step. - + ## Identity Provider configuration @@ -49,7 +49,7 @@ Once [Group Single Sign-On](index.md) has been configured, we can: ### Azure configuration steps -The SAML application that was created during [Single sign-on](index.md) setup for [Azure](https://docs.microsoft.com/en-us/azure/active-directory/manage-apps/configure-single-sign-on-non-gallery-applications) now needs to be set up for SCIM. +The SAML application that was created during [Single sign-on](index.md) setup for [Azure](https://docs.microsoft.com/en-us/azure/active-directory/manage-apps/view-applications-portal) now needs to be set up for SCIM. 1. Check the configuration for your GitLab SAML app and ensure that **Name identifier value** (NameID) points to `user.objectid` or another unique identifier. This will match the `extern_uid` used on GitLab. @@ -217,6 +217,10 @@ As a workaround, try an alternate mapping: #### How do I diagnose why a user is unable to sign in +Ensure that the user has been added to the SCIM app. + +If you receive "User is not linked to a SAML account", then most likely the user already exists in GitLab. Have the user follow the [User access and linking setup](#user-access-and-linking-setup) instructions. + The **Identity** (`extern_uid`) value stored by GitLab is updated by SCIM whenever `id` or `externalId` changes. Users won't be able to sign in unless the GitLab Identity (`extern_uid`) value matches the `NameId` sent by SAML. This value is also used by SCIM to match users on the `id`, and is updated by SCIM whenever the `id` or `externalId` values change. diff --git a/doc/user/group/settings/import_export.md b/doc/user/group/settings/import_export.md index cca918d44f3..ae83c8da462 100644 --- a/doc/user/group/settings/import_export.md +++ b/doc/user/group/settings/import_export.md @@ -21,7 +21,7 @@ See also: To enable GitLab import/export: -1. Navigate to **{admin}** **Admin Area >** **{settings}** **Settings > Visibility and access controls**. +1. Navigate to **Admin Area > Settings > Visibility and access controls**. 1. Scroll to **Import sources** 1. Enable desired **Import sources** @@ -33,13 +33,13 @@ Note the following: - To preserve group-level relationships from imported projects, run the Group Import/Export first, to allow projects to be imported into the desired group structure. - Imported groups are given a `private` visibility level, unless imported into a parent group. -- If imported into a parent group, subgroups will inherit the same level of visibility unless otherwise restricted. +- If imported into a parent group, a subgroup inherits the same level of visibility unless otherwise restricted. - To preserve the member list and their respective permissions on imported groups, review the users in these groups. Make sure these users exist before importing the desired groups. ### Exported Contents -The following items will be exported: +The following items are exported: - Milestones - Labels @@ -49,7 +49,7 @@ The following items will be exported: - Epics - Events -The following items will NOT be exported: +The following items are **not** exported: - Projects - Runners token @@ -63,7 +63,7 @@ For more details on the specific data persisted in a group export, see the 1. Navigate to your group's homepage. -1. Click **{settings}** **Settings** in the sidebar. +1. Click **Settings** in the sidebar. 1. In the **Advanced** section, click the **Export Group** button. @@ -83,7 +83,7 @@ As an administrator, you can modify the maximum import file size. To do so, use You can export groups from the [Community Edition to the Enterprise Edition](https://about.gitlab.com/install/ce-or-ee/) and vice versa. -If you're exporting a group from the Enterprise Edition to the Community Edition, you may lose data that is retained only in the Enterprise Edition. For more information, see [downgrading from EE to CE](../../../README.md). +The Enterprise Edition retains some group data that isn't part of the Community Edition. If you're exporting a group from the Enterprise Edition to the Community Edition, you may lose this data. For more information, see [downgrading from EE to CE](../../../README.md). ## Importing the group @@ -104,7 +104,7 @@ on an existing group's page. 1. Select the file that you exported in the [exporting a group](#exporting-a-group) section. -1. Click **Import group** to begin importing. Your newly imported group page will appear shortly. +1. Click **Import group** to begin importing. Your newly imported group page appears after the operation completes. ## Version history diff --git a/doc/user/group/subgroups/index.md b/doc/user/group/subgroups/index.md index a370c9cf136..235855b6e3a 100644 --- a/doc/user/group/subgroups/index.md +++ b/doc/user/group/subgroups/index.md @@ -7,13 +7,12 @@ type: reference, howto, concepts > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/2772) in GitLab 9.0. GitLab supports up to 20 levels of subgroups, also known as nested groups or hierarchical groups. -levels of groups. By using subgroups you can do the following: - **Separate internal / external organizations.** Since every group - can have its own visibility level, you are able to host groups for different - purposes under the same umbrella. + can have its own visibility level ([public, internal, or private](../../../development/permissions.md#general-permissions)), + you're able to host groups for different purposes under the same umbrella. - **Organize large projects.** For large projects, subgroups makes it potentially easier to separate permissions on parts of the source code. - **Make it easier to manage people and control visibility.** Give people diff --git a/doc/user/img/completed_tasks_v13_3.png b/doc/user/img/completed_tasks_v13_3.png Binary files differnew file mode 100644 index 00000000000..31e051852cb --- /dev/null +++ b/doc/user/img/completed_tasks_v13_3.png diff --git a/doc/user/img/inline_diff_01_v13_3.png b/doc/user/img/inline_diff_01_v13_3.png Binary files differnew file mode 100644 index 00000000000..b14317274d1 --- /dev/null +++ b/doc/user/img/inline_diff_01_v13_3.png diff --git a/doc/user/img/inline_diff_02_v13_3.png b/doc/user/img/inline_diff_02_v13_3.png Binary files differnew file mode 100644 index 00000000000..959f7014e18 --- /dev/null +++ b/doc/user/img/inline_diff_02_v13_3.png diff --git a/doc/user/incident_management/img/incident_management_settings.png b/doc/user/incident_management/img/incident_management_settings.png Binary files differdeleted file mode 100644 index 761a782ce61..00000000000 --- a/doc/user/incident_management/img/incident_management_settings.png +++ /dev/null diff --git a/doc/user/incident_management/img/pagerduty_incidents_integration_13_2.png b/doc/user/incident_management/img/pagerduty_incidents_integration_13_2.png Binary files differdeleted file mode 100644 index 9277b013676..00000000000 --- a/doc/user/incident_management/img/pagerduty_incidents_integration_13_2.png +++ /dev/null diff --git a/doc/user/incident_management/index.md b/doc/user/incident_management/index.md index 996f423e770..3d883a6b119 100644 --- a/doc/user/incident_management/index.md +++ b/doc/user/incident_management/index.md @@ -1,136 +1,5 @@ --- -stage: Monitor -group: Health -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +redirect_to: '../../operations/incident_management/index.md' --- -# Incident Management - -GitLab offers solutions for handling incidents in your applications and services, -such as setting up Prometheus alerts, displaying metrics, and sending notifications. - -## Configure incidents **(ULTIMATE)** - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/4925) in GitLab Ultimate 11.11. - -You can enable or disable Incident Management features in the GitLab user interface -to create issues when alerts are triggered: - -1. Navigate to **{settings}** **Settings > Operations > Incidents** and expand - **Incidents**: - -  - -1. For GitLab versions 11.11 and greater, you can select the **Create an issue** - checkbox to create an issue based on your own - [issue templates](../project/description_templates.md#creating-issue-templates). - For more information, see - [Trigger actions from alerts](../../operations/metrics/alerts.md#trigger-actions-from-alerts-ultimate) **(ULTIMATE)**. -1. To create issues from alerts, select the template in the **Issue Template** - select box. -1. To send [separate email notifications](#notify-developers-of-alerts) to users - with [Developer permissions](../permissions.md), select - **Send a separate email notification to Developers**. -1. Click **Save changes**. - -Appropriately configured alerts include an -[embedded chart](../../operations/metrics/embed.md#embedding-metrics-based-on-alerts-in-incident-issues) -for the query corresponding to the alert. You can also configure GitLab to -[close issues](../../operations/metrics/alerts.md#trigger-actions-from-alerts-ultimate) -when you receive notification that the alert is resolved. - -### Notify developers of alerts - -GitLab can react to the alerts triggered from your applications and services -by creating issues and alerting developers through email. By default, GitLab -sends these emails to [owners and maintainers](../permissions.md) of the project. -These emails contain details of the alert, and a link for more information. - -To send separate email notifications to users with -[Developer permissions](../permissions.md), see [Configure incidents](#configure-incidents-ultimate). - -## Configure PagerDuty integration - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/119018) in GitLab 13.2. - -You can set up a webhook with PagerDuty to automatically create a GitLab issue -for each PagerDuty incident. This configuration requires you to make changes -in both PagerDuty and GitLab: - -1. Sign in as a user with Maintainer [permissions](../permissions.md). -1. Navigate to **{settings}** **Settings > Operations > Incidents** and expand **Incidents**. -1. Select the **PagerDuty integration** tab: - -  - -1. Activate the integration, and save the changes in GitLab. -1. Copy the value of **Webhook URL**, as you'll need it in a later step. -1. Follow the steps described in the - [PagerDuty documentation](https://support.pagerduty.com/docs/webhooks) - to add the webhook URL to a PagerDuty webhook integration. - -To confirm the integration is successful, trigger a test incident from PagerDuty to -confirm that a GitLab issue is created from the incident. - -## Configure Prometheus alerts - -You can set up Prometheus alerts in: - -- [GitLab-managed Prometheus](../../operations/metrics/alerts.md) installations. -- [Self-managed Prometheus](../../operations/metrics/alerts.md#external-prometheus-instances) installations. - -Prometheus alerts are created by the special Alert Bot user. You can't remove this -user, but it does not count toward your license limit. - -## Configure external generic alerts - -GitLab can accept alerts from any source through a generic webhook receiver. When -[configuring the generic alerts integration](../project/integrations/generic_alerts.md), -GitLab creates a unique endpoint which receives a JSON-formatted, customizable payload. - -## Embed metrics in incidents and issues - -You can embed metrics anywhere [GitLab Markdown](../markdown.md) is used, such as descriptions, -comments on issues, and merge requests. Embedding metrics helps you share them -when discussing incidents or performance issues. You can output the dashboard directly -into any issue, merge request, epic, or any other Markdown text field in GitLab -by [copying and pasting the link to the metrics dashboard](../../operations/metrics/embed.md#embedding-gitlab-managed-kubernetes-metrics). - -You can embed both -[GitLab-hosted metrics](../../operations/metrics/embed.md) and -[Grafana metrics](../../operations/metrics/embed_grafana.md) -in incidents and issue templates. - -### Context menu - -You can view more details about an embedded metrics panel from the context menu. -To access the context menu, click the **{ellipsis_v}** **More actions** dropdown box -above the upper right corner of the panel. The options are: - -- [View logs](#view-logs-from-metrics-panel). -- **Download CSV** - Data from embedded charts can be - [downloaded as CSV](../../operations/metrics/dashboards/index.md#downloading-data-as-csv). - -#### View logs from metrics panel - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/201846) in GitLab Ultimate 12.8. -> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/25455) to [GitLab Core](https://about.gitlab.com/pricing/) 12.9. - -Viewing logs from a metrics panel can be useful if you're triaging an application -incident and need to [explore logs](../../operations/metrics/dashboards/index.md#view-logs-ultimate) -from across your application. These logs help you understand what is affecting -your application's performance and resolve any problems. - -## Integrate incidents with Slack - -Slack slash commands allow you to control GitLab and view GitLab content without leaving Slack. - -Learn how to [set up Slack slash commands](../project/integrations/slack_slash_commands.md) -and how to [use the available slash commands](../../integration/slash_commands.md). - -## Integrate issues with Zoom - -GitLab enables you to [associate a Zoom meeting with an issue](../project/issues/associate_zoom_meeting.md) -for synchronous communication during incident management. After starting a Zoom -call for an incident, you can associate the conference call with an issue. Your -team members can join the Zoom call without requesting a link. +This document was moved to [../../operations/incident_management/index.md](../../operations/incident_management/index.md). diff --git a/doc/user/index.md b/doc/user/index.md index f50b712e2c3..1ab3541575c 100644 --- a/doc/user/index.md +++ b/doc/user/index.md @@ -1,4 +1,5 @@ --- +type: reference, index description: 'Read through the GitLab User documentation to learn how to use, configure, and customize GitLab and GitLab.com to your own needs.' --- diff --git a/doc/user/infrastructure/index.md b/doc/user/infrastructure/index.md index e24e669d994..2a49ca18aaf 100644 --- a/doc/user/infrastructure/index.md +++ b/doc/user/infrastructure/index.md @@ -50,7 +50,7 @@ If you plan to only run `terraform plan` and `terraform apply` commands from you local machine, this is a simple way to get started: 1. Create your project on your GitLab instance. -1. Navigate to **{settings}** **Settings > General** and note your **Project name** +1. Navigate to **Settings > General** and note your **Project name** and **Project ID**. 1. Define the Terraform backend in your Terraform project to be: @@ -65,16 +65,16 @@ local machine, this is a simple way to get started: the `api` scope. 1. On your local machine, run `terraform init`, passing in the following options, - replacing `<YOUR-PROJECT-NAME>`, `<YOUR-PROJECT-ID>`, `<YOUR-USERNAME>` and + replacing `<YOUR-STATE-NAME>`, `<YOUR-PROJECT-ID>`, `<YOUR-USERNAME>` and `<YOUR-ACCESS-TOKEN>` with the relevant values. This command initializes your Terraform state, and stores that state within your GitLab project. This example uses `gitlab.com`: ```shell terraform init \ - -backend-config="address=https://gitlab.com/api/v4/projects/<YOUR-PROJECT-ID>/terraform/state/<YOUR-PROJECT-NAME>" \ - -backend-config="lock_address=https://gitlab.com/api/v4/projects/<YOUR-PROJECT-ID>/terraform/state/<YOUR-PROJECT-NAME>/lock" \ - -backend-config="unlock_address=https://gitlab.com/api/v4/projects/<YOUR-PROJECT-ID>/terraform/state/<YOUR-PROJECT-NAME>/lock" \ + -backend-config="address=https://gitlab.com/api/v4/projects/<YOUR-PROJECT-ID>/terraform/state/<YOUR-STATE-NAME>" \ + -backend-config="lock_address=https://gitlab.com/api/v4/projects/<YOUR-PROJECT-ID>/terraform/state/<YOUR-STATE-NAME>/lock" \ + -backend-config="unlock_address=https://gitlab.com/api/v4/projects/<YOUR-PROJECT-ID>/terraform/state/<YOUR-STATE-NAME>/lock" \ -backend-config="username=<YOUR-USERNAME>" \ -backend-config="password=<YOUR-ACCESS-TOKEN>" \ -backend-config="lock_method=POST" \ @@ -82,7 +82,7 @@ local machine, this is a simple way to get started: -backend-config="retry_wait_min=5" ``` -Next, [configure the backend](#configure-the-backend). +You can now run `terraform plan` and `terraform apply` as you normally would. ## Get started using GitLab CI @@ -121,17 +121,18 @@ and the CI YAML file: commands must be executed, `TF_ADDRESS` is the URL to the state on the GitLab instance where this pipeline runs, and the final path segment in `TF_ADDRESS` is the name of the Terraform state. Projects may have multiple states, and - this name is arbitrary, so in this example we will set it to the name of the - project, and we will ensure that the `.terraform` directory is cached between - jobs in the pipeline using a cache key based on the state name: + this name is arbitrary, so in this example we set it to `example-production` + which corresponds with the directory we're using as our `TF_ROOT`, and we + ensure that the `.terraform` directory is cached between jobs in the pipeline + using a cache key based on the state name (`example-production`): ```yaml variables: - TF_ROOT: ${CI_PROJECT_DIR}/environments/cloudflare/production - TF_ADDRESS: ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/terraform/state/${CI_PROJECT_NAME} + TF_ROOT: ${CI_PROJECT_DIR}/environments/example/production + TF_ADDRESS: ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/terraform/state/example-production cache: - key: ${CI_PROJECT_NAME} + key: example-production paths: - ${TF_ROOT}/.terraform ``` @@ -189,10 +190,113 @@ and the CI YAML file: The output from the above `terraform` commands should be viewable in the job logs. +CAUTION: **Caution:** +Like any other job artifact, Terraform plan data is [viewable by anyone with Guest access](../permissions.md) to the repository. +Neither Terraform nor GitLab encrypts the plan file by default. If your Terraform plan +includes sensitive data such as passwords, access tokens, or certificates, GitLab strongly +recommends encrypting plan output or modifying the project visibility settings. + ## Example project See [this reference project](https://gitlab.com/nicholasklick/gitlab-terraform-aws) using GitLab and Terraform to deploy a basic AWS EC2 within a custom VPC. +## Copy Terraform state between backends + +Terraform supports copying the state when the backend is changed or +reconfigured. This can be useful if you need to migrate from another backend to +GitLab managed Terraform state. It's also useful if you need to change the state +name as in the following example: + +```shell +PROJECT_ID="<gitlab-project-id>" +TF_USERNAME="<gitlab-username>" +TF_PASSWORD="<gitlab-personal-access-token>" +TF_ADDRESS="https://gitlab.com/api/v4/projects/${PROJECT_ID}/terraform/state/old-state-name" + +terraform init \ + -backend-config=address=${TF_ADDRESS} \ + -backend-config=lock_address=${TF_ADDRESS}/lock \ + -backend-config=unlock_address=${TF_ADDRESS}/lock \ + -backend-config=username=${TF_USERNAME} \ + -backend-config=password=${TF_PASSWORD} \ + -backend-config=lock_method=POST \ + -backend-config=unlock_method=DELETE \ + -backend-config=retry_wait_min=5 +``` + +```plaintext +Initializing the backend... + +Successfully configured the backend "http"! Terraform will automatically +use this backend unless the backend configuration changes. + +Initializing provider plugins... + +Terraform has been successfully initialized! + +You may now begin working with Terraform. Try running "terraform plan" to see +any changes that are required for your infrastructure. All Terraform commands +should now work. + +If you ever set or change modules or backend configuration for Terraform, +rerun this command to reinitialize your working directory. If you forget, other +commands will detect it and remind you to do so if necessary. +``` + +Now that `terraform init` has created a `.terraform/` directory that knows where +the old state is, you can tell it about the new location: + +```shell +TF_ADDRESS="https://gitlab.com/api/v4/projects/${PROJECT_ID}/terraform/state/new-state-name" + +terraform init \ + -backend-config=address=${TF_ADDRESS} \ + -backend-config=lock_address=${TF_ADDRESS}/lock \ + -backend-config=unlock_address=${TF_ADDRESS}/lock \ + -backend-config=username=${TF_USERNAME} \ + -backend-config=password=${TF_PASSWORD} \ + -backend-config=lock_method=POST \ + -backend-config=unlock_method=DELETE \ + -backend-config=retry_wait_min=5 +``` + +```plaintext +Initializing the backend... +Backend configuration changed! + +Terraform has detected that the configuration specified for the backend +has changed. Terraform will now check for existing state in the backends. + + +Acquiring state lock. This may take a few moments... +Do you want to copy existing state to the new backend? + Pre-existing state was found while migrating the previous "http" backend to the + newly configured "http" backend. No existing state was found in the newly + configured "http" backend. Do you want to copy this state to the new "http" + backend? Enter "yes" to copy and "no" to start with an empty state. + + Enter a value: yes + + +Successfully configured the backend "http"! Terraform will automatically +use this backend unless the backend configuration changes. + +Initializing provider plugins... + +Terraform has been successfully initialized! + +You may now begin working with Terraform. Try running "terraform plan" to see +any changes that are required for your infrastructure. All Terraform commands +should now work. + +If you ever set or change modules or backend configuration for Terraform, +rerun this command to reinitialize your working directory. If you forget, other +commands will detect it and remind you to do so if necessary. +``` + +If you type `yes`, it will copy your state from the old location to the new +location. You can then go back to running it from within GitLab CI. + ## Output Terraform Plan information into a merge request Using the [GitLab Terraform Report artifact](../../ci/pipelines/job_artifacts.md#artifactsreportsterraform), @@ -275,11 +379,11 @@ can configure this manually as follows: image: registry.gitlab.com/gitlab-org/terraform-images/stable:latest variables: - TF_ROOT: ${CI_PROJECT_DIR}/environments/cloudflare/production - TF_ADDRESS: ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/terraform/state/${CI_PROJECT_NAME} + TF_ROOT: ${CI_PROJECT_DIR}/environments/example/production + TF_ADDRESS: ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/terraform/state/example-production cache: - key: ${CI_PROJECT_NAME} + key: example-production paths: - ${TF_ROOT}/.terraform @@ -329,7 +433,7 @@ apply: ### Multiple Terraform Plan reports -Starting with 13.2, you can display mutiple reports on the Merge Request page. The reports will also display the `artifact: name:`. See example below for a suggested setup. +Starting with 13.2, you can display mutiple reports on the Merge Request page. The reports will also display the `artifacts: name:`. See example below for a suggested setup. ```yaml image: @@ -383,3 +487,47 @@ production_plan: artifacts: name: Production ``` + +## Using a GitLab managed Terraform state backend as a remote data source + +You can use a GitLab-managed Terraform state as a +[Terraform data source](https://www.terraform.io/docs/providers/terraform/d/remote_state.html). +To use your existing Terraform state backend as a data source, provide the following details +as [Terraform input variables](https://www.terraform.io/docs/configuration/variables.html): + +- **address**: The URL of the remote state backend you want to use as a data source. + For example, `https://gitlab.com/api/v4/projects/<TARGET-PROJECT-ID>/terraform/state/<TARGET-STATE-NAME>`. +- **username**: The username to authenticate with the data source. If you are using a [Personal Access Token](../profile/personal_access_tokens.md) for + authentication, this is your GitLab username. If you are using GitLab CI, this is `'gitlab-ci-token'`. +- **password**: The password to authenticate with the data source. If you are using a Personal Access Token for + authentication, this is the token value. If you are using GitLab CI, it is the contents of the `${CI_JOB_TOKEN}` CI variable. + +An example setup is shown below: + +1. Create a file named `example.auto.tfvars` with the following contents: + + ```plaintext + example_remote_state_address=https://gitlab.com/api/v4/projects/<TARGET-PROJECT-ID>/terraform/state/<TARGET-STATE-NAME> + example_username=<GitLab username> + example_access_token=<GitLab Personal Acceess Token> + ``` + +1. Define the data source by adding the following code block in a `.tf` file (such as `data.tf`): + + ```hcl + data "terraform_remote_state" "example" { + backend = "http" + + config = { + address = var.example_remote_state_address + username = var.example_username + password = var.example_access_token + } + } + ``` + +Outputs from the data source can now be referenced within your Terraform resources +using `data.terraform_remote_state.example.outputs.<OUTPUT-NAME>`. + +You need at least [developer access](../permissions.md) to the target project +to read the Terraform state. diff --git a/doc/user/instance_statistics/dev_ops_score.md b/doc/user/instance_statistics/dev_ops_score.md index 73b9532015d..35dcbd01916 100644 --- a/doc/user/instance_statistics/dev_ops_score.md +++ b/doc/user/instance_statistics/dev_ops_score.md @@ -7,7 +7,7 @@ NOTE: **Note:** Your GitLab instance's [usage ping](../admin_area/settings/usage_statistics.md#usage-ping-core-only) must be activated in order to use this feature. The DevOps Score gives you an overview of your entire instance's adoption of -[Concurrent DevOps](https://about.gitlab.com/concurrent-devops/) +[Concurrent DevOps](https://about.gitlab.com/topics/concurrent-devops/) from planning to monitoring. This displays the usage of these GitLab features over diff --git a/doc/user/markdown.md b/doc/user/markdown.md index b2b3f0f925c..12d65f75b37 100644 --- a/doc/user/markdown.md +++ b/doc/user/markdown.md @@ -1,7 +1,8 @@ --- -stage: Plan -group: Project Management -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference, howto --- # GitLab Markdown @@ -21,7 +22,7 @@ We encourage you to view this document as [rendered by GitLab itself](https://gi GitLab uses "GitLab Flavored Markdown" (GFM). It extends the [CommonMark specification](https://spec.commonmark.org/current/) (which is based on standard Markdown) in several ways to add additional useful functionality. -It was inspired by [GitHub Flavored Markdown](https://help.github.com/en/github/writing-on-github/basic-writing-and-formatting-syntax). +It was inspired by [GitHub Flavored Markdown](https://docs.github.com/en/github/writing-on-github/basic-writing-and-formatting-syntax). You can use GFM in the following areas: @@ -78,15 +79,15 @@ character of the top list item (`C` in this case): - milk NOTE: **Note:** -We will flag any significant differences between Redcarpet and CommonMark +We flag any significant differences between Redcarpet and CommonMark Markdown in this document. If you have a large volume of Markdown files, it can be tedious to determine -if they will display correctly or not. You can use the +if they display correctly or not. You can use the [diff_redcarpet_cmark](https://gitlab.com/digitalmoksha/diff_redcarpet_cmark) tool (not an officially supported product) to generate a list of files and the -differences between how RedCarpet and CommonMark render the files. It can give -an indication if anything needs to be changed - often nothing will need +differences between how RedCarpet and CommonMark render the files. It gives +an indication if anything needs to be changed - often nothing needs to change. ### GFM extends standard Markdown @@ -136,7 +137,7 @@ Supported formats (named colors are not supported): - RGB: `` `RGB[A](R, G, B[, A])` `` - HSL: `` `HSL[A](H, S, L[, A])` `` -Color written inside backticks will be followed by a color "chip": +Color written inside backticks is followed by a color "chip": ```markdown - `#F00` @@ -258,10 +259,11 @@ If you're new to this, don't be <img src="https://gitlab.com/gitlab-org/gitlab-f Consult the [Emoji Cheat Sheet](https://www.webfx.com/tools/emoji-cheat-sheet/) for a list of all supported emoji codes. <img src="https://gitlab.com/gitlab-org/gitlab-foss/raw/master/app/assets/images/emoji/thumbsup.png" width="20px" height="20px" style="display:inline;margin:0"> -> **Note:** The emoji example above uses hard-coded images for this documentation. The emoji, +NOTE: **Note:** +The emoji example above uses hard-coded images for this documentation. The emoji, when rendered within GitLab, may appear different depending on the OS and browser used. -Most emoji are natively supported on macOS, Windows, iOS, Android, and will fall back on image-based emoji where there is no support. +Most emoji are natively supported on macOS, Windows, iOS, Android, and fall back on image-based emoji where there is no support. NOTE: **Note:** On Linux, you can download [Noto Color Emoji](https://www.google.com/get/noto/help/emoji/) @@ -346,10 +348,7 @@ The wrapping tags can be either curly braces or square brackets: - [- deletion 4 -] ``` -- {+ addition 1 +} -- [+ addition 2 +] -- {- deletion 3 -} -- [- deletion 4 -] + --- @@ -363,7 +362,7 @@ However, the wrapping tags can't be mixed: ``` If your diff includes words in `` `code` `` font, make sure to escape each backtick `` ` `` with a -backslash `\`, otherwise the diff highlight won't render correctly: +backslash `\`, otherwise the diff highlight don't render correctly: ```markdown - {+ Just regular text +} @@ -371,9 +370,7 @@ backslash `\`, otherwise the diff highlight won't render correctly: - {+ Text with escaped \`backticks\` inside +} ``` -- {+ Just regular text +} -- {+ Text with `backticks` inside +} -- {+ Text with escaped \`backticks\` inside +} + ### Math @@ -381,8 +378,8 @@ backslash `\`, otherwise the diff highlight won't render correctly: It's possible to have math written with LaTeX syntax rendered using [KaTeX](https://github.com/KaTeX/KaTeX). -Math written between dollar signs `$` will be rendered inline with the text. Math written -inside a [code block](#code-spans-and-blocks) with the language declared as `math`, will be rendered +Math written between dollar signs `$` are rendered inline with the text. Math written +inside a [code block](#code-spans-and-blocks) with the language declared as `math`, are rendered on a separate line: ````markdown @@ -412,13 +409,13 @@ the [Asciidoctor user manual](https://asciidoctor.org/docs/user-manual/#activati ### Special GitLab references GFM recognizes special GitLab related references. For example, you can reference -an issue, a commit, a team member, or even the whole team within a project. GFM will turn +an issue, a commit, a team member, or even the whole team within a project. GFM turns that reference into a link so you can navigate between them. Additionally, GFM recognizes certain cross-project references and also has a shorthand version to reference other projects from the same namespace. -GFM will recognize the following: +GFM recognizes the following: | references | input | cross-project reference | shortcut within same namespace | | :------------------------------ | :------------------------- | :-------------------------------------- | :----------------------------- | @@ -444,9 +441,9 @@ GFM will recognize the following: In addition to this, links to some objects are also recognized and formatted. Some examples of these are: -- Comments on issues: `"https://gitlab.com/gitlab-org/gitlab/-/issues/1234#note_101075757"`, which will be rendered as `#1234 (note1)` -- The issues designs tab: `"https://gitlab.com/gitlab-org/gitlab/-/issues/1234/designs"`, which will be rendered as `#1234 (designs)`. -- Links to individual designs: `"https://gitlab.com/gitlab-org/gitlab/-/issues/1234/designs/layout.png"`, which will be rendered as `#1234[layout.png]`. +- Comments on issues: `"https://gitlab.com/gitlab-org/gitlab/-/issues/1234#note_101075757"`, which are rendered as `#1234 (note1)` +- The issues designs tab: `"https://gitlab.com/gitlab-org/gitlab/-/issues/1234/designs"`, which are rendered as `#1234 (designs)`. +- Links to individual designs: `"https://gitlab.com/gitlab-org/gitlab/-/issues/1234/designs/layout.png"`, which are rendered as `#1234[layout.png]`. ### Task lists @@ -473,22 +470,13 @@ unordered or ordered lists: 1. [x] Sub-task 2 ``` -- [x] Completed task -- [ ] Incomplete task - - [ ] Sub-task 1 - - [x] Sub-task 2 - - [ ] Sub-task 3 - -1. [x] Completed task -1. [ ] Incomplete task - 1. [ ] Sub-task 1 - 1. [x] Sub-task 2 + ### Table of contents You can add a table of contents to a Markdown file, wiki page, or issue/merge request description, by adding the tag `[[_TOC_]]` on its own line. -It will appear as an unordered list that links to the various headers. +It appears as an unordered list that links to the various headers. ```markdown This is an intro sentence to my Wiki page. @@ -512,7 +500,7 @@ The following examples show how links inside wikis behave. #### Wiki - direct page link -A link which just includes the slug for a page will point to that page, +A link which just includes the slug for a page points to that page, _at the base level of the wiki_. This snippet would link to a `documentation` page at the root of your wiki: @@ -583,13 +571,13 @@ This snippet links to `<wiki_root>/miscellaneous.md`: ### Embedding metrics in GitLab Flavored Markdown -Metric charts can be embedded within GitLab Flavored Markdown. See [Embedding Metrics within GitLab flavored Markdown](../operations/metrics/embed.md#embedding-metric-charts-within-gitlab-flavored-markdown) for more details. +Metric charts can be embedded within GitLab Flavored Markdown. See [Embedding Metrics within GitLab flavored Markdown](../operations/metrics/embed.md) for more details. ## Standard Markdown and extensions in GitLab All standard Markdown formatting should work as expected within GitLab. Some standard functionality is extended with additional features, without affecting the standard usage. -If a functionality is extended, the new option will be listed as a sub-section. +If a functionality is extended, the new option is listed as a sub-section. ### Blockquotes @@ -602,7 +590,7 @@ by starting the lines of the blockquote with `>`: Quote break. -> This is a very long line that will still be quoted properly when it wraps. Oh boy let's keep writing to make sure this is long enough to actually wrap for everyone. Oh, you can *put* **Markdown** into a blockquote. +> This is a very long line that is still quoted properly when it wraps. Oh boy let's keep writing to make sure this is long enough to actually wrap for everyone. Oh, you can *put* **Markdown** into a blockquote. ``` > Blockquotes are very handy to emulate reply text. @@ -610,7 +598,7 @@ Quote break. Quote break. -> This is a very long line that will still be quoted properly when it wraps. Oh boy let's keep writing to make sure this is long enough to actually wrap for everyone. Oh, you can *put* **Markdown** into a blockquote. +> This is a very long line that is still quoted properly when it wraps. Oh boy let's keep writing to make sure this is long enough to actually wrap for everyone. Oh, you can *put* **Markdown** into a blockquote. #### Multiline blockquote @@ -825,7 +813,7 @@ do*this*and*do*that*and*another thing ### Footnotes -Footnotes add a link to a note that will be rendered at the end of a Markdown file. +Footnotes add a link to a note that are rendered at the end of a Markdown file. To make a footnote, you need both a reference tag and a separate line (anywhere in the file) with the note content. @@ -1016,7 +1004,7 @@ Here's a sample audio clip: > To see the Markdown rendered within HTML in the second example, [view it in GitLab itself](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/user/markdown.md#inline-html). -You can also use raw HTML in your Markdown, and it will usually work pretty well. +You can also use raw HTML in your Markdown, and it usually works pretty well. See the documentation for HTML::Pipeline's [SanitizationFilter](https://github.com/jch/html-pipeline/blob/v2.12.3/lib/html/pipeline/sanitization_filter.rb#L42) class for the list of allowed HTML tags and attributes. In addition to the default @@ -1028,7 +1016,7 @@ class for the list of allowed HTML tags and attributes. In addition to the defau <dd>Is something people use sometimes.</dd> <dt>Markdown in HTML</dt> - <dd>Does *not* work **very** well. HTML <em>tags</em> will <b>work</b>, in most cases.</dd> + <dd>Does *not* work **very** well. HTML <em>tags</em> do <b>work</b>, in most cases.</dd> </dl> ``` @@ -1037,7 +1025,7 @@ class for the list of allowed HTML tags and attributes. In addition to the defau <dd>Is something people use sometimes.</dd> <dt>Markdown in HTML</dt> - <dd>Does *not* work **very** well. HTML <em>tags</em> will <b>work</b>, in most cases.</dd> + <dd>Does *not* work **very** well. HTML <em>tags</em> do <b>work</b>, in most cases.</dd> </dl> --- @@ -1048,12 +1036,12 @@ are separated into their own lines: ```html <dl> <dt>Markdown in HTML</dt> - <dd>Does *not* work **very** well. HTML tags will work, in most cases.</dd> + <dd>Does *not* work **very** well. HTML tags work, in most cases.</dd> <dt>Markdown in HTML</dt> <dd> - Does *not* work **very** well. HTML tags will work, in most cases. + Does *not* work **very** well. HTML tags work, in most cases. </dd> </dl> @@ -1061,17 +1049,17 @@ are separated into their own lines: <!-- Note: The example below uses HTML to force correct rendering on docs.gitlab.com, -Markdown will be fine in GitLab. +Markdown is fine in GitLab. --> <dl> <dt>Markdown in HTML</dt> - <dd>Does *not* work **very** well. HTML tags will work, in most cases.</dd> + <dd>Does *not* work **very** well. HTML tags work, in most cases.</dd> <dt>Markdown in HTML</dt> <dd> - Does <em>not</em> work <b>very</b> well. HTML tags will work, in most cases. + Does <em>not</em> work <b>very</b> well. HTML tags work, in most cases. </dd> </dl> @@ -1089,7 +1077,7 @@ tags. This is especially useful for collapsing long logs so they take up less sc <details> <summary>Click this to collapse/fold.</summary> -These details <em>will</em> remain <strong>hidden</strong> until expanded. +These details <em>remain</em> <strong>hidden</strong> until expanded. <pre><code>PASTE LOGS HERE</code></pre> @@ -1101,7 +1089,7 @@ These details <em>will</em> remain <strong>hidden</strong> until expanded. <details> <summary>Click this to collapse/fold.</summary> -These details <em>will</em> remain <strong>hidden</strong> until expanded. +These details <em>remain</em> <strong>hidden</strong> until expanded. <pre><code>PASTE LOGS HERE</code></pre> @@ -1124,7 +1112,7 @@ as shown in the example: <details> <summary>Click this to collapse/fold.</summary> -These details _will_ remain **hidden** until expanded. +These details _remain_ **hidden** until expanded. ``` PASTE LOGS HERE @@ -1135,13 +1123,13 @@ PASTE LOGS HERE <!-- The example below uses HTML to force correct rendering on docs.gitlab.com, Markdown -will work correctly in GitLab. +works correctly in GitLab. --> <details> <summary>Click this to collapse/fold.</summary> -These details <em>will</em> remain <b>hidden</b> until expanded. +These details <em>remain</em> <b>hidden</b> until expanded. <pre><code>PASTE LOGS HERE</code></pre> @@ -1149,16 +1137,16 @@ These details <em>will</em> remain <b>hidden</b> until expanded. ### Line breaks -A line break will be inserted (a new paragraph will start) if the previous text is +A line break is inserted (a new paragraph starts) if the previous text is ended with two newlines, like when you hit <kbd>Enter</kbd> twice in a row. If you only -use one newline (hit <kbd>Enter</kbd> once), the next sentence will be part of the +use one newline (hit <kbd>Enter</kbd> once), the next sentence remains part of the same paragraph. This is useful if you want to keep long lines from wrapping, and keep them editable: ```markdown Here's a line for us to start with. -This longer line is separated from the one above by two newlines, so it will be a *separate paragraph*. +This longer line is separated from the one above by two newlines, so it is a *separate paragraph*. This line is also a separate paragraph, but... These lines are only separated by single newlines, @@ -1168,7 +1156,7 @@ in the *same paragraph*. Here's a line for us to start with. -This longer line is separated from the one above by two newlines, so it will be a *separate paragraph*. +This longer line is separated from the one above by two newlines, so it is a *separate paragraph*. This line is also a separate paragraph, but... These lines are only separated by single newlines, @@ -1183,7 +1171,7 @@ A paragraph is one or more consecutive lines of text, separated by one or more blank lines (two newlines at the end of the first paragraph), as [explained above](#line-breaks). If you need more control over line breaks or soft returns, you can add a single line break -by ending a line with a backslash, or two or more spaces. Two newlines in a row will create a new +by ending a line with a backslash, or two or more spaces. Two newlines in a row create a new paragraph, with a blank line in between: ```markdown @@ -1255,11 +1243,11 @@ NOTE: **Note:** Relative links do not allow the referencing of project files in a wiki page, or a wiki page in a project file. The reason for this is that a wiki is always in a separate Git repository in GitLab. For example, `[I'm a reference-style link](style)` -will point the link to `wikis/style` only when the link is inside of a wiki Markdown file. +points the link to `wikis/style` only when the link is inside of a wiki Markdown file. #### URL auto-linking -GFM will auto-link almost any URL you put into your text: +GFM auto-links almost any URL you put into your text: ```markdown - https://www.google.com @@ -1283,9 +1271,9 @@ Ordered and unordered lists can be created. For an ordered list, add the number you want the list to start with, like `1.`, followed by a space, at the start of each line for ordered lists. -After the first number, it does not matter what number you use, ordered lists will be +After the first number, it does not matter what number you use, ordered lists are numbered automatically by vertical order, so repeating `1.` for all items in the -same list is common. If you start with a number other than `1.`, it will use that as the first +same list is common. If you start with a number other than `1.`, it uses that as the first number, and count up from there. Examples: @@ -1379,7 +1367,7 @@ Example: --- If the paragraph of the first item is not indented with the proper number of spaces, -the paragraph will appear outside the list, instead of properly indented under the list item. +the paragraph appears outside the list, instead of properly indented under the list item. Example: @@ -1429,18 +1417,18 @@ Example: | header 1 | header 2 | header 3 | | --- | ------ |---------:| | cell 1 | cell 2 | cell 3 | -| cell 4 | cell 5 is longer | cell 6 is much longer than the others, but that's ok. It will eventually wrap the text when the cell is too large for the display size. | +| cell 4 | cell 5 is longer | cell 6 is much longer than the others, but that's ok. It eventually wraps the text when the cell is too large for the display size. | | cell 7 | | cell <br> 9 | ``` | header 1 | header 2 | header 3 | | --- | ------ |---------:| | cell 1 | cell 2 | cell 3 | -| cell 4 | cell 5 is longer | cell 6 is much longer than the others, but that's okay. It will eventually wrap the text when the cell is too large for the display size. | +| cell 4 | cell 5 is longer | cell 6 is much longer than the others, but that's okay. It eventually wraps the text when the cell is too large for the display size. | | cell 7 | | cell <br> 9 | Additionally, you can choose the alignment of text within columns by adding colons (`:`) -to the sides of the "dash" lines in the second row. This will affect every cell in the column. +to the sides of the "dash" lines in the second row. This affects every cell in the column. NOTE: **Note:** [Within GitLab itself](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/user/markdown.md#tables), @@ -1463,8 +1451,8 @@ the headers are always left-aligned in Chrome and Firefox, and centered in Safar [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/27205) in GitLab 12.7. If you're working in spreadsheet software (for example, Microsoft Excel, Google -Sheets, or Apple Numbers), you can copy from a spreadsheet, and GitLab will -paste it as a Markdown table. For example, suppose you have the +Sheets, or Apple Numbers), you can copy from a spreadsheet, and GitLab +pastes it as a Markdown table. For example, suppose you have the following spreadsheet:  diff --git a/doc/user/packages/composer_repository/index.md b/doc/user/packages/composer_repository/index.md index 542efba1fae..9b1f23f6d59 100644 --- a/doc/user/packages/composer_repository/index.md +++ b/doc/user/packages/composer_repository/index.md @@ -4,9 +4,10 @@ group: Package info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# GitLab Composer Repository **(PREMIUM)** +# GitLab Composer Repository -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/15886) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.2. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/15886) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.2. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/221259) to GitLab Core in 13.3. With the GitLab Composer Repository, every project can have its own space to store [Composer](https://getcomposer.org/) packages. @@ -14,9 +15,9 @@ With the GitLab Composer Repository, every project can have its own space to sto NOTE: **Note:** This option is available only if your GitLab administrator has -[enabled support for the Package Registry](../../../administration/packages/index.md). **(PREMIUM ONLY)** +[enabled support for the Package Registry](../../../administration/packages/index.md). -After the Composer Repository is enabled, it will be available for all new projects +When the Composer Repository is enabled, it is available for all new projects by default. To enable it for existing projects, or if you want to disable it: 1. Navigate to your project's **Settings > General > Visibility, project features, permissions**. @@ -27,10 +28,10 @@ You should then be able to see the **Packages & Registries** section on the left ## Getting started -This section will cover creating a new example Composer package to publish. This is a +This section covers creating a new example Composer package to publish. This is a quickstart to test out the **GitLab Composer Registry**. -You will need a recent version of [Composer](https://getcomposer.org/). +To complete this section, you need a recent version of [Composer](https://getcomposer.org/). ### Creating a package project @@ -68,19 +69,21 @@ git init git add composer.json git commit -m 'Composer package test' git tag v1.0.0 -git add origin git@gitlab.com:<namespace>/<project-name>.git +git remote add origin git@gitlab.com:<namespace>/<project-name>.git +git push --set-upstream origin master git push origin v1.0.0 ``` ### Publishing the package Now that the basics of our project is completed, we can publish the package. -For accomplishing this you will need the following: +To publish the package, you need: - A personal access token. You can generate a [personal access token](../../../user/profile/personal_access_tokens.md) with the scope set to `api` for repository authentication. - Your project ID which can be found on the home page of your project. -To publish the package hosted on GitLab we'll need to make a `POST` to the GitLab package API using a tool like `curl`: +To publish the package hosted on GitLab, make a `POST` request to the GitLab package API. +A tool like `curl` can be used to make this request: ```shell curl --data tag=<tag> 'https://__token__:<personal-access-token>@gitlab.com/api/v4/projects/<project_id>/packages/composer' @@ -96,7 +99,7 @@ If the above command succeeds, you now should be able to see the package under t ### Installing a package -To install your package you will need: +To install your package, you need: - A personal access token. You can generate a [personal access token](../../../user/profile/personal_access_tokens.md) with the scope set to `api` for repository authentication. - Your group ID which can be found on the home page of your project's group. @@ -123,7 +126,7 @@ Where: - `<package_name>` is your package name as defined in your package's `composer.json` file. - `<version>` is your package version (`1.0.0` in this example). -You will also need to create a `auth.json` file with your GitLab credentials: +You also need to create a `auth.json` file with your GitLab credentials: ```json { diff --git a/doc/user/packages/conan_repository/index.md b/doc/user/packages/conan_repository/index.md index 41b420ce7f6..e8014ad2b84 100644 --- a/doc/user/packages/conan_repository/index.md +++ b/doc/user/packages/conan_repository/index.md @@ -4,9 +4,10 @@ group: Package info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# GitLab Conan Repository **(PREMIUM)** +# GitLab Conan Repository -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/8248) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.6. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/8248) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.6. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/221259) to GitLab Core in 13.3. With the GitLab Conan Repository, every project can have its own space to store Conan packages. @@ -17,9 +18,9 @@ project can have its own space to store Conan packages. NOTE: **Note:** This option is available only if your GitLab administrator has -[enabled support for the Conan Repository](../../../administration/packages/index.md).**(PREMIUM ONLY)** +[enabled support for the Conan Repository](../../../administration/packages/index.md). -After the Conan Repository is enabled, it will be available for all new projects +After the Conan Repository is enabled, it's available for all new projects by default. To enable it for existing projects, or if you want to disable it: 1. Navigate to your project's **Settings > General > Visibility, project features, permissions**. @@ -30,7 +31,7 @@ You should then be able to see the **Packages & Registries** section on the left ## Getting started -This section will cover installing Conan and building a package for your C/C++ project. This is a quickstart if you are new +This section covers installing Conan and building a package for your C/C++ project. This is a quickstart if you're new to Conan. If you already are using Conan and understand how to build your own packages, move on to the [next section](#adding-the-gitlab-package-registry-as-a-conan-remote). ### Installing Conan @@ -65,10 +66,10 @@ instructions at [cmake.org](https://cmake.org/install/) for your operating syste ### Creating a project -Understanding what is needed to create a valid and compilable C++ project is out of the scope of this guide, but if you are new to C++ and want to try out the GitLab +Understanding what is needed to create a valid and compilable C++ project is out of the scope of this guide, but if you're new to C++ and want to try out the GitLab package registry, Conan.io has a great [hello world starter project](https://github.com/conan-io/hello) that you can clone to get started. -Clone the repo and it can be used for the rest of the tutorial if you don't have your own C++ project. +Clone the repository and it can be used for the rest of the tutorial if you don't have your own C++ project. ### Building a package @@ -79,7 +80,7 @@ package name and version: conan new Hello/0.1 -t ``` -Next, you will create a package for that recipe by running `conan create` providing the Conan user and channel: +Next, create a package for that recipe by running `conan create` providing the Conan user and channel: ```shell conan create . my-org+my-group+my-project/beta @@ -90,11 +91,11 @@ Current [naming restrictions](#package-recipe-naming-convention) require you to The example above would create a package belonging to this project: `https://gitlab.com/my-org/my-group/my-project` with a channel of `beta`. -These two example commands will generate a final package with the recipe `Hello/0.1@my-org+my-group+my-project/beta`. +These two example commands generate a final package with the recipe `Hello/0.1@my-org+my-group+my-project/beta`. For more advanced details on creating and managing your packages, refer to the [Conan docs](https://docs.conan.io/en/latest/creating_packages.html). -You are now ready to upload your package to the GitLab registry. To get started, first you will need to set GitLab as a remote, then you will need to add a Conan user for that remote to authenticate your requests. +You are now ready to upload your package to the GitLab registry. To get started, first you need to set GitLab as a remote. Then you need to add a Conan user for that remote to authenticate your requests. ## Adding the GitLab Package Registry as a Conan remote @@ -114,7 +115,7 @@ conan search Hello* --all --remote=gitlab ## Authenticating to the GitLab Conan Repository -You will need a personal access token or deploy token. +You need a personal access token or deploy token. For repository authentication: @@ -123,7 +124,7 @@ For repository authentication: ### Adding a Conan user to the GitLab remote -Once you have a personal access token and have [set your Conan remote](#adding-the-gitlab-package-registry-as-a-conan-remote), you can associate the token with the remote so you do not have to explicitly add them to each Conan command you run: +Once you have a personal access token and have [set your Conan remote](#adding-the-gitlab-package-registry-as-a-conan-remote), you can associate the token with the remote so you don't have to explicitly add them to each Conan command you run: ```shell conan user <gitlab_username or deploy_token_username> -r gitlab -p <personal_access_token or deploy_token> @@ -132,10 +133,10 @@ conan user <gitlab_username or deploy_token_username> -r gitlab -p <personal_acc NOTE: **Note:** If you named your remote something other than `gitlab`, your remote name should be used in this command instead of `gitlab`. -From now on, when you run commands using `--remote=gitlab`, your username and password will automatically be included in the requests. +From now on, when you run commands using `--remote=gitlab`, your username and password is automatically included in the requests. NOTE: **Note:** -The personal access token is not stored locally at any moment. Conan uses JWT, so when you run this command, Conan will request an expirable token from GitLab using your token. The JWT does expire on a regular basis, so you will need to re-enter your personal access token when that happens. +The personal access token is not stored locally at any moment. Conan uses JSON Web Tokens (JWT), so when you run this command, Conan requests an expirable token from GitLab using your token. The JWT does expire on a regular basis, so you need to re-enter your personal access token when that happens. Alternatively, you could explicitly include your credentials in any given command. For example: @@ -156,7 +157,7 @@ NOTE: **Note:** The package recipe does include the version, so setting the default remote for `Hello/0.1@user/channel` will not work for `Hello/0.2@user/channel`. This functionality is best suited for when you want to consume or install packages from the GitLab registry without having to specify a remote. -The rest of the example commands in this documentation assume that you have added a Conan user with your credentials to the `gitlab` remote and will not include the explicit credentials or remote option, but be aware that any of the commands could be run without having added a user or default remote: +The rest of the example commands in this documentation assume that you've added a Conan user with your credentials to the `gitlab` remote and will not include the explicit credentials or remote option. With that said, be aware that any of the commands could be run without having added a user or default remote: ```shell `CONAN_LOGIN_USERNAME=<gitlab_username or deploy_token_username> CONAN_PASSWORD=<personal_access_token or deploy_token> <conan command> --remote=gitlab @@ -166,7 +167,7 @@ The rest of the example commands in this documentation assume that you have adde First you need to [create your Conan package locally](https://docs.conan.io/en/latest/creating_packages/getting_started.html). In order to work with the GitLab Package Registry, a specific [naming convention](#package-recipe-naming-convention) must be followed. -Ensure you have a project created on GitLab and that the personal access token you are using has the correct permissions for write access to the container registry by selecting the `api` [scope](../../../user/profile/personal_access_tokens.md#limiting-scopes-of-a-personal-access-token). +Ensure you have a project created on GitLab and that the personal access token you're using has the correct permissions for write access to the container registry by selecting the `api` [scope](../../../user/profile/personal_access_tokens.md#limiting-scopes-of-a-personal-access-token). You can upload your package to the GitLab Package Registry using the `conan upload` command: @@ -180,7 +181,7 @@ Standard Conan recipe convention looks like `package_name/version@user/channel`. **The recipe user must be the `+` separated project path**. The package name may be anything, but it is preferred that the project name be used unless -it is not possible due to a naming collision. For example: +it's not possible due to a naming collision. For example: | Project | Package | Supported | | ---------------------------------- | ----------------------------------------------- | --------- | @@ -190,7 +191,7 @@ it is not possible due to a naming collision. For example: | `gitlab-org/gitlab-ce` | `my-package/1.0.0@foo/stable` | No | NOTE: **Note:** -A future iteration will extend support to [project and group level](https://gitlab.com/gitlab-org/gitlab/-/issues/11679) remotes which will allow for more flexible naming conventions. +A future iteration will extend support to [project and group level](https://gitlab.com/gitlab-org/gitlab/-/issues/11679) remotes which allows for more flexible naming conventions. ## Installing a package @@ -222,7 +223,7 @@ conan install .. ``` NOTE: **Note:** -If you are trying to install the package you just created in this tutorial, not much will happen since that package +If you're trying to install the package you just created in this tutorial, not much will happen since that package already exists on your local machine. ## Removing a package @@ -235,11 +236,11 @@ There are two ways to remove a Conan package from the GitLab Package Registry. conan remove Hello/0.2@user/channel --remote=gitlab ``` - You need to explicitly include the remote in this command, otherwise the package will only be removed from your + You need to explicitly include the remote in this command, otherwise the package is only removed from your local system cache. NOTE: **Note:** - This command will remove all recipe and binary package files from the Package Registry. + This command removes all recipe and binary package files from the Package Registry. - **GitLab project interface**: in the packages view of your project page, you can delete packages by clicking the red trash icons. @@ -247,7 +248,7 @@ There are two ways to remove a Conan package from the GitLab Package Registry. The `conan search` command can be run searching by full or partial package name, or by exact recipe. -To search using a partial name, use the wildcard symbol `*`, which should be placed at the end of your search (e.g., `my-packa*`): +To search using a partial name, use the wildcard symbol `*`, which should be placed at the end of your search (for example, `my-packa*`): ```shell conan search Hello --all --remote=gitlab @@ -255,11 +256,11 @@ conan search He* --all --remote=gitlab conan search Hello/0.1@my-group+my-project/beta --all --remote=gitlab ``` -The scope of your search will include all projects you have permission to access, this includes your private projects as well as all public projects. +The scope of your search includes all projects you have permission to access, this includes your private projects as well as all public projects. ## Fetching Conan package information from the GitLab Package Registry -The `conan info` command will return information about a given package: +The `conan info` command returns information about a given package: ```shell conan info Hello/0.1@my-group+my-project/beta @@ -282,8 +283,8 @@ The GitLab Conan repository supports the following Conan CLI commands: To work with Conan commands within [GitLab CI/CD](./../../../ci/README.md), you can use `CI_JOB_TOKEN` in place of the personal access token in your commands. -It is easiest to provide the `CONAN_LOGIN_USERNAME` and `CONAN_PASSWORD` with each -Conan command in your `.gitlab-ci.yml` file: +It's easiest to provide the `CONAN_LOGIN_USERNAME` and `CONAN_PASSWORD` with each +Conan command in your `.gitlab-ci.yml` file. For example: ```yaml image: conanio/gcc7 @@ -292,8 +293,10 @@ create_package: stage: deploy script: - conan remote add gitlab https://gitlab.example.com/api/v4/packages/conan - - conan create . my-group+my-project/beta - - CONAN_LOGIN_USERNAME=ci_user CONAN_PASSWORD=${CI_JOB_TOKEN} conan upload Hello/0.1@root+ci-conan/beta1 --all --remote=gitlab + - conan new <package-name>/0.1 -t + - conan create . <group-name>+<project-name>/stable + - CONAN_LOGIN_USERNAME=ci_user CONAN_PASSWORD=${CI_JOB_TOKEN} conan upload <package-name>/0.1@<group-name>+<project-name>/stable --all --remote=gitlab + ``` You can find additional Conan images to use as the base of your CI file diff --git a/doc/user/packages/container_registry/index.md b/doc/user/packages/container_registry/index.md index 429d29b7677..f46ad99e573 100644 --- a/doc/user/packages/container_registry/index.md +++ b/doc/user/packages/container_registry/index.md @@ -49,7 +49,7 @@ project: 1. Expand the **Visibility, project features, permissions** section and enable the **Container Registry** feature on your project. For new projects this might be enabled by default. For existing projects - (prior GitLab 8.8), you will have to explicitly enable it. + (prior GitLab 8.8), enable it explicitly. 1. Press **Save changes** for the changes to take effect. You should now be able to see the **Packages & Registries > Container Registry** link in the sidebar. @@ -64,14 +64,14 @@ Navigate to your project's **{package}** **Packages & Registries > Container Reg  -This view will: +This view allows you to: - Show all the image repositories that belong to the project. -- Allow you to filter image repositories by their name. -- Allow you to [delete](#delete-images-from-within-gitlab) one or more image repository. -- Allow you to navigate to the image repository details page. +- Filter image repositories by their name. +- [Delete](#delete-images-from-within-gitlab) one or more image repository. +- Navigate to the image repository details page. - Show a **Quick start** dropdown with the most common commands to log in, build and push -- Optionally, a banner will be visible if the [cleanup policy](#cleanup-policy) is enabled for this project. +- Show a banner if the optional [cleanup policy](#cleanup-policy) is enabled for this project. ### Control Container Registry for your group @@ -79,15 +79,15 @@ Navigate to your groups's **{package}** **Packages & Registries > Container Regi  -This view will: +This view allows you to: - Show all the image repositories of the projects that belong to this group. -- Allow to [delete](#delete-images-from-within-gitlab) one or more image repositories. -- Allow to navigate to a specific image repository details page. +- [Delete](#delete-images-from-within-gitlab) one or more image repositories. +- Navigate to a specific image repository details page. ### Image Repository details page -Clicking on the name of any image repository will navigate to the details. +Clicking on the name of any image repository navigates to the details.  @@ -133,8 +133,10 @@ enabled in your account, you need to pass a [personal access token](../../profile/personal_access_tokens.md) instead of your password in order to login to GitLab's Container Registry. -If a project is private, credentials will need to be provided for authorization. -There are two ways to do this: +Credentials must be provided for authorization to any non-public registry. Only project members can access private, +GitLab-hosted registries. + +There are two ways to authenticate: - By using a [personal access token](../../profile/personal_access_tokens.md). - By using a [deploy token](../../project/deploy_tokens/index.md). @@ -158,7 +160,7 @@ docker build -t registry.example.com/group/project/image . docker push registry.example.com/group/project/image ``` -Your image will be named after the following scheme: +Your image is named after the following scheme: ```plaintext <registry URL>/<namespace>/<project>/<image> @@ -175,8 +177,8 @@ registry.example.com/group/project/my/image:rc1 ## Build and push images using GitLab CI/CD -While you can build and push your images from your local machine, the true -power of the Container Registry comes when you combine it with GitLab CI/CD. +While you can build and push your images from your local machine, take +full advantage of the Container Registry by combining it with GitLab CI/CD. You can then create workflows and automate any processes that involve testing, building, and eventually deploying your project from the Docker image you created. @@ -192,7 +194,7 @@ Before diving into the details, some things you should be aware of: - Doing an explicit `docker pull` before each `docker run` fetches the latest image that was just built. This is especially important if you are using multiple Runners that cache images locally. Using the Git SHA in your - image tag makes this less necessary since each job will be unique and you + image tag makes this less necessary since each job is unique and you shouldn't ever have a stale image. However, it's still possible to have a stale image if you re-build a given commit after a dependency has changed. - You don't want to build directly to `latest` tag in case there are multiple jobs @@ -201,10 +203,7 @@ Before diving into the details, some things you should be aware of: ### Authenticating to the Container Registry with GitLab CI/CD There are three ways to authenticate to the Container Registry via -[GitLab CI/CD](../../../ci/yaml/README.md) which depend on the visibility of -your project. - -Available for all projects, though more suitable for public ones: +[GitLab CI/CD](../../../ci/yaml/README.md): - **Using the special `CI_REGISTRY_USER` variable**: The user specified by this variable is created for you in order to push to the Registry connected to your project. Its password is automatically @@ -216,14 +215,22 @@ Available for all projects, though more suitable for public ones: docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY ``` -For private and internal projects: +- **Using the GitLab Deploy Token**: You can create and use a + [special deploy token](../../project/deploy_tokens/index.md#gitlab-deploy-token) + with your projects. + Once created, you can use the special environment variables, and GitLab CI/CD + fills them in for you. You can use the following example as-is: + + ```shell + docker login -u $CI_DEPLOY_USER -p $CI_DEPLOY_PASSWORD $CI_REGISTRY + ``` - **Using a personal access token**: You can create and use a [personal access token](../../profile/personal_access_tokens.md) in case your project is private: - For read (pull) access, the scope should be `read_registry`. - - For read/write (pull/push) access, use `api`. + - For write (push) access, the scope should be `write_registry`. Replace the `<username>` and `<access_token>` in the following example: @@ -231,16 +238,6 @@ For private and internal projects: docker login -u <username> -p <access_token> $CI_REGISTRY ``` -- **Using the GitLab Deploy Token**: You can create and use a - [special deploy token](../../project/deploy_tokens/index.md#gitlab-deploy-token) - with your private projects. It provides read-only (pull) access to the Registry. - Once created, you can use the special environment variables, and GitLab CI/CD - will fill them in for you. You can use the following example as-is: - - ```shell - docker login -u $CI_DEPLOY_USER -p $CI_DEPLOY_PASSWORD $CI_REGISTRY - ``` - ### Container Registry examples with GitLab CI/CD If you're using Docker-in-Docker on your Runners, this is how your `.gitlab-ci.yml` @@ -276,7 +273,7 @@ build: Here, `$CI_REGISTRY_IMAGE` would be resolved to the address of the registry tied to this project. Since `$CI_COMMIT_REF_NAME` resolves to the branch or tag name, -and your branch-name can contain forward slashes (e.g., feature/my-feature), it is +and your branch name can contain forward slashes (for example, `feature/my-feature`), it is safer to use `$CI_COMMIT_REF_SLUG` as the image tag. This is due to that image tags cannot contain forward slashes. We also declare our own variable, `$IMAGE_TAG`, combining the two to save us some typing in the `script` section. @@ -352,8 +349,8 @@ is set to `always`. ### Using a Docker-in-Docker image from your Container Registry -If you want to use your own Docker images for Docker-in-Docker, there are a few -things you need to do in addition to the steps in the +To use your own Docker images for Docker-in-Docker, follow these steps +in addition to the steps in the [Docker-in-Docker](../../../ci/docker/using_docker_build.md#use-docker-in-docker-workflow-with-docker-executor) section: 1. Update the `image` and `service` to point to your registry. @@ -373,8 +370,8 @@ Below is an example of what your `.gitlab-ci.yml` should look like: - docker run my-docker-image /script/to/run/tests ``` -If you forget to set the service alias, the `docker:19.03.12` image won't find the -`dind` service, and an error like the following will be thrown: +If you forget to set the service alias, the `docker:19.03.12` image is unable to find the +`dind` service, and an error like the following is thrown: ```plaintext error during connect: Get http://docker:2376/v1.39/info: dial tcp: lookup docker on 192.168.0.1:53: no such host @@ -496,81 +493,71 @@ Container Registry. > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/15398) in GitLab 12.8. > - [Renamed](https://gitlab.com/gitlab-org/gitlab/-/issues/218737) from "expiration policy" to "cleanup policy" in GitLab 13.2. -For a specific project, if you want to remove tags you no longer need, -you can create a cleanup policy. When the policy is applied, tags matching the regex pattern are removed. +The cleanup policy is a scheduled job you can use to remove tags from the Container Registry. +For the project where it's defined, tags matching the regex pattern are removed. The underlying layers and images remain. -To delete the underlying layers and images no longer associated with any tags, Instance Administrators can use +To delete the underlying layers and images that aren't associated with any tags, administrators can use [garbage collection](../../../administration/packages/container_registry.md#removing-unused-layers-not-referenced-by-manifests) with the `-m` switch. -NOTE: **Note:** -For GitLab.com, cleanup policies are not available for projects created -before this feature was deployed to production (February 2020). -Support for pre-existing projects on GitLab.com -[is planned](https://gitlab.com/gitlab-org/gitlab/-/issues/196124). -For self-managed instances, cleanup policies may be enabled by an admin in the -[GitLab application settings](../../../api/settings.md#change-application-settings) by setting `container_expiration_policies_enable_historic_entries` to true. -Note the inherent [risks involved](./index.md#use-with-external-container-registries). - -The cleanup policy algorithm starts by collecting all the tags for a given repository in a list, -then goes through a process of excluding tags from it until only the ones to be deleted remain: - -1. Collect all the tags for a given repository in a list. -1. Excludes the tag named `latest` from the list. -1. Evaluates the `name_regex`, excluding non-matching names from the list. -1. Excludes any tags that do not have a manifest (not part of the options). -1. Orders the remaining tags by `created_date`. -1. Excludes from the list the N tags based on the `keep_n` value (Number of tags to retain). -1. Excludes from the list the tags more recent than the `older_than` value (Cleanup interval). -1. Excludes from the list any tags matching the `name_regex_keep` value (Images to preserve). -1. Finally, the remaining tags in the list are deleted from the Container Registry. +### Enable the cleanup policy -### Managing project cleanup policy through the UI +Cleanup policies can be run on all projects, with these exceptions: -To manage project cleanup policy, navigate to **{settings}** **Settings > CI/CD > Container Registry tag cleanup policy**. +- For GitLab.com, the project must have been created after 2020-02-22. + Support for projects created earlier + [is planned](https://gitlab.com/gitlab-org/gitlab/-/issues/196124). +- For self-managed GitLab instances, the project must have been created + in GitLab 12.8 or later. However, an administrator can enable the cleanup policy + for all projects (even those created before 12.8) in + [GitLab application settings](../../../api/settings.md#change-application-settings) + by setting `container_expiration_policies_enable_historic_entries` to true. -The UI allows you to configure the following: + There are performance risks with enabling it for all projects, especially if you + are using an [external registry](./index.md#use-with-external-container-registries). -- **Cleanup policy:** enable or disable the cleanup policy. -- **Cleanup interval:** how long tags are exempt from being deleted. -- **Cleanup schedule:** how often the cron job checking the tags should run. -- **Number of tags to retain:** how many tags to _always_ keep for each image. -- **Docker tags with names matching this regex pattern will expire:** the regex used to determine what tags should be cleaned up. To qualify all tags for cleanup, use the default value of `.*`. -- **Docker tags with names matching this regex pattern will be preserved:** the regex used to determine what tags should be preserved. To preserve all tags, use the default value of `.*`. +### How the cleanup policy works -#### Troubleshooting cleanup policies +The cleanup policy collects all tags in the Container Registry and excludes tags +until only the tags to be deleted remain. -If you see the following message: - -"Something went wrong while updating the cleanup policy." - -Check the regex patterns to ensure they are valid. +The cleanup policy: -You can use [Rubular](https://rubular.com/) to check your regex. -View some common [regex pattern examples](#regex-pattern-examples). +1. Collects all tags for a given repository in a list. +1. Excludes the tag named `latest` from the list. +1. Evaluates the `name_regex` (tags to expire), excluding non-matching names from the list. +1. Excludes any tags that do not have a manifest (not part of the options in the UI). +1. Orders the remaining tags by `created_date`. +1. Excludes from the list the N tags based on the `keep_n` value (Number of tags to retain). +1. Excludes from the list the tags more recent than the `older_than` value (Expiration interval). +1. Excludes from the list any tags matching the `name_regex_keep` value (tags to preserve). +1. Finally, the remaining tags in the list are deleted from the Container Registry. -### Managing project cleanup policy through the API +### Create a cleanup policy -You can set, update, and disable the cleanup policies using the GitLab API. +You can create a cleanup policy in [the API](#use-the-cleanup-policy-api) or the UI. -Examples: +To create a cleanup policy in the UI: -- Select all tags, keep at least 1 tag per image, clean up any tag older than 14 days, run once a month, preserve any images with the name `master` and the policy is enabled: +1. For your project, go to **Settings > CI/CD**. +1. Expand the **Cleanup policy for tags** section. +1. Complete the fields. - ```shell - curl --request PUT --header 'Content-Type: application/json;charset=UTF-8' --header "PRIVATE-TOKEN: <your_access_token>" --data-binary '{"container_expiration_policy_attributes":{"cadence":"1month","enabled":true,"keep_n":1,"older_than":"14d","name_regex":"","name_regex_delete":".*","name_regex_keep":".*-master"}}' 'https://gitlab.example.com/api/v4/projects/2' - ``` + | Field | Description | + |---------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------| + | **Cleanup policy** | Turn the policy on or off. | + | **Expiration interval** | How long tags are exempt from being deleted. | + | **Expiration schedule** | How often the policy should run. | + | **Number of tags to retain** | How many tags to _always_ keep for each image. | + | **Tags with names matching this regex pattern expire:** | The regex pattern that determines which tags to remove. For all tags, use `.*`. See other [regex pattern examples](#regex-pattern-examples). | + | **Tags with names matching this regex pattern are preserved:** | The regex pattern that determines which tags to preserve. The `latest` tag is always preserved. For all tags, use `.*`. See other [regex pattern examples](#regex-pattern-examples). | -See the API documentation for further details: [Edit project](../../../api/projects.md#edit-project). +1. Click **Set cleanup policy**. -### Use with external container registries +Depending on the interval you chose, the policy is scheduled to run. -When using an [external container registry](./../../../administration/packages/container_registry.md#use-an-external-container-registry-with-gitlab-as-an-auth-endpoint), -running a cleanup policy on a project may have some performance risks. If a project is going to run -a policy that will remove large quantities of tags (in the thousands), the GitLab background jobs that -run the policy may get backed up or fail completely. It is recommended you only enable container cleanup -policies for projects that were created before GitLab 12.8 if you are confident the amount of tags -being cleaned up will be minimal. +NOTE: **Note:** +If you edit the policy and click **Set cleanup policy** again, the interval is reset. ### Regex pattern examples @@ -602,6 +589,41 @@ Here are examples of regex patterns you may want to use: (?:v.+|master|release) ``` +### Use the cleanup policy API + +You can set, update, and disable the cleanup policies using the GitLab API. + +Examples: + +- Select all tags, keep at least 1 tag per image, clean up any tag older than 14 days, run once a month, preserve any images with the name `master` and the policy is enabled: + + ```shell + curl --request PUT --header 'Content-Type: application/json;charset=UTF-8' --header "PRIVATE-TOKEN: <your_access_token>" --data-binary '{"container_expiration_policy_attributes":{"cadence":"1month","enabled":true,"keep_n":1,"older_than":"14d","name_regex":"","name_regex_delete":".*","name_regex_keep":".*-master"}}' 'https://gitlab.example.com/api/v4/projects/2' + ``` + +See the API documentation for further details: [Edit project](../../../api/projects.md#edit-project). + +### Use with external container registries + +When using an [external container registry](./../../../administration/packages/container_registry.md#use-an-external-container-registry-with-gitlab-as-an-auth-endpoint), +running a cleanup policy on a project may have some performance risks. +If a project runs a policy to remove thousands of tags +the GitLab background jobs may get backed up or fail completely. +It is recommended you only enable container cleanup +policies for projects that were created before GitLab 12.8 if you are confident the number of tags +being cleaned up is minimal. + +### Troubleshooting cleanup policies + +If you see the following message: + +"Something went wrong while updating the cleanup policy." + +Check the regex patterns to ensure they are valid. + +You can use [Rubular](https://rubular.com/) to check your regex. +View some common [regex pattern examples](#regex-pattern-examples). + ## Use the Container Registry to store Helm Charts With the launch of [Helm v3](https://helm.sh/docs/topics/registries/), @@ -616,9 +638,9 @@ You can read more about the above challenges [here](https://gitlab.com/gitlab-or - Moving or renaming existing Container Registry repositories is not supported once you have pushed images, because the images are signed, and the signature includes the repository name. To move or rename a repository with a -Container Registry, you will have to delete all existing images. +Container Registry, you must delete all existing images. - Prior to GitLab 12.10, any tags that use the same image ID as the `latest` tag -will not be deleted by the cleanup policy. +are not deleted by the cleanup policy. ## Troubleshooting the GitLab Container Registry @@ -637,7 +659,7 @@ name. ### Troubleshoot as a GitLab server admin Troubleshooting the GitLab Container Registry, most of the times, requires -administration access to the GitLab server. +administrator access to the GitLab server. [Read how to troubleshoot the Container Registry](../../../administration/packages/container_registry.md#troubleshooting). @@ -655,22 +677,22 @@ the project. The following procedure uses these sample project names: -- For the current project: `example.gitlab.com/org/build/sample_project/cr:v2.9.1` -- For the new project: `example.gitlab.com/new_org/build/new_sample_project/cr:v2.9.1` +- For the current project: `gitlab.example.com/org/build/sample_project/cr:v2.9.1` +- For the new project: `gitlab.example.com/new_org/build/new_sample_project/cr:v2.9.1` Use your own URLs to complete the following steps: 1. Download the Docker images on your computer: ```shell - docker login example.gitlab.com - docker pull example.gitlab.com/org/build/sample_project/cr:v2.9.1 + docker login gitlab.example.com + docker pull gitlab.example.com/org/build/sample_project/cr:v2.9.1 ``` 1. Rename the images to match the new project name: ```shell - docker tag example.gitlab.com/org/build/sample_project/cr:v2.9.1 example.gitlab.com/new_org/build/new_sample_project/cr:v2.9.1 + docker tag gitlab.example.com/org/build/sample_project/cr:v2.9.1 gitlab.example.com/new_org/build/new_sample_project/cr:v2.9.1 ``` 1. Delete the images in both projects by using the [UI](#delete-images) or [API](../../../api/packages.md#delete-a-project-package). @@ -680,7 +702,7 @@ Use your own URLs to complete the following steps: 1. Restore the images: ```shell - docker push example.gitlab.com/new_org/build/new_sample_project/cr:v2.9.1 + docker push gitlab.example.com/new_org/build/new_sample_project/cr:v2.9.1 ``` Follow [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/18383) for details. diff --git a/doc/user/packages/dependency_proxy/index.md b/doc/user/packages/dependency_proxy/index.md index e68883a1382..e3ee8909165 100644 --- a/doc/user/packages/dependency_proxy/index.md +++ b/doc/user/packages/dependency_proxy/index.md @@ -4,9 +4,9 @@ group: Package info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# Dependency Proxy **(PREMIUM ONLY)** +# Dependency Proxy **(PREMIUM)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7934) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.11. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7934) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.11. NOTE: **Note:** This is the user guide. In order to use the dependency proxy, an administrator @@ -63,10 +63,10 @@ To get a Docker image into the dependency proxy: image: gitlab.com/groupname/dependency_proxy/containers/alpine:latest ``` -GitLab will then pull the Docker image from Docker Hub and will cache the blobs -on the GitLab server. The next time you pull the same image, it will get the latest -information about the image from Docker Hub but will serve the existing blobs -from GitLab. +GitLab pulls the Docker image from Docker Hub and caches the blobs +on the GitLab server. The next time you pull the same image, GitLab gets the latest +information about the image from Docker Hub but serves the existing blobs +from the GitLab server. The blobs are kept forever, and there is no hard limit on how much data can be stored. @@ -82,6 +82,6 @@ for more details. The following limitations apply: -- Only public groups are supported (authentication is not supported yet). +- Only [public groups are supported](https://gitlab.com/gitlab-org/gitlab/-/issues/11582) (authentication is not supported yet). - Only Docker Hub is supported. - This feature requires Docker Hub being available. diff --git a/doc/user/packages/go_proxy/index.md b/doc/user/packages/go_proxy/index.md index a705b956d30..edf1528a751 100644 --- a/doc/user/packages/go_proxy/index.md +++ b/doc/user/packages/go_proxy/index.md @@ -4,13 +4,14 @@ group: Package info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# GitLab Go Proxy **(PREMIUM)** +# GitLab Go Proxy > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/27376) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.1. > - It's deployed behind a feature flag, disabled by default. > - It's disabled on GitLab.com. > - It's not recommended for production use. -> - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enable-the-go-proxy). **(PREMIUM)** +> - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enable-the-go-proxy). +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/221259) to GitLab Core in 13.3. With the Go proxy for GitLab, every project in GitLab can be fetched with the [Go proxy protocol](https://proxy.golang.org/). @@ -53,7 +54,7 @@ the **{package}** **Packages > List** entry under your project's sidebar, verify the following: 1. Your GitLab administrator has - [enabled support for the Package Registry](../../../administration/packages/index.md). **(PREMIUM ONLY)** + [enabled support for the Package Registry](../../../administration/packages/index.md). 1. The Package Registry is [enabled for your project](../index.md). NOTE: **Note:** diff --git a/doc/user/packages/index.md b/doc/user/packages/index.md index ab9cdc204f8..92d31c31986 100644 --- a/doc/user/packages/index.md +++ b/doc/user/packages/index.md @@ -4,106 +4,37 @@ group: Package info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# GitLab Package Registry +# Packages & Registries -With the GitLab Package Registry, you can use GitLab as a private or public repository -for a variety of common package managers. You can build and publish +The GitLab [Package Registry](package_registry/index.md) acts as a private or public registry +for a variety of common package managers. You can publish and share packages, which can be easily consumed as a dependency in downstream projects. -GitLab acts as a repository for the following: - -| Software repository | Description | Available in GitLab version | -| ------------------- | ----------- | --------------------------- | -| [Container Registry](container_registry/index.md) | The GitLab Container Registry enables every project in GitLab to have its own space to store [Docker](https://www.docker.com/) images. | 8.8+ | -| [Dependency Proxy](dependency_proxy/index.md) **(PREMIUM)** | The GitLab Dependency Proxy sets up a local proxy for frequently used upstream images/packages. | 11.11+ | -| [Conan Repository](conan_repository/index.md) **(PREMIUM)** | The GitLab Conan Repository enables every project in GitLab to have its own space to store [Conan](https://conan.io/) packages. | 12.6+ | -| [Maven Repository](maven_repository/index.md) **(PREMIUM)** | The GitLab Maven Repository enables every project in GitLab to have its own space to store [Maven](https://maven.apache.org/) packages. | 11.3+ | -| [NPM Registry](npm_registry/index.md) **(PREMIUM)** | The GitLab NPM Registry enables every project in GitLab to have its own space to store [NPM](https://www.npmjs.com/) packages. | 11.7+ | -| [NuGet Repository](nuget_repository/index.md) **(PREMIUM)** | The GitLab NuGet Repository will enable every project in GitLab to have its own space to store [NuGet](https://www.nuget.org/) packages. | 12.8+ | -| [PyPi Repository](pypi_repository/index.md) **(PREMIUM)** | The GitLab PyPi Repository will enable every project in GitLab to have its own space to store [PyPi](https://pypi.org/) packages. | 12.10+ | -| [Go Proxy](go_proxy/index.md) **(PREMIUM)** | The Go proxy for GitLab enables every project in GitLab to be fetched with the [Go proxy protocol](https://proxy.golang.org/). | 13.1+ | -| [Composer Repository](composer_repository/index.md) **(PREMIUM)** | The GitLab Composer Repository will enable every project in GitLab to have its own space to store [Composer](https://getcomposer.org/) packages. | 13.2+ | - -## View packages - -You can view packages for your project or group. - -1. Go to the project or group. -1. Go to **{package}** **Packages & Registries > Package Registry**. - -You can search, sort, and filter packages on this page. - -For information on how to create and upload a package, view the GitLab documentation for your package type. - -## Use GitLab CI/CD to build packages - -You can use [GitLab CI/CD](./../../ci/README.md) to build packages. -For Maven and NPM packages, and Composer dependencies, you can -authenticate with GitLab by using the `CI_JOB_TOKEN`. - -CI/CD templates, which you can use to get started, are in [this repo](https://gitlab.com/gitlab-org/gitlab/-/tree/master/lib/gitlab/ci/templates). - -Learn more about [using CI/CD to build Maven packages](maven_repository/index.md#creating-maven-packages-with-gitlab-cicd) -and [NPM packages](npm_registry/index.md#publishing-a-package-with-cicd). - -If you use CI/CD to build a package, extended activity -information is displayed when you view the package details: - - - -You can view which pipeline published the package, as well as the commit and -user who triggered it. - -## Download a package - -To download a package: - -1. Go to **{package}** **Packages & Registries > Package Registry**. -1. Click the name of the package you want to download. -1. In the **Activity** section, click the name of the package you want to download. - -## Delete a package - -You cannot edit a package after you publish it in the Package Registry. Instead, you -must delete and recreate it. - -- You cannot delete packages from the group view. You must delete them from the project view instead. - See [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/227714) for details. -- You must have suitable [permissions](../permissions.md). - -You can delete packages by using [the API](../../api/packages.md#delete-a-project-package) or the UI. - -To delete a package in the UI: - -1. Go to **{package}** **Packages & Registries > Package Registry**. -1. Find the name of the package you want to delete. -1. Click **Delete**. - -The package is permanently deleted. - -## Disable the Package Registry - -The Package Registry is automatically enabled. - -If you are using a self-managed instance of GitLab, your administrator can remove -the menu item, **{package}** **Packages & Registries**, from the GitLab sidebar. For more information, -see the [administration documentation](../../administration/packages/index.md). - -You can also remove the Package Registry for your project specifically: - -1. In your project, go to **{settings}** **Settings > General**. -1. Expand the **Visibility, project features, permissions** section and disable the - **Packages** feature. -1. Click **Save changes**. - -The **{package}** **Packages & Registries > Package Registry** entry is removed from the sidebar. - -## Package workflows - -Learn how to use the GitLab Package Registry to build your own custom package workflow. - -- [Use a project as a package registry](./workflows/project_registry.md) to publish all of your packages to one project. -- Publish multiple different packages from one [monorepo project](./workflows/monorepo.md). +The Package Registry supports the following formats: + +<div class="row"> +<div class="col-md-9"> +<table align="left" style="width:50%"> +<tr style="background:#dfdfdf"><th>Package type</th><th>GitLab version</th></tr> +<tr><td><a href="https://docs.gitlab.com/ee/user/packages/composer_repository/index.html">Composer</a></td><td>13.2+</td></tr> +<tr><td><a href="https://docs.gitlab.com/ee/user/packages/conan_repository/index.html">Conan</a></td><td>12.6+</td></tr> +<tr><td><a href="https://docs.gitlab.com/ee/user/packages/go_proxy/index.html">Go</a></td><td>13.1+</td></tr> +<tr><td><a href="https://docs.gitlab.com/ee/user/packages/maven_repository/index.html">Maven</a></td><td>11.3+</td></tr> +<tr><td><a href="https://docs.gitlab.com/ee/user/packages/npm_registry/index.html">NPM</a></td><td>11.7+</td></tr> +<tr><td><a href="https://docs.gitlab.com/ee/user/packages/nuget_repository/index.html">NuGet</a></td><td>12.8+</td></tr> +<tr><td><a href="https://docs.gitlab.com/ee/user/packages/pypi_repository/index.html">PyPI</a></td><td>12.10+</td></tr> +</table> +</div> +</div> + +You can also use the [API](../../api/packages.md) to administer the Package Registry. + +The GitLab [Container Registry](container_registry/index.md) is a secure and private registry for container images. +It's built on open source software and completely integrated within GitLab. +Use GitLab CI/CD to create and publish images. Use the GitLab [API](../../api/container_registry.md) to +manage the registry across groups and projects. + +The [Dependency Proxy](dependency_proxy/index.md) is a local proxy for frequently-used upstream images and packages. ## Suggested contributions diff --git a/doc/user/packages/maven_repository/index.md b/doc/user/packages/maven_repository/index.md index b194b7d837a..f98a8eb9c6d 100644 --- a/doc/user/packages/maven_repository/index.md +++ b/doc/user/packages/maven_repository/index.md @@ -4,9 +4,10 @@ group: Package info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# GitLab Maven Repository **(PREMIUM)** +# GitLab Maven Repository -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/5811) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.3. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/5811) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.3. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/221259) to GitLab Core in 13.3. With the GitLab [Maven](https://maven.apache.org) Repository, every project can have its own space to store its Maven artifacts. @@ -17,7 +18,7 @@ project can have its own space to store its Maven artifacts. NOTE: **Note:** This option is available only if your GitLab administrator has -[enabled support for the Maven repository](../../../administration/packages/index.md).**(PREMIUM ONLY)** +[enabled support for the Maven repository](../../../administration/packages/index.md). After the Packages feature is enabled, the Maven Repository will be available for all new projects by default. To enable it for existing projects, or if you want @@ -321,7 +322,7 @@ repositories { name "GitLab" credentials(HttpHeaderCredentials) { name = 'Job-Token' - value = '${CI_JOB_TOKEN}' + value = System.getenv("CI_JOB_TOKEN") } authentication { header(HttpHeaderAuthentication) @@ -689,7 +690,7 @@ downloaded from the GitLab Package Registry: Downloading from gitlab-maven: http://gitlab.com/api/v4/projects/PROJECT_ID/packages/maven/com/mycompany/mydepartment/my-project/1.0-SNAPSHOT/my-project-1.0-20200128.120857-1.pom ``` -#### Install with `mvn dependency:get` +### Install using Maven with `mvn dependency:get` The second way to install packages is to use the Maven commands directly. Inside your project directory, run: @@ -782,7 +783,7 @@ is updated: ```yaml deploy: - image: maven:3.3.9-jdk-8 + image: maven:3.6-jdk-11 script: - 'mvn deploy -s ci_settings.xml' only: @@ -807,7 +808,7 @@ is updated: ```yaml deploy: - image: gradle:latest + image: gradle:6.5-jdk11 script: - 'gradle publish' only: @@ -816,11 +817,6 @@ is updated: 1. Push those files to your repository. -The next time the `deploy` job runs, it will copy `ci_settings.xml` to the -user's home location (in this case the user is `root` since it runs in a -Docker container), and Maven will use the configured CI -[environment variables](../../../ci/variables/README.md#predefined-environment-variables). - ### Version validation The version string is validated using the following regex. @@ -833,10 +829,25 @@ You can play around with the regex and try your version strings on [this regular ## Troubleshooting -### Useful Maven command line options +### Review network trace logs + +If you are having issues with the Maven Repository, you may want to review network trace logs. + +For example, try to run `mvn deploy` locally with a PAT token and use these options: + +```shell +mvn deploy \ +-Dorg.slf4j.simpleLogger.log.org.apache.maven.wagon.providers.http.httpclient=trace \ +-Dorg.slf4j.simpleLogger.log.org.apache.maven.wagon.providers.http.httpclient.wire=trace +``` + +CAUTION: **Caution:** +When you set these options, all network requests are logged and a large amount of output is generated. + +### Useful Maven command-line options -There's some [maven command line options](https://maven.apache.org/ref/current/maven-embedder/cli.html) -which maybe useful when doing tasks with GitLab CI/CD. +There are some [Maven command-line options](https://maven.apache.org/ref/current/maven-embedder/cli.html) +that may be useful when performing tasks with GitLab CI/CD. - File transfer progress can make the CI logs hard to read. Option `-ntp,--no-transfer-progress` was added in diff --git a/doc/user/packages/npm_registry/index.md b/doc/user/packages/npm_registry/index.md index 390b2c28e10..5b90ec6f18c 100644 --- a/doc/user/packages/npm_registry/index.md +++ b/doc/user/packages/npm_registry/index.md @@ -4,9 +4,10 @@ group: Package info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# GitLab NPM Registry **(PREMIUM)** +# GitLab NPM Registry -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/5934) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.7. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/5934) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.7. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/221259) to GitLab Core in 13.3. With the GitLab NPM Registry, every project can have its own space to store NPM packages. @@ -20,9 +21,9 @@ Only [scoped](https://docs.npmjs.com/misc/scope) packages are supported. NOTE: **Note:** This option is available only if your GitLab administrator has -[enabled support for the NPM registry](../../../administration/packages/index.md).**(PREMIUM ONLY)** +[enabled support for the NPM registry](../../../administration/packages/index.md). -After the NPM registry is enabled, it will be available for all new projects +Enabling the NPM registry makes it available for all new projects by default. To enable it for existing projects, or if you want to disable it: 1. Navigate to your project's **Settings > General > Visibility, project features, permissions**. @@ -36,7 +37,7 @@ get familiar with the package naming convention. ## Getting started -This section will cover installing NPM (or Yarn) and building a package for your +This section covers how to install NPM (or Yarn) and build a package for your JavaScript project. This is a quickstart if you are new to NPM packages. If you are already using NPM and understand how to build your own packages, move on to the [next section](#authenticating-to-the-gitlab-npm-registry). @@ -93,24 +94,24 @@ Or if you're using Yarn: yarn init ``` -This will take you through a series of questions to produce a `package.json` +This takes you through a series of questions to produce a `package.json` file, which is required for all NPM packages. The most important question is the package name. NPM packages must [follow the naming convention](#package-naming-convention) and be scoped to the project or group where the registry exists. Once you have completed the setup, you are now ready to upload your package to -the GitLab registry. To get started, you will need to set up authentication then +the GitLab registry. To get started, you need to set up authentication and then configure GitLab as a remote registry. ## Authenticating to the GitLab NPM Registry If a project is private or you want to upload an NPM package to GitLab, -credentials will need to be provided for authentication. [Personal access tokens](../../profile/personal_access_tokens.md) +you need to provide credentials for authentication. [Personal access tokens](../../profile/personal_access_tokens.md) and [deploy tokens](../../project/deploy_tokens/index.md) are preferred, but support is available for [OAuth tokens](../../../api/oauth2.md#resource-owner-password-credentials-flow). CAUTION: **Two-factor authentication (2FA) is only supported with personal access tokens:** -If you have 2FA enabled, you need to use a [personal access token](../../profile/personal_access_tokens.md) with OAuth headers with the scope set to `api` or a [deploy token](../../project/deploy_tokens/index.md) with `read_package_registry` or `write_package_registry` scopes. Standard OAuth tokens won't be able to authenticate to the GitLab NPM Registry. +If you have 2FA enabled, you need to use a [personal access token](../../profile/personal_access_tokens.md) with OAuth headers with the scope set to `api` or a [deploy token](../../project/deploy_tokens/index.md) with `read_package_registry` or `write_package_registry` scopes. Standard OAuth tokens cannot authenticate to the GitLab NPM Registry. ### Authenticating with a personal access token or deploy token @@ -168,7 +169,7 @@ Then, you could run `npm publish` either locally or via GitLab CI/CD: > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9104) in GitLab Premium 12.5. If you’re using NPM with GitLab CI/CD, a CI job token can be used instead of a personal access token or deploy token. -The token will inherit the permissions of the user that generates the pipeline. +The token inherits the permissions of the user that generates the pipeline. Add a corresponding section to your `.npmrc` file: @@ -180,7 +181,7 @@ Add a corresponding section to your `.npmrc` file: ## Uploading packages -Before you will be able to upload a package, you need to specify the registry +Before you can upload a package, you need to specify the registry for NPM. To do this, add the following section to the bottom of `package.json`: ```json @@ -205,8 +206,8 @@ npm publish You can then navigate to your project's **Packages & Registries** page and see the uploaded packages or even delete them. -If you attempt to publish a package with a name that already exists within -a given scope, you will receive a `403 Forbidden!` error. +Attempting to publish a package with a name that already exists within +a given scope causes a `403 Forbidden!` error. ## Uploading a package with the same version twice @@ -245,7 +246,7 @@ project path is `My-Group/project-foo`, your package must be named `@My-Group/an `@my-group/any-package-name` will not work. CAUTION: **When updating the path of a user/group or transferring a (sub)group/project:** -If you update the root namespace of a project with NPM packages, your changes will be rejected. To be allowed to do that, make sure to remove any NPM package first. Don't forget to update your `.npmrc` files to follow the above naming convention and run `npm publish` if necessary. +Make sure to remove any NPM packages first. You cannot update the root namespace of a project with NPM packages. Don't forget to update your `.npmrc` files to follow the above naming convention and run `npm publish` if necessary. Now, you can configure your project to authenticate with the GitLab NPM Registry. @@ -253,16 +254,16 @@ Registry. ## Installing a package NPM packages are commonly installed using the `npm` or `yarn` commands -inside a JavaScript project. If you haven't already, you will need to set the +inside a JavaScript project. If you haven't already, set the URL for scoped packages. You can do this with the following command: ```shell npm config set @foo:registry https://gitlab.com/api/v4/packages/npm/ ``` -You will need to replace `@foo` with your scope. +Replace `@foo` with your scope. -Next, you will need to ensure [authentication](#authenticating-to-the-gitlab-npm-registry) +Next, you need to ensure [authentication](#authenticating-to-the-gitlab-npm-registry) is setup so you can successfully install the package. Once this has been completed, you can run the following command inside your project to install a package: @@ -281,7 +282,7 @@ yarn add @my-project-scope/my-package > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/55344) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.9. -By default, when an NPM package is not found in the GitLab NPM Registry, the request will be forwarded to [npmjs.com](https://www.npmjs.com/). +By default, when an NPM package is not found in the GitLab NPM Registry, the request is forwarded to [npmjs.com](https://www.npmjs.com/). Administrators can disable this behavior in the [Continuous Integration settings](../../admin_area/settings/continuous_integration.md). @@ -367,7 +368,7 @@ And the `.npmrc` file should look like: ### `npm install` returns `Error: Failed to replace env in config: ${NPM_TOKEN}` -You do not need a token to run `npm install` unless your project is private (the token is only required to publish). If the `.npmrc` file was checked in with a reference to `$NPM_TOKEN`, you can remove it. If you prefer to leave the reference in, you'll need to set a value prior to running `npm install` or set the value using [GitLab environment variables](./../../../ci/variables/README.md): +You do not need a token to run `npm install` unless your project is private (the token is only required to publish). If the `.npmrc` file was checked in with a reference to `$NPM_TOKEN`, you can remove it. If you prefer to leave the reference in, you need to set a value prior to running `npm install` or set the value using [GitLab environment variables](./../../../ci/variables/README.md): ```shell NPM_TOKEN=<your_token> npm install diff --git a/doc/user/packages/nuget_repository/index.md b/doc/user/packages/nuget_repository/index.md index 4ee5e5c4627..9fb50ce71fb 100644 --- a/doc/user/packages/nuget_repository/index.md +++ b/doc/user/packages/nuget_repository/index.md @@ -4,9 +4,10 @@ group: Package info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# GitLab NuGet Repository **(PREMIUM)** +# GitLab NuGet Repository -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/20050) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.8. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/20050) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.8. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/221259) to GitLab Core in 13.3. With the GitLab NuGet Repository, every project can have its own space to store NuGet packages. @@ -18,7 +19,7 @@ The GitLab NuGet Repository works with: ## Setting up your development environment -You will need [NuGet CLI 5.2 or later](https://www.nuget.org/downloads). Earlier versions have not been tested +[NuGet CLI 5.2 or later](https://www.nuget.org/downloads) is required. Earlier versions have not been tested against the GitLab NuGet Repository and might not work. If you have [Visual Studio](https://visualstudio.microsoft.com/vs/), NuGet CLI is probably already installed. @@ -42,6 +43,9 @@ Available commands: [output truncated] ``` +NOTE: **Note:** +GitLab currently only supports NuGet v3. Earlier versions are not supported. + ### macOS support For macOS, you can also use [Mono](https://www.mono-project.com/) to run @@ -58,9 +62,9 @@ mono nuget.exe NOTE: **Note:** This option is available only if your GitLab administrator has -[enabled support for the Package Registry](../../../administration/packages/index.md). **(PREMIUM ONLY)** +[enabled support for the Package Registry](../../../administration/packages/index.md). -After the NuGet Repository is enabled, it will be available for all new projects +When the NuGet Repository is enabled, it is available for all new projects by default. To enable it for existing projects, or if you want to disable it: 1. Navigate to your project's **Settings > General > Visibility, project features, permissions**. @@ -71,7 +75,7 @@ You should then be able to see the **Packages & Registries** section on the left ## Adding the GitLab NuGet Repository as a source to NuGet -You will need the following: +You need the following: - Your GitLab username. - A personal access token or deploy token. For repository authentication: @@ -108,7 +112,7 @@ nuget source Add -Name "GitLab" -Source "https://gitlab.example/api/v4/projects/ 1. Open [Visual Studio](https://visualstudio.microsoft.com/vs/). 1. Open the **FILE > OPTIONS** (Windows) or **Visual Studio > Preferences** (Mac OS). -1. In the **NuGet** section, open **Sources**. You will see a list of all your NuGet sources. +1. In the **NuGet** section, open **Sources** to see a list of all your NuGet sources. 1. Click **Add**. 1. Fill the fields with: - **Name**: Desired name for the source @@ -152,8 +156,8 @@ When uploading packages, note that: - The maximum allowed size is 50 Megabytes. - If you upload the same package with the same version multiple times, each consecutive upload - is saved as a separate file. When installing a package, GitLab will serve the most recent file. -- When uploading packages to GitLab, they will not be displayed in the packages UI of your project + is saved as a separate file. When installing a package, GitLab serves the most recent file. +- When uploading packages to GitLab, they are not displayed in the packages UI of your project immediately. It can take up to 10 minutes to process a package. ### Upload packages with NuGet CLI @@ -197,7 +201,7 @@ dotnet nuget push MyPackage.1.0.0.nupkg --source gitlab CAUTION: **Warning:** By default, `nuget` checks the official source at `nuget.org` first. If you have a package in the GitLab NuGet Repository with the same name as a package at `nuget.org`, you must specify the source -name or the wrong package will be installed. +name to install the correct package. Install the latest version of a package using the following command: @@ -210,7 +214,7 @@ nuget install <package_id> -OutputDirectory <output_directory> \ Where: - `<package_id>` is the package ID. -- `<output_directory>` is the output directory, where the package will be installed. +- `<output_directory>` is the output directory, where the package is installed. - `<package_version>` (Optional) is the package version. - `<source_name>` (Optional) is the source name. @@ -232,3 +236,35 @@ Where: - `<package_id>` is the package ID. - `<package_version>` (Optional) is the package version. + +## Publishing a NuGet package with CI/CD + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/36424) in GitLab 13.3. + +If you’re using NuGet with GitLab CI/CD, a CI job token can be used instead of a personal access token or deploy token. +The token inherits the permissions of the user that generates the pipeline. + +This example shows how to create a new package each time the `master` branch +is updated: + +1. Add a `deploy` job to your `.gitlab-ci.yml` file: + + ```yaml + image: mcr.microsoft.com/dotnet/core/sdk:3.1 + + stages: + - deploy + + deploy: + stage: deploy + script: + - dotnet restore -p:Configuration=Release + - dotnet build -c Release + - dotnet pack -c Release + - dotnet nuget add source "$CI_SERVER_URL/api/v4/projects/$CI_PROJECT_ID/packages/nuget/index.json" --name gitlab --username gitlab-ci-token --password $CI_JOB_TOKEN --store-password-in-clear-text + - dotnet nuget push "bin/Release/*.nupkg" --source gitlab + only: + - master + ``` + +1. Commit the changes and push it to your GitLab repository to trigger a new CI build. diff --git a/doc/user/packages/img/package_activity_v12_10.png b/doc/user/packages/package_registry/img/package_activity_v12_10.png Binary files differindex 4fea9a7ca3f..4fea9a7ca3f 100644 --- a/doc/user/packages/img/package_activity_v12_10.png +++ b/doc/user/packages/package_registry/img/package_activity_v12_10.png diff --git a/doc/user/packages/package_registry/index.md b/doc/user/packages/package_registry/index.md new file mode 100644 index 00000000000..fd250c9ac95 --- /dev/null +++ b/doc/user/packages/package_registry/index.md @@ -0,0 +1,93 @@ +--- +stage: Package +group: Package +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Package Registry + +> [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/221259) to GitLab Core in 13.3. + +With the GitLab Package Registry, you can use GitLab as a private or public registry +for a variety of common package managers. You can publish and share +packages, which can be easily consumed as a dependency in downstream projects. + +## View packages + +You can view packages for your project or group. + +1. Go to the project or group. +1. Go to **{package}** **Packages & Registries > Package Registry**. + +You can search, sort, and filter packages on this page. + +For information on how to create and upload a package, view the GitLab documentation for your package type. + +## Use GitLab CI/CD to build packages + +You can use [GitLab CI/CD](../../../ci/README.md) to build packages. +For Maven, NuGet and NPM packages, and Composer dependencies, you can +authenticate with GitLab by using the `CI_JOB_TOKEN`. + +CI/CD templates, which you can use to get started, are in [this repo](https://gitlab.com/gitlab-org/gitlab/-/tree/master/lib/gitlab/ci/templates). + +Learn more about [using CI/CD to build Maven packages](../maven_repository/index.md#creating-maven-packages-with-gitlab-cicd), [NPM packages](../npm_registry/index.md#publishing-a-package-with-cicd) and [NuGet Packages](../nuget_repository/index.md#publishing-a-nuget-package-with-cicd). + +If you use CI/CD to build a package, extended activity +information is displayed when you view the package details: + + + +You can view which pipeline published the package, as well as the commit and +user who triggered it. + +## Download a package + +To download a package: + +1. Go to **{package}** **Packages & Registries > Package Registry**. +1. Click the name of the package you want to download. +1. In the **Activity** section, click the name of the package you want to download. + +## Delete a package + +You cannot edit a package after you publish it in the Package Registry. Instead, you +must delete and recreate it. + +- You cannot delete packages from the group view. You must delete them from the project view instead. + See [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/227714) for details. +- You must have suitable [permissions](../../permissions.md). + +You can delete packages by using [the API](../../../api/packages.md#delete-a-project-package) or the UI. + +To delete a package in the UI: + +1. Go to **{package}** **Packages & Registries > Package Registry**. +1. Find the name of the package you want to delete. +1. Click **Delete**. + +The package is permanently deleted. + +## Disable the Package Registry + +The Package Registry is automatically enabled. + +If you are using a self-managed instance of GitLab, your administrator can remove +the menu item, **{package}** **Packages & Registries**, from the GitLab sidebar. For more information, +see the [administration documentation](../../../administration/packages/index.md). + +You can also remove the Package Registry for your project specifically: + +1. In your project, go to **Settings > General**. +1. Expand the **Visibility, project features, permissions** section and disable the + **Packages** feature. +1. Click **Save changes**. + +The **{package}** **Packages & Registries > Package Registry** entry is removed from the sidebar. + +## Package workflows + +Learn how to use the GitLab Package Registry to build your own custom package workflow. + +- [Use a project as a package registry](../workflows/project_registry.md) to publish all of your packages to one project. +- Publish multiple different packages from one [monorepo project](../workflows/monorepo.md). diff --git a/doc/user/packages/pypi_repository/index.md b/doc/user/packages/pypi_repository/index.md index 615741cc303..63e6cd7b5b4 100644 --- a/doc/user/packages/pypi_repository/index.md +++ b/doc/user/packages/pypi_repository/index.md @@ -4,9 +4,10 @@ group: Package info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# GitLab PyPi Repository **(PREMIUM)** +# GitLab PyPi Repository -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/208747) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.10. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/208747) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.10. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/221259) to GitLab Core in 13.3. With the GitLab PyPi Repository, every project can have its own space to store PyPi packages. @@ -17,15 +18,15 @@ The GitLab PyPi Repository works with: ## Setting up your development environment -You will need a recent version of [pip](https://pypi.org/project/pip/) and [twine](https://pypi.org/project/twine/). +You need a recent version of [pip](https://pypi.org/project/pip/) and [twine](https://pypi.org/project/twine/). ## Enabling the PyPi Repository NOTE: **Note:** This option is available only if your GitLab administrator has -[enabled support for the Package Registry](../../../administration/packages/index.md). **(PREMIUM ONLY)** +[enabled support for the Package Registry](../../../administration/packages/index.md). -After the PyPi Repository is enabled, it will be available for all new projects +After the PyPi Repository is enabled, it is available for all new projects by default. To enable it for existing projects, or if you want to disable it: 1. Navigate to your project's **Settings > General > Visibility, project features, permissions**. @@ -36,7 +37,7 @@ You should then be able to see the **Packages & Registries** section on the left ## Getting started -This section will cover creating a new example PyPi package to upload. This is a +This section covers creating a new example PyPi package to upload. This is a quickstart to test out the **GitLab PyPi Registry**. If you already understand how to build and publish your own packages, move on to the [next section](#adding-the-gitlab-pypi-repository-as-a-source). @@ -158,7 +159,7 @@ Package Registry**. Before we do so, we next need to set up authentication. ### Authenticating with a personal access token -You will need the following: +You need the following: - A personal access token. You can generate a [personal access token](../../../user/profile/personal_access_tokens.md) with the scope set to `api` for repository authentication. - A suitable name for your source. @@ -179,7 +180,7 @@ password = <your personal access token> ### Authenticating with a deploy token -You will need the following: +You need the following: - A deploy token. You can generate a [deploy token](./../../project/deploy_tokens/index.md) with the `read_package_registry` and/or `write_package_registry` scopes for repository authentication. - A suitable name for your source. @@ -204,7 +205,7 @@ When uploading packages, note that: - The maximum allowed size is 50 Megabytes. - If you upload the same package with the same version multiple times, each consecutive upload - is saved as a separate file. When installing a package, GitLab will serve the most recent file. + is saved as a separate file. When installing a package, GitLab serves the most recent file. ### Upload packages with Twine @@ -228,8 +229,8 @@ Uploading mypypipackage-0.0.1.tar.gz This indicates that the package was uploaded successfully. You can then navigate to your project's **Packages & Registries** page and see the uploaded packages. -If you did not follow the guide above, the you'll need to ensure your package -has been properly built and you [created a PyPi package with setuptools](https://packaging.python.org/tutorials/packaging-projects/). +If you did not follow the guide above, then you need to ensure your package +has been properly built and you [created a PyPi package with `setuptools`](https://packaging.python.org/tutorials/packaging-projects/). You can then upload your package using the following command: diff --git a/doc/user/packages/workflows/monorepo.md b/doc/user/packages/workflows/monorepo.md index 5acd4fd0735..c87f181bf82 100644 --- a/doc/user/packages/workflows/monorepo.md +++ b/doc/user/packages/workflows/monorepo.md @@ -1,3 +1,9 @@ +--- +stage: Package +group: Package +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Monorepo package management workflows Oftentimes, one project or Git repository may contain multiple different @@ -76,44 +82,39 @@ Using the example project above, this `gitlab-ci.yml` file will publish and publish `MyPackage` anytime changes are made to anywhere _except_ the `Foo` directory on the `master` branch. -```shell +```yaml +image: node:latest + stages: - build -.default-rule: &default-rule - if: '$CI_MERGE_REQUEST_IID || $CI_COMMIT_REF_SLUG == "master"' - -.foo-package: +build-foo-package: + stage: build variables: PACKAGE: "Foo" - before_script: + script: - cd src/components/Foo + - echo "Building $PACKAGE" + - npm publish only: + refs: + - master + - merge_requests changes: - "src/components/Foo/**/*" -.parent-package: +build-my-project-package: + stage: build variables: PACKAGE: "MyPackage" - except: - changes: - - "src/components/Foo/**/*" - -.build-package: - stage: build script: - echo "Building $PACKAGE" - npm publish - rules: - - <<: *default-rule - -build-foo-package: - extends: - - .build-package - - .foo-package - -build-my-project-package: - extends: - - .build-package - - .parent-package + only: + refs: + - master + - merge_requests + except: + changes: + - "src/components/Foo/**/*" ``` diff --git a/doc/user/packages/workflows/project_registry.md b/doc/user/packages/workflows/project_registry.md index df330931ac5..a7bc4436d0e 100644 --- a/doc/user/packages/workflows/project_registry.md +++ b/doc/user/packages/workflows/project_registry.md @@ -1,3 +1,9 @@ +--- +stage: Package +group: Package +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Project as a package registry Using the features of the package registry, it is possible to use one project to store all of your packages. diff --git a/doc/user/permissions.md b/doc/user/permissions.md index ba62cf81847..a89d534c782 100644 --- a/doc/user/permissions.md +++ b/doc/user/permissions.md @@ -8,15 +8,14 @@ Users have different abilities depending on the access level they have in a particular group or project. If a user is both in a project's group and the project itself, the highest permission level is used. -On public and internal projects the Guest role is not enforced. All users will -be able to: +On public and internal projects the Guest role is not enforced. All users can: - Create issues. - Leave comments. - Clone or download the project code. When a member leaves a team's project, all the assigned [Issues](project/issues/index.md) and [Merge Requests](project/merge_requests/index.md) -will be unassigned automatically. +are unassigned automatically. GitLab [administrators](../administration/index.md) receive all permissions. @@ -39,15 +38,15 @@ NOTE: **Note:** In GitLab 11.0, the Master role was renamed to Maintainer. While Maintainer is the highest project-level role, some actions can only be performed by a personal namespace or group owner, -or an instance admin, who receives all permissions. For more information, see [projects members documentation](project/members/index.md). +or an instance administrator, who receives all permissions. For more information, see [projects members documentation](project/members/index.md). The following table depicts the various user permission levels in a project. | Action | Guest | Reporter | Developer |Maintainer| Owner* | |---------------------------------------------------|---------|------------|-------------|----------|--------| | Download project | ✓ (*1*) | ✓ | ✓ | ✓ | ✓ | -| Leave comments | ✓ (*1*) | ✓ | ✓ | ✓ | ✓ | -| View allowed and denied licenses **(ULTIMATE)** | ✓ (*1*) | ✓ | ✓ | ✓ | ✓ | +| Leave comments | ✓ | ✓ | ✓ | ✓ | ✓ | +| View allowed and denied licenses **(ULTIMATE)** | ✓ (*1*) | ✓ | ✓ | ✓ | ✓ | | View License Compliance reports **(ULTIMATE)** | ✓ (*1*) | ✓ | ✓ | ✓ | ✓ | | View Security reports **(ULTIMATE)** | ✓ (*3*) | ✓ | ✓ | ✓ | ✓ | | View Dependency list **(ULTIMATE)** | ✓ (*1*) | ✓ | ✓ | ✓ | ✓ | @@ -61,9 +60,9 @@ The following table depicts the various user permission levels in a project. | See a list of jobs | ✓ (*3*) | ✓ | ✓ | ✓ | ✓ | | See a job log | ✓ (*3*) | ✓ | ✓ | ✓ | ✓ | | Download and browse job artifacts | ✓ (*3*) | ✓ | ✓ | ✓ | ✓ | -| Create new issue | ✓ (*1*) | ✓ | ✓ | ✓ | ✓ | +| Create new issue | ✓ | ✓ | ✓ | ✓ | ✓ | | See related issues | ✓ | ✓ | ✓ | ✓ | ✓ | -| Create confidential issue | ✓ (*1*) | ✓ | ✓ | ✓ | ✓ | +| Create confidential issue | ✓ | ✓ | ✓ | ✓ | ✓ | | View confidential issues | (*2*) | ✓ | ✓ | ✓ | ✓ | | View [Releases](project/releases/index.md) | ✓ (*6*) | ✓ | ✓ | ✓ | ✓ | | View requirements **(ULTIMATE)** | ✓ | ✓ | ✓ | ✓ | ✓ | @@ -85,8 +84,10 @@ The following table depicts the various user permission levels in a project. | Create new merge request | | ✓ | ✓ | ✓ | ✓ | | View metrics dashboard annotations | | ✓ | ✓ | ✓ | ✓ | | Create/edit requirements **(ULTIMATE)** | | ✓ | ✓ | ✓ | ✓ | -| Pull [packages](packages/index.md) **(PREMIUM)** | | ✓ | ✓ | ✓ | ✓ | -| Publish [packages](packages/index.md) **(PREMIUM)**| | | ✓ | ✓ | ✓ | +| Pull [packages](packages/index.md) | | ✓ | ✓ | ✓ | ✓ | +| Publish [packages](packages/index.md) | | | ✓ | ✓ | ✓ | +| Delete [packages](packages/index.md) | | | | ✓ | ✓ | +| Create/edit/delete a Cleanup policy | | | ✓ | ✓ | ✓ | | Upload [Design Management](project/issues/design_management.md) files | | | ✓ | ✓ | ✓ | | Create/edit/delete [Releases](project/releases/index.md)| | | ✓ | ✓ | ✓ | | Create new branches | | | ✓ | ✓ | ✓ | @@ -128,7 +129,7 @@ The following table depicts the various user permission levels in a project. | Push to protected branches | | | | ✓ | ✓ | | Turn on/off protected branch push for devs | | | | ✓ | ✓ | | Enable/disable tag protections | | | | ✓ | ✓ | -| Edit project | | | | ✓ | ✓ | +| Edit project settings | | | | ✓ | ✓ | | Edit project badges | | | | ✓ | ✓ | | Share (invite) projects with groups | | | | ✓ (*8*) | ✓ (*8*)| | Add deploy keys to project | | | | ✓ | ✓ | @@ -141,7 +142,7 @@ The following table depicts the various user permission levels in a project. | Remove GitLab Pages | | | | ✓ | ✓ | | Manage clusters | | | | ✓ | ✓ | | Manage Project Operations | | | | ✓ | ✓ | -| View Pods logs | | | | ✓ | ✓ | +| View Pods logs | | | ✓ | ✓ | ✓ | | Read Terraform state | | | ✓ | ✓ | ✓ | | Manage Terraform state | | | | ✓ | ✓ | | Manage license policy **(ULTIMATE)** | | | | ✓ | ✓ | @@ -155,7 +156,7 @@ The following table depicts the various user permission levels in a project. | Transfer project to another namespace | | | | | ✓ | | Rename project | | | | | ✓ | | Remove fork relationship | | | | | ✓ | -| Remove project | | | | | ✓ | +| Delete project | | | | | ✓ | | Archive project | | | | | ✓ | | Delete issues | | | | | ✓ | | Delete pipelines | | | | | ✓ | @@ -166,7 +167,8 @@ The following table depicts the various user permission levels in a project. | View CI\CD analytics | | ✓ | ✓ | ✓ | ✓ | | View Code Review analytics **(STARTER)** | | ✓ | ✓ | ✓ | ✓ | | View Insights **(ULTIMATE)** | ✓ | ✓ | ✓ | ✓ | ✓ | -| View Issues analytics **(PREMIUM)** | ✓ | ✓ | ✓ | ✓ | ✓ | +| View Issue analytics **(PREMIUM)** | ✓ | ✓ | ✓ | ✓ | ✓ | +| View Merge Request analytics **(STARTER)** | ✓ | ✓ | ✓ | ✓ | ✓ | | View Repository analytics | | ✓ | ✓ | ✓ | ✓ | | View Value Stream analytics | ✓ | ✓ | ✓ | ✓ | ✓ | @@ -191,7 +193,7 @@ Project features like wiki and issues can be hidden from users depending on which visibility level you select on project settings. - Disabled: disabled for everyone -- Only team members: only team members will see even if your project is public or internal +- Only team members: only team members can see even if your project is public or internal - Everyone with access: everyone can see depending on your project visibility level - Everyone: enabled for everyone (only available for GitLab Pages) @@ -245,8 +247,8 @@ group. | Create/edit group epic **(ULTIMATE)** | | ✓ | ✓ | ✓ | ✓ | | Manage group labels | | ✓ | ✓ | ✓ | ✓ | | See a container registry | | ✓ | ✓ | ✓ | ✓ | -| Pull [packages](packages/index.md) **(PREMIUM)** | | ✓ | ✓ | ✓ | ✓ | -| Publish [packages](packages/index.md) **(PREMIUM)** | | | ✓ | ✓ | ✓ | +| Pull [packages](packages/index.md) | | ✓ | ✓ | ✓ | ✓ | +| Publish [packages](packages/index.md) | | | ✓ | ✓ | ✓ | | View metrics dashboard annotations | | ✓ | ✓ | ✓ | ✓ | | Create project in group | | | ✓ (3) | ✓ (3) | ✓ (3) | | Share (invite) groups with groups | | | | | ✓ | @@ -270,7 +272,7 @@ group. | Disable notification emails | | | | | ✓ | | View Contribution analytics | ✓ | ✓ | ✓ | ✓ | ✓ | | View Insights **(ULTIMATE)** | ✓ | ✓ | ✓ | ✓ | ✓ | -| View Issues analytics **(PREMIUM)** | ✓ | ✓ | ✓ | ✓ | ✓ | +| View Issue analytics **(PREMIUM)** | ✓ | ✓ | ✓ | ✓ | ✓ | | View Productivity analytics **(PREMIUM)** | | ✓ | ✓ | ✓ | ✓ | | View Value Stream analytics | ✓ | ✓ | ✓ | ✓ | ✓ | | View Billing **(FREE ONLY)** | | | | | ✓ (4) | @@ -308,10 +310,10 @@ External users: logged out). Access can be granted by adding the user as member to the project or group. -Like usual users, they will receive a role in the project or group with all +Like usual users, they receive a role in the project or group with all the abilities that are mentioned in the [permissions table above](#project-members-permissions). For example, if an external user is added as Guest, and your project is -private, they will not have access to the code; you would need to grant the external +private, they do not have access to the code; you need to grant the external user access at the Reporter level or above if you want them to have access to the code. You should always take into account the [project's visibility and permissions settings](project/settings/index.md#sharing-and-permissions) @@ -324,7 +326,7 @@ An administrator can flag a user as external by either of the following methods: - Either [through the API](../api/users.md#user-modification). - Or by navigating to the **Admin Area > Overview > Users** to create a new user - or edit an existing one. There, you will find the option to flag the user as + or edit an existing one. There, you can find the option to flag the user as external. ### Setting new users to external @@ -332,14 +334,14 @@ An administrator can flag a user as external by either of the following methods: By default, new users are not set as external users. This behavior can be changed by an administrator on the **Admin Area > Settings > General** page, under **Account and limit**. -If you change the default behavior of creating new users as external, you will +If you change the default behavior of creating new users as external, you have the option to narrow it down by defining a set of internal users. The **Internal users** field allows specifying an email address regex pattern to identify default internal users. New users whose email address matches the regex -pattern will be set to internal by default rather than an external collaborator. +pattern are set to internal by default rather than an external collaborator. The regex pattern format is Ruby, but it needs to be convertible to JavaScript, -and the ignore case flag will be set (`/regex pattern/i`). Here are some examples: +and the ignore case flag is set (`/regex pattern/i`). Here are some examples: - Use `\.internal@domain\.com$` to mark email addresses ending with `.internal@domain.com` as internal. @@ -354,21 +356,21 @@ Be aware that this regex could lead to a When a user is given Guest permissions on a project, group, or both, and holds no higher permission level on any other project or group on the GitLab instance, -the user is considered a guest user by GitLab and will not consume a license seat. +the user is considered a guest user by GitLab and does not consume a license seat. There is no other specific "guest" designation for newly created users. -If the user is assigned a higher role on any projects or groups, the user will -take a license seat. If a user creates a project, the user becomes a Maintainer +If the user is assigned a higher role on any projects or groups, the user +takes a license seat. If a user creates a project, the user becomes a Maintainer on the project, resulting in the use of a license seat. Also, note that if your -project is internal or private, Guest users will have all the abilities that are +project is internal or private, Guest users have all the abilities that are mentioned in the [permissions table above](#project-members-permissions) (they -will not be able to browse the project's repository for example). +are unable to browse the project's repository, for example). TIP: **Tip:** To prevent a guest user from creating projects, as an admin, you can edit the user's profile to mark the user as [external](#external-users-core-only). Beware though that even if a user is external, if they already have Reporter or -higher permissions in any project or group, they will **not** be counted as a +higher permissions in any project or group, they are **not** counted as a free guest user. ## Auditor users **(PREMIUM ONLY)** @@ -415,7 +417,7 @@ instance and project. In addition, all admins can use the admin interface under | See commits and jobs | ✓ | ✓ | ✓ | ✓ | | Retry or cancel job | | ✓ | ✓ | ✓ | | Erase job artifacts and trace | | ✓ (*1*) | ✓ | ✓ | -| Remove project | | | ✓ | ✓ | +| Delete project | | | ✓ | ✓ | | Create project | | | ✓ | ✓ | | Change project configuration | | | ✓ | ✓ | | Add specific runners | | | ✓ | ✓ | @@ -432,7 +434,7 @@ instance and project. In addition, all admins can use the admin interface under NOTE: **Note:** In GitLab 11.0, the Master role was renamed to Maintainer. ->**Note:** +NOTE: **Note:** GitLab 8.12 has a completely redesigned job permissions system. Read all about the [new model and its implications](project/new_ci_build_permissions_model.md). @@ -473,7 +475,7 @@ for details about the pipelines security model. ## LDAP users permissions Since GitLab 8.15, LDAP user permissions can now be manually overridden by an admin user. -Read through the documentation on [LDAP users permissions](group/index.md#manage-group-memberships-via-ldap) to learn more. +Read through the documentation on [LDAP users permissions](group/index.md#manage-group-memberships-via-ldap-starter-only) to learn more. ## Project aliases diff --git a/doc/user/profile/account/two_factor_authentication.md b/doc/user/profile/account/two_factor_authentication.md index 4f769f9a671..0e645e1b4a3 100644 --- a/doc/user/profile/account/two_factor_authentication.md +++ b/doc/user/profile/account/two_factor_authentication.md @@ -93,7 +93,7 @@ To set up 2FA with a U2F device: 1. Go to your [**Profile settings**](../index.md#profile-settings). 1. Go to **Account**. 1. Click **Enable Two-Factor Authentication**. -1. Plug in your U2F device. +1. Connect your U2F device. 1. Click on **Set up New U2F Device**. 1. A light will start blinking on your device. Activate it by pressing its button. @@ -109,9 +109,9 @@ CAUTION: **Caution:** Each code can be used only once to log in to your account. Immediately after successfully enabling two-factor authentication, you'll be -prompted to download a set of set recovery codes. Should you ever lose access -to your one time password authenticator, you can use one of them to log in to -your account. We suggest copying them, printing them, or downloading them using +prompted to download a set of generated recovery codes. Should you ever lose access +to your one-time password authenticator, you can use one of these recovery codes to log in to +your account. We suggest copying and printing them, or downloading them using the **Download codes** button for storage in a safe place. If you choose to download them, the file will be called `gitlab-recovery-codes.txt`. diff --git a/doc/user/profile/index.md b/doc/user/profile/index.md index 7a871afd861..b6ef6d7fdb7 100644 --- a/doc/user/profile/index.md +++ b/doc/user/profile/index.md @@ -22,7 +22,7 @@ See the [authentication topic](../../topics/authentication/index.md) for more de ### Unknown sign-in -GitLab will notify you if a sign-in occurs that is from an unknown IP address or device. +GitLab notifies you if a sign-in occurs that is from an unknown IP address or device. See [Unknown Sign-In Notification](unknown_sign_in_notification.md) for more details. ## User profile @@ -32,7 +32,7 @@ To access your profile: 1. Click on your avatar. 1. Select **Profile**. -On your profile page, you will see the following information: +On your profile page, you can see the following information: - Personal information - Activity stream: see your activity streamline and the history of your contributions @@ -85,7 +85,7 @@ If you don't know your current password, select the 'I forgot my password' link. Your `username` is a unique [`namespace`](../group/index.md#namespaces) related to your user ID. Changing it can have unintended side effects, read -[how redirects will behave](../project/index.md#redirects-when-changing-repository-paths) +[how redirects behave](../project/index.md#redirects-when-changing-repository-paths) before proceeding. To change your `username`: @@ -109,7 +109,7 @@ which also covers the case where you have projects hosted with ## Private profile -The following information will be hidden from the user profile page (`https://gitlab.example.com/username`) if this feature is enabled: +The following information is hidden from the user profile page (`https://gitlab.example.com/username`) if this feature is enabled: - Atom feed - Date when account is created @@ -152,7 +152,7 @@ To add links to other accounts: > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/14078) in GitLab 11.3. -Enabling private contributions will include contributions to private projects, in the user contribution calendar graph and user recent activity. +Enabling private contributions includes contributions to private projects, in the user contribution calendar graph and user recent activity. To enable private contributions: @@ -225,7 +225,7 @@ To enable this option: 1. Select **Use a private email** option. 1. Click **Update profile settings**. -Once this option is enabled, every Git-related action will be performed using the private commit email. +Once this option is enabled, every Git-related action is performed using the private commit email. To stay fully anonymous, you can also copy this private commit email and configure it on your local machine using the following command: @@ -253,7 +253,7 @@ When the `_gitlab_session` expires or isn't available, GitLab uses the `remember to get you a new `_gitlab_session` and keep you signed in through browser restarts. After your `remember_user_token` expires and your `_gitlab_session` is cleared/expired, -you will be asked to sign in again to verify your identity for security reasons. +you are asked to sign in again to verify your identity for security reasons. ### Increased sign-in time diff --git a/doc/user/profile/notifications.md b/doc/user/profile/notifications.md index dbf486e399e..336c1b8f254 100644 --- a/doc/user/profile/notifications.md +++ b/doc/user/profile/notifications.md @@ -187,7 +187,7 @@ To minimize the number of notifications that do not require any action, from [Gi | Remove milestone merge request | Subscribers, participants mentioned, and Custom notification level with this event selected | | New comment | The above, plus anyone mentioned by `@username` in the comment, with notification level "Mention" or higher | | Failed pipeline | The author of the pipeline | -| Fixed pipeline | The author of the pipeline. Enabled by default. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/24309) in GitLab 13.1. Administrators can disable this notification option using the `ci_pipeline_fixed_notifications` [feature flag](../../administration/feature_flags.md). | +| Fixed pipeline | The author of the pipeline. Enabled by default. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/24309) in GitLab 13.1. | | Successful pipeline | The author of the pipeline, if they have the custom notification setting for successful pipelines set. If the pipeline failed previously, a `Fixed pipeline` message will be sent for the first successful pipeline after the failure, then a `Successful pipeline` message for any further successful pipelines. | | New epic **(ULTIMATE)** | | | Close epic **(ULTIMATE)** | | diff --git a/doc/user/profile/personal_access_tokens.md b/doc/user/profile/personal_access_tokens.md index 59ca124f566..ae73842dd98 100644 --- a/doc/user/profile/personal_access_tokens.md +++ b/doc/user/profile/personal_access_tokens.md @@ -9,6 +9,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w > - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/3749) in GitLab 8.8. > - [Notifications about expiring tokens](https://gitlab.com/gitlab-org/gitlab/-/issues/3649) added in GitLab 12.6. +> - [Notifications about expired tokens](https://gitlab.com/gitlab-org/gitlab/-/issues/214721) added in GitLab 13.3. > - [Token lifetime limits](https://gitlab.com/gitlab-org/gitlab/-/issues/3649) added in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.6. If you're unable to use [OAuth2](../../api/oauth2.md), you can use a personal access token to authenticate with the [GitLab API](../../api/README.md#personalproject-access-tokens). @@ -17,7 +18,9 @@ You can also use personal access tokens with Git to authenticate over HTTP or SS Personal access tokens expire on the date you define, at midnight UTC. -- GitLab runs a check at 01:00 AM UTC every day to identify personal access tokens that will expire in under seven days. The owners of these tokens are notified by email. +- GitLab runs a check at 01:00 AM UTC every day to identify personal access tokens that expire in under seven days. The owners of these tokens are notified by email. +- GitLab runs a check at 02:00 AM UTC every day to identify personal access tokens that expired on the current date. The owners of these tokens are notified by email. +To turn on the notification for expired personal access tokens in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enable-or-disable-notification-for-expired-personal-access-token-core-only). **(CORE ONLY)** - In GitLab Ultimate, administrators may [limit the lifetime of personal access tokens](../admin_area/settings/account_and_limit_settings.md#limiting-lifetime-of-personal-access-tokens-ultimate-only). - In GitLab Ultimate, administrators may [toggle enforcement of personal access token expiry](../admin_area/settings/account_and_limit_settings.md#optional-enforcement-of-personal-access-token-expiry-ultimate-only). @@ -25,6 +28,23 @@ For examples of how you can use a personal access token to authenticate with the GitLab also offers [impersonation tokens](../../api/README.md#impersonation-tokens) which are created by administrators via the API. They're a great fit for automated authentication as a specific user. +## Enable or disable notification for Expired personal access token **(CORE ONLY)** + +[GitLab administrators with access to the GitLab Rails console](../../administration/feature_flags.md) +can enable it for your instance. + +To enable it: + +```ruby +Feature.enable(:expired_pat_email_notification) +``` + +To disable it: + +```ruby +Feature.disable(:expired_pat_email_notification) +``` + ## Creating a personal access token You can create as many personal access tokens as you like from your GitLab @@ -36,8 +56,8 @@ profile. 1. Choose a name and optional expiry date for the token. 1. Choose the [desired scopes](#limiting-scopes-of-a-personal-access-token). 1. Click the **Create personal access token** button. -1. Save the personal access token somewhere safe. Once you leave or refresh - the page, you won't be able to access it again. +1. Save the personal access token somewhere safe. If you navigate away or refresh +your page, and you did not save the token, you must create a new one. ### Revoking a personal access token @@ -46,7 +66,7 @@ respective **Revoke** button under the **Active Personal Access Token** area. ### Token activity -You can see when a token was last used from the **Personal Access Tokens** page. Updates to the token usage is fixed at once per 24 hours. Requests to [API resources](../../api/api_resources.md) and the [GraphQL API](../../api/graphql/index.md) will update a token's usage. +You can see when a token was last used from the **Personal Access Tokens** page. Updates to the token usage is fixed at once per 24 hours. Requests to [API resources](../../api/api_resources.md) and the [GraphQL API](../../api/graphql/index.md) update a token's usage. ## Limiting scopes of a personal access token @@ -60,14 +80,15 @@ the following table. | `api` | [GitLab 8.15](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/5951) | Grants complete read/write access to the API, including all groups and projects, the container registry, and the package registry. | | `read_api` | [GitLab 12.10](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/28944) | Grants read access to the API, including all groups and projects, the container registry, and the package registry. | | `read_registry` | [GitLab 9.3](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/11845) | Allows to read (pull) [container registry](../packages/container_registry/index.md) images if a project is private and authorization is required. | -| `sudo` | [GitLab 10.2](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/14838) | Allows performing API actions as any user in the system (if the authenticated user is an admin). | +| `write_registry` | [GitLab 12.10](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/28958) | Allows to write (push) [container registry](../packages/container_registry/index.md) images if a project is private and authorization is required. | +| `sudo` | [GitLab 10.2](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/14838) | Allows performing API actions as any user in the system (if the authenticated user is an administrator). | | `read_repository` | [GitLab 10.7](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/17894) | Allows read-only access (pull) to the repository through `git clone`. | | `write_repository` | [GitLab 11.11](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/26021) | Allows read-write access (pull, push) to the repository through `git clone`. Required for accessing Git repositories over HTTP when 2FA is enabled. | ## Programmatically creating a personal access token You can programmatically create a predetermined personal access token for use in -automation or tests. You will need sufficient access to run a +automation or tests. You need sufficient access to run a [Rails console session](../../administration/troubleshooting/debug.md#starting-a-rails-console-session) for your GitLab instance. @@ -89,15 +110,15 @@ sudo gitlab-rails runner "token = User.find_by_username('automation-bot').person ``` NOTE: **Note:** -The token string must be 20 characters in length, or it will not be -recognized as a personal access token. +The token string must be 20 characters in length to be +recognized as a valid personal access token. The list of valid scopes and what they do can be found [in the source code](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/auth.rb). ## Programmatically revoking a personal access token -You can programmatically revoke a personal access token. You will need +You can programmatically revoke a personal access token. You need sufficient access to run a [Rails console session](../../administration/troubleshooting/debug.md#starting-a-rails-console-session) for your GitLab instance. diff --git a/doc/user/project/autocomplete_characters.md b/doc/user/project/autocomplete_characters.md index 9ebf7f821a1..1acdc56de54 100644 --- a/doc/user/project/autocomplete_characters.md +++ b/doc/user/project/autocomplete_characters.md @@ -1,3 +1,11 @@ +--- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference +description: "Autocomplete chars in Markdown fields." +--- + # Autocomplete characters The autocomplete characters provide a quick way of entering field values into diff --git a/doc/user/project/badges.md b/doc/user/project/badges.md index 542a3763008..ed6e2460554 100644 --- a/doc/user/project/badges.md +++ b/doc/user/project/badges.md @@ -1,3 +1,10 @@ +--- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference, howto +--- + # Badges > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/41174) in GitLab 10.7. @@ -29,6 +36,20 @@ clicking on the trash icon. Badges associated with a group can only be edited or deleted on the [group level](#group-badges). +### Example project badge: Pipeline Status + +A common project badge presents the GitLab CI pipeline status. + +To add this badge to a project: + +1. Navigate to your project's **Settings > General > Badges**. +1. Under **Name**, enter _Pipeline Status_. +1. Under **Link**, enter the following URL: + `https://gitlab.com/%{project_path}/-/commits/%{default_branch}` +1. Under **Badge image URL**, enter the following URL: + `https://gitlab.com/%{project_path}/badges/%{default_branch}/pipeline.svg` +1. Submit the badge by clicking the **Add badge** button. + ## Group badges Badges can be added to a group and will then be visible on every project's diff --git a/doc/user/project/clusters/add_eks_clusters.md b/doc/user/project/clusters/add_eks_clusters.md index b11483a7446..d5713f20257 100644 --- a/doc/user/project/clusters/add_eks_clusters.md +++ b/doc/user/project/clusters/add_eks_clusters.md @@ -56,12 +56,17 @@ Generate an access key for the IAM user, and configure GitLab with the credentia To create and add a new Kubernetes cluster to your project, group, or instance: 1. Navigate to your: - - Project's **{cloud-gear}** **Operations > Kubernetes** page, for a project-level cluster. - - Group's **{cloud-gear}** **Kubernetes** page, for a group-level cluster. - - **{admin}** **Admin Area >** **{cloud-gear}** **Kubernetes**, for an instance-level cluster. + - Project's **Operations > Kubernetes** page, for a project-level cluster. + - Group's **Kubernetes** page, for a group-level cluster. + - **Admin Area > Kubernetes**, for an instance-level cluster. 1. Click **Add Kubernetes cluster**. 1. Under the **Create new cluster** tab, click **Amazon EKS**. You will be provided with an `Account ID` and `External ID` to use in the next step. +1. In the [IAM Management Console](https://console.aws.amazon.com/iam/home), create an EKS management IAM role. + To do so, follow the [Amazon EKS cluster IAM role](https://docs.aws.amazon.com/eks/latest/userguide/service_IAM_role.html) instructions + to create a IAM role suitable for managing the AWS EKS cluster's resources on your behalf. + In addition to the policies that guide suggests, you must also include the `AmazonEKSClusterPolicy` + policy for this role in order for GitLab to manage the EKS cluster correctly. 1. In the [IAM Management Console](https://console.aws.amazon.com/iam/home), create an IAM role: 1. From the left panel, select **Roles**. 1. Click **Create role**. @@ -135,11 +140,17 @@ To create and add a new Kubernetes cluster to your project, group, or instance: 1. Click **Authenticate with AWS**. 1. Choose your cluster's settings: - **Kubernetes cluster name** - The name you wish to give the cluster. - - **Environment scope** - The [associated environment](index.md#setting-the-environment-scope-premium) to this cluster. + - **Environment scope** - The [associated environment](index.md#setting-the-environment-scope) to this cluster. - **Kubernetes version** - The Kubernetes version to use. Currently the only version supported is 1.14. - - **Role name** - Select the [IAM role](https://docs.aws.amazon.com/eks/latest/userguide/service_IAM_role.html) - to allow Amazon EKS and the Kubernetes control plane to manage AWS resources on your behalf. This IAM role is separate - to the IAM role created above, you will need to create it if it does not yet exist. + - **Service role** - Select the **EKS IAM role** you created earlier to allow Amazon EKS + and the Kubernetes control plane to manage AWS resources on your behalf. + + NOTE: **Note:** + This IAM role is _not_ the IAM role you created in the previous step. It should be + the one you created much earlier by following the + [Amazon EKS cluster IAM role](https://docs.aws.amazon.com/eks/latest/userguide/service_IAM_role.html) + guide. + - **Region** - The [region](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html) in which the cluster will be created. - **Key pair name** - Select the [key pair](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-key-pairs.html) @@ -194,10 +205,10 @@ If the `Cluster` resource failed with the error the role specified in **Role name** is not configured correctly. NOTE: **Note:** -This role should not be the same as the one created above. If you don't have an -existing -[EKS cluster IAM role](https://docs.aws.amazon.com/eks/latest/userguide/service_IAM_role.html), -you must create one. +This role should be the role you created by following the +[EKS cluster IAM role](https://docs.aws.amazon.com/eks/latest/userguide/service_IAM_role.html) guide. +In addition to the policies that guide suggests, you must also include the +`AmazonEKSClusterPolicy` policy for this role in order for GitLab to manage the EKS cluster correctly. ## Existing EKS cluster diff --git a/doc/user/project/clusters/add_gke_clusters.md b/doc/user/project/clusters/add_gke_clusters.md index 2746076befe..720f9bdf253 100644 --- a/doc/user/project/clusters/add_gke_clusters.md +++ b/doc/user/project/clusters/add_gke_clusters.md @@ -48,14 +48,14 @@ To create and add a new Kubernetes cluster to your project, group, or instance: 1. Navigate to your: - Project's **{cloud-gear}** **Operations > Kubernetes** page, for a project-level cluster. - Group's **{cloud-gear}** **Kubernetes** page, for a group-level cluster. - - **{admin}** **Admin Area >** **{cloud-gear}** **Kubernetes** page, for an instance-level cluster. + - **Admin Area >** **{cloud-gear}** **Kubernetes** page, for an instance-level cluster. 1. Click **Add Kubernetes cluster**. 1. Under the **Create new cluster** tab, click **Google GKE**. 1. Connect your Google account if you haven't done already by clicking the **Sign in with Google** button. 1. Choose your cluster's settings: - **Kubernetes cluster name** - The name you wish to give the cluster. - - **Environment scope** - The [associated environment](index.md#setting-the-environment-scope-premium) to this cluster. + - **Environment scope** - The [associated environment](index.md#setting-the-environment-scope) to this cluster. - **Google Cloud Platform project** - Choose the project you created in your GCP console that will host the Kubernetes cluster. Learn more about [Google Cloud Platform projects](https://cloud.google.com/resource-manager/docs/creating-managing-projects). diff --git a/doc/user/project/clusters/add_remove_clusters.md b/doc/user/project/clusters/add_remove_clusters.md index 65f1c59f4ca..e4a750084c9 100644 --- a/doc/user/project/clusters/add_remove_clusters.md +++ b/doc/user/project/clusters/add_remove_clusters.md @@ -142,7 +142,7 @@ Amazon Elastic Kubernetes Service (EKS) at the project, group, or instance level 1. Navigate to your: - Project's **{cloud-gear}** **Operations > Kubernetes** page, for a project-level cluster. - Group's **{cloud-gear}** **Kubernetes** page, for a group-level cluster. - - **{admin}** **Admin Area >** **{cloud-gear}** **Kubernetes** page, for an instance-level cluster. + - **Admin Area >** **{cloud-gear}** **Kubernetes** page, for an instance-level cluster. 1. Click **Add Kubernetes cluster**. 1. Click the **Create new cluster** tab. 1. Click either **Amazon EKS** or **Google GKE**, and follow the instructions for your desired service: @@ -164,12 +164,12 @@ To add a Kubernetes cluster to your project, group, or instance: 1. Navigate to your: 1. Project's **{cloud-gear}** **Operations > Kubernetes** page, for a project-level cluster. 1. Group's **{cloud-gear}** **Kubernetes** page, for a group-level cluster. - 1. **{admin}** **Admin Area >** **{cloud-gear}** **Kubernetes** page, for an instance-level cluster. + 1. **Admin Area >** **{cloud-gear}** **Kubernetes** page, for an instance-level cluster. 1. Click **Add Kubernetes cluster**. 1. Click the **Add existing cluster** tab and fill in the details: 1. **Kubernetes cluster name** (required) - The name you wish to give the cluster. 1. **Environment scope** (required) - The - [associated environment](index.md#setting-the-environment-scope-premium) to this cluster. + [associated environment](index.md#setting-the-environment-scope) to this cluster. 1. **API URL** (required) - It's the URL that GitLab uses to access the Kubernetes API. Kubernetes exposes several APIs, we want the "base" URL that is common to all of them. @@ -331,7 +331,7 @@ a new cluster or added an existing one. To disable Kubernetes cluster integratio 1. Navigate to your: - Project's **{cloud-gear}** **Operations > Kubernetes** page, for a project-level cluster. - Group's **{cloud-gear}** **Kubernetes** page, for a group-level cluster. - - **{admin}** **Admin Area >** **{cloud-gear}** **Kubernetes** page, for an instance-level cluster. + - **Admin Area >** **{cloud-gear}** **Kubernetes** page, for an instance-level cluster. 1. Click on the name of the cluster. 1. Click the **GitLab Integration** toggle. 1. Click **Save changes**. diff --git a/doc/user/project/clusters/img/sidebar_menu_pod_logs_v12_10.png b/doc/user/project/clusters/img/sidebar_menu_pod_logs_v12_10.png Binary files differdeleted file mode 100644 index ee37970d867..00000000000 --- a/doc/user/project/clusters/img/sidebar_menu_pod_logs_v12_10.png +++ /dev/null diff --git a/doc/user/project/clusters/index.md b/doc/user/project/clusters/index.md index ddcfd376d89..98078854050 100644 --- a/doc/user/project/clusters/index.md +++ b/doc/user/project/clusters/index.md @@ -67,17 +67,17 @@ to: ### Multiple Kubernetes clusters > - Introduced in [GitLab Premium](https://about.gitlab.com/pricing/) 10.3 -> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/35094) to GitLab core in 13.2. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/35094) to GitLab Core in 13.2. You can associate more than one Kubernetes cluster to your project. That way you can have different clusters for different environments, like dev, staging, production, and so on. Simply add another cluster, like you did the first time, and make sure to -[set an environment scope](#setting-the-environment-scope-premium) that will +[set an environment scope](#setting-the-environment-scope) that will differentiate the new cluster with the rest. -#### Setting the environment scope **(PREMIUM)** +#### Setting the environment scope When adding more than one Kubernetes cluster to your project, you need to differentiate them with an environment scope. The environment scope associates clusters with [environments](../../../ci/environments/index.md) similar to how the @@ -368,7 +368,7 @@ Automatically detect and monitor Kubernetes metrics. Automatic monitoring of ### Visualizing cluster health > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/4701) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 10.6. -> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/208224) to GitLab core in 13.2. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/208224) to GitLab Core in 13.2. When [Prometheus is deployed](#installing-applications), GitLab will automatically monitor the cluster's health. At the top of the cluster settings page, CPU and Memory utilization is displayed, along with the total amount available. Keeping an eye on cluster resources can be important, if the cluster runs out of memory pods may be shutdown or fail to start. diff --git a/doc/user/project/clusters/kubernetes_pod_logs.md b/doc/user/project/clusters/kubernetes_pod_logs.md index ee642dc18cf..afb6d016f45 100644 --- a/doc/user/project/clusters/kubernetes_pod_logs.md +++ b/doc/user/project/clusters/kubernetes_pod_logs.md @@ -1,6 +1,6 @@ --- -stage: Configure -group: Configure +stage: Monitor +group: APM info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- @@ -9,56 +9,54 @@ info: To determine the technical writer assigned to the Stage/Group associated w > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/4752) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.0. > - [Moved](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/26383) to [GitLab Core](https://about.gitlab.com/pricing/) 12.9. -GitLab makes it easy to view the logs of running pods in [connected Kubernetes clusters](index.md). -By displaying the logs directly in GitLab in the **Log Explorer**, developers can avoid -managing console tools or jumping to a different interface. - -NOTE: **Note:** -[Learn more about Kubernetes + GitLab](https://about.gitlab.com/solutions/kubernetes/). -Everything you need to build, test, deploy, and run your application at scale. - -## Overview - -[Kubernetes](https://kubernetes.io) logs can be viewed directly within GitLab with -the **Log Explorer**. +GitLab makes it easy to view the logs of running pods or managed applications in +[connected Kubernetes clusters](index.md). By displaying the logs directly in GitLab +in the **Log Explorer**, developers can avoid managing console tools or jumping +to a different interface. The **Log Explorer** interface provides a set of filters +above the log file data, depending on your configuration:  +- **Namespace** - Select the environment to display. Users with Maintainer or + greater [permissions](../../permissions.md) can also select Managed Apps. +- **Search** - Only available if the Elastic Stack managed application is installed. +- **Select time range** - Select the range of time to display. Only available if the + Elastic Stack managed application is installed. +- **Scroll to bottom** **{scroll_down}** - Scroll to the end of the displayed logs. +- **Refresh** **{retry}** - Reload the displayed logs. + <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> -To learn more, see [APM - Log Explorer](https://www.youtube.com/watch?v=hWclZHA7Dgw). +To learn more about the Log Explorer, see [APM - Log Explorer](https://www.youtube.com/watch?v=hWclZHA7Dgw). + +NOTE: **Note:** +[Learn more about Kubernetes + GitLab](https://about.gitlab.com/solutions/kubernetes/). +Everything you need to build, test, deploy, and run your application at scale. ## Requirements [Deploying to a Kubernetes environment](../deploy_boards.md#enabling-deploy-boards) is required to use Logs. -## Usage - -To access logs, you must have the right [permissions](../../permissions.md#project-members-permissions). - -You can access them in two ways. - -### From the project sidebar +## Accessing the log explorer -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/22011) in GitLab 12.5. +To access the **Log explorer**, click the **More actions** **{ellipsis_v}** menu on +a [metrics dashboard](../../../operations/metrics/index.md) and select **View logs**, or: -Go to **{cloud-gear}** **Operations > Pod logs** on the sidebar menu to display -the **Log Explorer**. +1. Sign in as a user with the _View pod logs_ + [permissions](../../permissions.md#project-members-permissions) in the project. +1. *To navigate to the **Log Explorer** from the sidebar menu,* go to + **{cloud-gear}** **Operations > Pod logs**. + ([Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/22011) in GitLab 12.5.) +1. *To navigate to the **Log Explorer** from a specific pod on a [Deploy Board](../deploy_boards.md):* - - -### From Deploy Boards - -Logs can be displayed by clicking on a specific pod from [Deploy Boards](../deploy_boards.md): - -1. Go to **{cloud-gear}** **Operations > Environments** and find the environment - which contains the desired pod, like `production`. -1. On the **Environments** page, you should see the status of the environment's - pods with [Deploy Boards](../deploy_boards.md). -1. When mousing over the list of pods, a tooltip will appear with the exact pod name - and status. -  -1. Click on the desired pod to display the **Log Explorer**. + 1. Go to **{cloud-gear}** **Operations > Environments** and find the environment + which contains the desired pod, like `production`. + 1. On the **Environments** page, you should see the status of the environment's + pods with [Deploy Boards](../deploy_boards.md). + 1. When mousing over the list of pods, GitLab displays a tooltip with the exact pod name + and status. +  + 1. Click on the desired pod to display the **Log Explorer**. ### Logs view @@ -69,6 +67,7 @@ The **Log Explorer** lets you filter the logs by: - [From GitLab 12.7](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/21656), [full text search](#full-text-search). - [From GitLab 12.8](https://gitlab.com/gitlab-org/gitlab/-/issues/197879), dates. +- [From GitLab 13.2](https://gitlab.com/gitlab-org/gitlab/-/issues/208790), managed apps. Loading more than 500 log lines is possible from [GitLab 12.9](https://gitlab.com/gitlab-org/gitlab/-/issues/198050) onward. @@ -93,17 +92,16 @@ Click **Show last** in the **Log Explorer** to see the available options. > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/21656) in GitLab 12.7. When you enable [Elastic Stack](../../clusters/applications.md#elastic-stack) on your cluster, -you can search the content of your logs through a search bar. - -The search is passed on to Elasticsearch using the +you can search the content of your logs through a search bar. The search is passed +to Elasticsearch using the [simple_query_string](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-simple-query-string-query.html) Elasticsearch function, which supports the following operators: -| Operator | Description | -|----------------------------|------------------------------------------------------------| -| `\|` | An OR operation. | +| Operator | Description | +|----------------------------|-------------------------------------------------------------| +| `\|` | An `OR` operation. | | `-` | Negates a single token. | -| `+` | An AND operation. | +| `+` | An `AND` operation. | | `"` | Wraps a number of tokens to signify a phrase for searching. | | `*` (at the end of a term) | A prefix query. | | `(` and `)` | Precedence. | diff --git a/doc/user/project/clusters/runbooks/index.md b/doc/user/project/clusters/runbooks/index.md index a592d59f964..360b02efb69 100644 --- a/doc/user/project/clusters/runbooks/index.md +++ b/doc/user/project/clusters/runbooks/index.md @@ -129,7 +129,7 @@ the components outlined above and the pre-loaded demo runbook. %env DB_NAME={project.variables.get('DB_NAME').value} ``` - 1. Navigate to **{settings}** **Settings >> CI/CD >> Variables** to create + 1. Navigate to **Settings > CI/CD > Variables** to create the variables in your project.  diff --git a/doc/user/project/clusters/securing.md b/doc/user/project/clusters/securing.md index b4c20cb8dbc..5b9f776080b 100644 --- a/doc/user/project/clusters/securing.md +++ b/doc/user/project/clusters/securing.md @@ -9,7 +9,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w GitLab makes it easy to secure applications deployed in [connected Kubernetes clusters](index.md). You can benefit from the protection of a [Web Application Firewall](../../../topics/web_application_firewall/quick_start_guide.md), [Network Policies](../../../topics/autodevops/stages.md#network-policy), -or even [Container Host Security](../../clusters/applications.md#install-falco-using-gitlab-cicd). +and [Container Host Security](../../clusters/applications.md#install-falco-using-gitlab-cicd). This page contains full end-to-end steps and instructions to connect your cluster to GitLab and install these features, whether or not your applications are deployed through GitLab CI/CD. If you @@ -25,7 +25,7 @@ At a high level, the required steps include the following: - Connect the cluster to GitLab. - Set up one or more runners. - Set up a cluster management project. -- Install a Web Application Firewall, Network Policies, and/or Container Host +- Install a Web Application Firewall, and/or Network Policies, and/or Container Host Security. - Install Prometheus to get statistics and metrics in the [threat monitoring](../../application_security/threat_monitoring/) @@ -40,6 +40,10 @@ Minimum requirements (depending on the GitLab Manage Application you want to ins ### Understanding how GitLab Managed Apps are installed +NOTE: **Note:** +These diagrams use the term _Kubernetes_ for simplicity. In practice, Sidekiq connects to a Helm +Tiller daemon running in a pod in the cluster. + You install GitLab Managed Apps from the GitLab web interface with a one-click setup process. GitLab uses Sidekiq (a background processing service) to facilitate this. @@ -52,12 +56,8 @@ uses Sidekiq (a background processing service) to facilitate this. Sidekiq-->>-GitLab: Refresh UI ``` -NOTE: **Note:** -This diagram uses the term _Kubernetes_ for simplicity. In practice, Sidekiq connects to a Helm -Tiller daemon running in a pod in the cluster. - Although this installation method is easier because it's a point-and-click action in the user -interface, it's inflexible and hard to debug. When something goes wrong, you can't see the +interface, it's inflexible and harder to debug. If something goes wrong, you can't see the deployment logs. The Web Application Firewall feature uses this installation method. However, the next generation of GitLab Managed Apps V2 ([CI/CD-based GitLab Managed Apps](https://gitlab.com/groups/gitlab-org/-/epics/2103)) @@ -75,10 +75,10 @@ sequenceDiagram ``` Debugging is easier because you have access to the raw logs of these jobs (the Helm Tiller output is -available as an artifact in case of failure) and the flexibility is much better. Since these +available as an artifact in case of failure), and the flexibility is much better. Since these deployments are only triggered when a pipeline is running (most likely when there's a new commit in the cluster management repository), every action has a paper trail and follows the classic merge -request workflow (approvals, merge, deploy). The Network Policy (Cilium) Managed App and Container +request workflow (approvals, merge, deploy). The Network Policy (Cilium) Managed App, and Container Host Security (Falco) are deployed with this model. ## Connect the cluster to GitLab @@ -151,4 +151,5 @@ falco: installed: true ``` -[Read more] about configuring Container Host Security. +[Read more](../../clusters/applications.md#install-falco-using-gitlab-cicd) +about configuring Container Host Security. diff --git a/doc/user/project/clusters/serverless/aws.md b/doc/user/project/clusters/serverless/aws.md index 595d8fb3895..543ffdbce8f 100644 --- a/doc/user/project/clusters/serverless/aws.md +++ b/doc/user/project/clusters/serverless/aws.md @@ -373,7 +373,7 @@ variables. To set these: -1. Navigate to the project's **{settings}** **Settings > CI / CD**. +1. Navigate to the project's **Settings > CI / CD**. 1. Expand the **Variables** section and create entries for `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`. 1. Mask the credentials so they do not show in logs using the **Masked** toggle. diff --git a/doc/user/project/code_intelligence.md b/doc/user/project/code_intelligence.md index e2c2cae3158..be34053cdc7 100644 --- a/doc/user/project/code_intelligence.md +++ b/doc/user/project/code_intelligence.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" type: reference --- @@ -10,7 +13,7 @@ Code Intelligence adds code navigation features common to interactive development environments (IDE), including: - Type signatures and symbol documentation. -- Go-to definition +- Go-to definition. Code Intelligence is built into GitLab and powered by [LSIF](https://lsif.dev/) (Language Server Index Format), a file format for precomputed code @@ -39,6 +42,36 @@ After the job succeeds, code intelligence data can be viewed while browsing the  +## Find references + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/217392) in GitLab 13.2. +> - [Became enabled by default](https://gitlab.com/gitlab-org/gitlab/-/issues/225621) on GitLab 13.3. +> - It's enabled on GitLab.com. + +To find where a particular object is being used, you can see links to specific lines of code +under the **References** tab: + + + +### Enable or disable find references + +Find references is under development but ready for production use. +It is deployed behind a feature flag that is **enabled by default**. +[GitLab administrators with access to the GitLab Rails console](../../administration/feature_flags.md) +can opt to disable it for your instance. + +To disable it: + +```ruby +Feature.disable(:code_navigation_references) +``` + +To enable it: + +```ruby +Feature.enable(:code_navigation_references) +``` + ## Language support Generating an LSIF file requires a language server indexer implementation for the diff --git a/doc/user/project/code_owners.md b/doc/user/project/code_owners.md index b35d04dfdfc..dbe3f3dc891 100644 --- a/doc/user/project/code_owners.md +++ b/doc/user/project/code_owners.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" type: reference --- @@ -175,7 +178,7 @@ Owners" section: ```plaintext [README Owners] -README.md @user1 @user 2 +README.md @user1 @user2 internal/README.md @user2 ``` diff --git a/doc/user/project/deploy_boards.md b/doc/user/project/deploy_boards.md index 0a613ff918b..50fb24b555b 100644 --- a/doc/user/project/deploy_boards.md +++ b/doc/user/project/deploy_boards.md @@ -36,7 +36,7 @@ to the latest release. Since Deploy Boards are tightly coupled with Kubernetes, there is some required knowledge. In particular, you should be familiar with: -- [Kubernetes pods](https://kubernetes.io/docs/concepts/workloads/pods/pod/) +- [Kubernetes pods](https://kubernetes.io/docs/concepts/workloads/pods/) - [Kubernetes labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) - [Kubernetes namespaces](https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/) - [Kubernetes canary deployments](https://kubernetes.io/docs/concepts/cluster-administration/manage-deployment/#canary-deployments) diff --git a/doc/user/project/deploy_keys/img/deploy_keys_v13_0.png b/doc/user/project/deploy_keys/img/deploy_keys_v13_0.png Binary files differindex 462141ef82a..15e6e71803c 100644 --- a/doc/user/project/deploy_keys/img/deploy_keys_v13_0.png +++ b/doc/user/project/deploy_keys/img/deploy_keys_v13_0.png diff --git a/doc/user/project/deploy_keys/img/public_deploy_key_v13_0.png b/doc/user/project/deploy_keys/img/public_deploy_key_v13_0.png Binary files differindex 3e6d1605f95..2f708555af1 100644 --- a/doc/user/project/deploy_keys/img/public_deploy_key_v13_0.png +++ b/doc/user/project/deploy_keys/img/public_deploy_key_v13_0.png diff --git a/doc/user/project/deploy_keys/index.md b/doc/user/project/deploy_keys/index.md index 32d3eab5e62..81c9008c5b3 100644 --- a/doc/user/project/deploy_keys/index.md +++ b/doc/user/project/deploy_keys/index.md @@ -121,7 +121,7 @@ repositories to secure, shared services, such as CI/CD. Instance administrators can add public deploy keys: -1. Go to **Admin Area** (**{admin}**) **> Deploy Keys**. +1. Go to **Admin Area > Deploy Keys**. 1. Click on **New deploy key**. Make sure your new key has a meaningful title, as it is the primary way for project diff --git a/doc/user/project/deploy_tokens/index.md b/doc/user/project/deploy_tokens/index.md index b7daca567f4..5ca421dda5b 100644 --- a/doc/user/project/deploy_tokens/index.md +++ b/doc/user/project/deploy_tokens/index.md @@ -25,7 +25,7 @@ You can create as many deploy tokens as you like from the settings of your proje 1. Log in to your GitLab account. 1. Go to the project (or group) you want to create Deploy Tokens for. -1. Go to **{settings}** **Settings** > **Repository**. +1. Go to **Settings > Repository**. 1. Click on "Expand" on **Deploy Tokens** section. 1. Choose a name, expiry date (optional), and username (optional) for the token. 1. Choose the [desired scopes](#limiting-scopes-of-a-deploy-token). diff --git a/doc/user/project/file_lock.md b/doc/user/project/file_lock.md index 9069a231db4..ed80e5f6a32 100644 --- a/doc/user/project/file_lock.md +++ b/doc/user/project/file_lock.md @@ -1,3 +1,10 @@ +--- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference, howto +--- + # File Locking **(PREMIUM)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/440) in [GitLab Premium](https://about.gitlab.com/pricing/) 8.9. diff --git a/doc/user/project/git_attributes.md b/doc/user/project/git_attributes.md index 260e618ba03..577f0f1f754 100644 --- a/doc/user/project/git_attributes.md +++ b/doc/user/project/git_attributes.md @@ -1,3 +1,10 @@ +--- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference +--- + # Git Attributes GitLab supports defining custom [Git attributes](https://git-scm.com/docs/gitattributes) such as what diff --git a/doc/user/project/highlighting.md b/doc/user/project/highlighting.md index a2740294e62..5f7c754ec75 100644 --- a/doc/user/project/highlighting.md +++ b/doc/user/project/highlighting.md @@ -1,3 +1,10 @@ +--- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference +--- + # Syntax Highlighting GitLab provides syntax highlighting on all files through the [Rouge](https://rubygems.org/gems/rouge) Ruby gem. It will try to guess what language to use based on the file extension, which most of the time is sufficient. diff --git a/doc/user/project/img/code_intelligence_find_references_v13_3.png b/doc/user/project/img/code_intelligence_find_references_v13_3.png Binary files differnew file mode 100644 index 00000000000..415fe86cc75 --- /dev/null +++ b/doc/user/project/img/code_intelligence_find_references_v13_3.png diff --git a/doc/user/project/img/code_intelligence_v13_1.png b/doc/user/project/img/code_intelligence_v13_1.png Binary files differindex 0dff27bab43..744195caed2 100644 --- a/doc/user/project/img/code_intelligence_v13_1.png +++ b/doc/user/project/img/code_intelligence_v13_1.png diff --git a/doc/user/project/img/sectional_code_owners_v13.2.png b/doc/user/project/img/sectional_code_owners_v13.2.png Binary files differindex 894771c26af..58b331950f4 100644 --- a/doc/user/project/img/sectional_code_owners_v13.2.png +++ b/doc/user/project/img/sectional_code_owners_v13.2.png diff --git a/doc/user/project/img/service_desk_custom_email_address_v13_0.png b/doc/user/project/img/service_desk_custom_email_address_v13_0.png Binary files differindex 6ce8bf45085..3789e039904 100644 --- a/doc/user/project/img/service_desk_custom_email_address_v13_0.png +++ b/doc/user/project/img/service_desk_custom_email_address_v13_0.png diff --git a/doc/user/project/import/clearcase.md b/doc/user/project/import/clearcase.md index 173ba71b167..a825084dd1a 100644 --- a/doc/user/project/import/clearcase.md +++ b/doc/user/project/import/clearcase.md @@ -27,7 +27,7 @@ The following table illustrates the main differences between ClearCase and Git: | Server | UNIX, Windows legacy systems | UNIX, macOS | | License | Proprietary | GPL | -_Taken from the slides [ClearCase and the journey to Git](https://www.open.collab.net/media/pdfs/ClearCase-and-the-journey-to-Git.pdf) provided by collab.net_ +_Taken from the slides [ClearCase and the journey to Git](https://docplayer.net/42708453-Clearcase-the-journey-to-git-migrating-your-skills-and-vobs-to-git.html) provided by [collab.net](https://www.collab.net/)_ ## Why migrate @@ -47,7 +47,7 @@ While there doesn't exist a tool to fully migrate from ClearCase to Git, here are some useful links to get you started: - [Bridge for Git and ClearCase](https://github.com/charleso/git-cc) -- [Slides "ClearCase and the journey to Git"](https://www.open.collab.net/media/pdfs/ClearCase-and-the-journey-to-Git.pdf) +- [Slides "ClearCase and the journey to Git"](https://docplayer.net/42708453-Clearcase-the-journey-to-git-migrating-your-skills-and-vobs-to-git.html) - [ClearCase to Git](https://therub.org/2013/07/19/clearcase-to-git/) - [Dual syncing ClearCase to Git](https://therub.org/2013/10/22/dual-syncing-clearcase-and-git/) - [Moving to Git from ClearCase](https://sateeshkumarb.wordpress.com/2011/01/15/moving-to-git-from-clearcase/) diff --git a/doc/user/project/import/gitea.md b/doc/user/project/import/gitea.md index 543fffd33d6..81ab16590dc 100644 --- a/doc/user/project/import/gitea.md +++ b/doc/user/project/import/gitea.md @@ -11,7 +11,7 @@ Import your projects from Gitea to GitLab with minimal effort. ## Overview ->**Note:** +NOTE: **Note:** This requires Gitea `v1.0.0` or newer. - At its current state, Gitea importer can import: diff --git a/doc/user/project/import/github.md b/doc/user/project/import/github.md index b7754e60837..531b308111a 100644 --- a/doc/user/project/import/github.md +++ b/doc/user/project/import/github.md @@ -52,7 +52,7 @@ must meet one of the following conditions prior to the import: - Have previously logged in to a GitLab account using the GitHub icon. - Have a GitHub account with a - [primary email address](https://help.github.com/en/github/setting-up-and-managing-your-github-user-account/setting-your-commit-email-address) + [primary email address](https://docs.github.com/en/github/setting-up-and-managing-your-github-user-account/setting-your-commit-email-address) that matches their GitLab account's email address. If a user referenced in the project is not found in GitLab's database, the project creator (typically the user @@ -81,7 +81,7 @@ Before you begin, ensure that any GitHub users who you want to map to GitLab use - A GitLab account that has logged in using the GitHub icon \- or - -- A GitLab account with an email address that matches the [public email address](https://help.github.com/en/github/setting-up-and-managing-your-github-user-account/setting-your-commit-email-address) of the GitHub user +- A GitLab account with an email address that matches the [public email address](https://docs.github.com/en/github/setting-up-and-managing-your-github-user-account/setting-your-commit-email-address) of the GitHub user User-matching attempts occur in that order, and if a user is not identified either way, the activity is associated with the user account that is performing the import. diff --git a/doc/user/project/import/gitlab_com.md b/doc/user/project/import/gitlab_com.md index b9aea629e85..6c622ece4ac 100644 --- a/doc/user/project/import/gitlab_com.md +++ b/doc/user/project/import/gitlab_com.md @@ -13,7 +13,7 @@ mind that it is possible only if GitLab.com integration is enabled on your GitLa To get to the importer page you need to go to "New project" page. ->**Note:** +NOTE: **Note:** If you are interested in importing Wiki and Merge Request data to your new instance, you'll need to follow the instructions for [exporting a project](../settings/import_export.md#exporting-a-project-and-its-data) diff --git a/doc/user/project/import/img/jira/import_issues_from_jira_form_v12_10.png b/doc/user/project/import/img/jira/import_issues_from_jira_form_v12_10.png Binary files differindex d98ad2aaa6e..317a426394c 100644 --- a/doc/user/project/import/img/jira/import_issues_from_jira_form_v12_10.png +++ b/doc/user/project/import/img/jira/import_issues_from_jira_form_v12_10.png diff --git a/doc/user/project/import/img/jira/import_issues_from_jira_form_v13_2.png b/doc/user/project/import/img/jira/import_issues_from_jira_form_v13_2.png Binary files differindex 9cbffe2bb36..8a94d33d47b 100644 --- a/doc/user/project/import/img/jira/import_issues_from_jira_form_v13_2.png +++ b/doc/user/project/import/img/jira/import_issues_from_jira_form_v13_2.png diff --git a/doc/user/project/import/img/manifest_status.png b/doc/user/project/import/img/manifest_status.png Binary files differdeleted file mode 100644 index b706116a2ac..00000000000 --- a/doc/user/project/import/img/manifest_status.png +++ /dev/null diff --git a/doc/user/project/import/img/manifest_status_v13_3.png b/doc/user/project/import/img/manifest_status_v13_3.png Binary files differnew file mode 100644 index 00000000000..3f0063e6715 --- /dev/null +++ b/doc/user/project/import/img/manifest_status_v13_3.png diff --git a/doc/user/project/import/jira.md b/doc/user/project/import/jira.md index 482e632baa2..7f179865f4f 100644 --- a/doc/user/project/import/jira.md +++ b/doc/user/project/import/jira.md @@ -77,10 +77,9 @@ Importing large projects may take several minutes depending on the size of the i In the **Jira-GitLab user mapping template** section, the table shows to which GitLab users your Jira users will be mapped. - If it wasn't possible to match a Jira user with a GitLab user, the dropdown defaults to the user - conducting the import. + When the form appears, the dropdown defaults to the user conducting the import. -1. To change any of the suggested mappings, click the dropdown in the **GitLab username** column and +1. To change any of the mappings, click the dropdown in the **GitLab username** column and select the user you want to map to each Jira user. The dropdown may not show all the users, so use the search bar to find a specific diff --git a/doc/user/project/import/manifest.md b/doc/user/project/import/manifest.md index 0374e0acf9a..60524f3cc69 100644 --- a/doc/user/project/import/manifest.md +++ b/doc/user/project/import/manifest.md @@ -62,4 +62,4 @@ You can start the import with: to the import status page with projects list based on the manifest file. 1. Check the list and click **Import all repositories** to start the import. -  +  diff --git a/doc/user/project/import/tfvc.md b/doc/user/project/import/tfvc.md index 1c9ee7476bd..faa2a9b4927 100644 --- a/doc/user/project/import/tfvc.md +++ b/doc/user/project/import/tfvc.md @@ -41,6 +41,14 @@ Advantages of migrating to Git/GitLab: ## How to migrate -The best option to migrate from TFVC to Git is to use the [`git-tfs`](https://github.com/git-tfs/git-tfs) -tool. Read the [Migrate TFS to Git](https://github.com/git-tfs/git-tfs/blob/master/doc/usecases/migrate_tfs_to_git.md) -guide for more details. +Migration options from TFVC to Git depend on your operating system. + +- If you're migrating on Microsoft Windows: + + Use the [`git-tfs`](https://github.com/git-tfs/git-tfs) +tool. + Read the [Migrate TFS to Git](https://github.com/git-tfs/git-tfs/blob/master/doc/usecases/migrate_tfs_to_git.md) guide for details. + +- If you're on a Unix-based system: + + Follow the procedures described with this [TFVC to Git migration tool](https://github.com/turbo/gtfotfs). diff --git a/doc/user/project/index.md b/doc/user/project/index.md index 1e71674c16f..4e5b924a1b7 100644 --- a/doc/user/project/index.md +++ b/doc/user/project/index.md @@ -1,3 +1,10 @@ +--- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference +--- + # Projects In GitLab, you can create projects for hosting @@ -96,9 +103,9 @@ When you create a project in GitLab, you'll have access to a large number of - [Releases](releases/index.md): a way to track deliverables in your project as snapshot in time of the source, build output, other metadata, and other artifacts associated with a released version of your code. -- [Conan packages](../packages/conan_repository/index.md): your private Conan repository in GitLab. **(PREMIUM)** -- [Maven packages](../packages/maven_repository/index.md): your private Maven repository in GitLab. **(PREMIUM)** -- [NPM packages](../packages/npm_registry/index.md): your private NPM package registry in GitLab. **(PREMIUM)** +- [Conan packages](../packages/conan_repository/index.md): your private Conan repository in GitLab. +- [Maven packages](../packages/maven_repository/index.md): your private Maven repository in GitLab. +- [NPM packages](../packages/npm_registry/index.md): your private NPM package registry in GitLab. - [Code owners](code_owners.md): specify code owners for certain files **(STARTER)** - [License Compliance](../compliance/license_compliance/index.md): approve and deny licenses for projects. **(ULTIMATE)** - [Dependency List](../application_security/dependency_list/index.md): view project dependencies. **(ULTIMATE)** @@ -173,22 +180,22 @@ Read through the documentation on [project settings](settings/index.md). - [Export a project from GitLab](settings/import_export.md#exporting-a-project-and-its-data) - [Importing and exporting projects between GitLab instances](settings/import_export.md) -## Remove a project +## Delete a project -To remove a project, first navigate to the home page for that project. +To delete a project, first navigate to the home page for that project. 1. Navigate to **Settings > General**. 1. Expand the **Advanced** section. -1. Scroll down to the **Remove project** section. -1. Click **Remove project** +1. Scroll down to the **Delete project** section. +1. Click **Delete project** 1. Confirm this action by typing in the expected text. -### Delayed removal **(PREMIUM)** +### Delayed deletion **(PREMIUM)** -By default, clicking to remove a project is followed by a seven day delay. Admins can restore the project during this period of time. -This delay [may be changed by an admin](../admin_area/settings/visibility_and_access_controls.md#default-deletion-adjourned-period-premium-only). +By default, clicking to delete a project is followed by a seven day delay. Admins can restore the project during this period of time. +This delay [may be changed by an admin](../admin_area/settings/visibility_and_access_controls.md#default-deletion-delay-premium-only). -Admins can view all projects pending deletion. If you're an administrator, go to the top navigation bar, click **Projects > Your projects**, and then select the **Removed projects** tab. +Admins can view all projects pending deletion. If you're an administrator, go to the top navigation bar, click **Projects > Your projects**, and then select the **Deleted projects** tab. From this tab an admin can restore any project. ## CI/CD for external repositories **(PREMIUM)** diff --git a/doc/user/project/integrations/bamboo.md b/doc/user/project/integrations/bamboo.md index 7b21c590c8a..9cade323ed2 100644 --- a/doc/user/project/integrations/bamboo.md +++ b/doc/user/project/integrations/bamboo.md @@ -1,3 +1,9 @@ +--- +stage: Create +group: Ecosystem +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Atlassian Bamboo CI Service GitLab provides integration with Atlassian Bamboo for continuous integration. diff --git a/doc/user/project/integrations/bugzilla.md b/doc/user/project/integrations/bugzilla.md index 6d44c56743e..2ed14a4c69c 100644 --- a/doc/user/project/integrations/bugzilla.md +++ b/doc/user/project/integrations/bugzilla.md @@ -1,3 +1,9 @@ +--- +stage: Create +group: Ecosystem +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Bugzilla Service Navigate to the [Integrations page](overview.md#accessing-integrations), diff --git a/doc/user/project/integrations/custom_issue_tracker.md b/doc/user/project/integrations/custom_issue_tracker.md index 7d15ae82b6f..1329f584fdc 100644 --- a/doc/user/project/integrations/custom_issue_tracker.md +++ b/doc/user/project/integrations/custom_issue_tracker.md @@ -1,8 +1,14 @@ +--- +stage: Create +group: Ecosystem +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Custom Issue Tracker service To enable the Custom Issue Tracker integration in a project: -1. Go to **{settings}** **Settings > Integrations**. +1. Go to **Settings > Integrations**. 1. Click **Custom Issue Tracker** 1. Fill in the tracker's details, such as title, description, and URLs. You will be able to edit these fields later as well. diff --git a/doc/user/project/integrations/discord_notifications.md b/doc/user/project/integrations/discord_notifications.md index aa45cc38cb5..f261362eeae 100644 --- a/doc/user/project/integrations/discord_notifications.md +++ b/doc/user/project/integrations/discord_notifications.md @@ -1,3 +1,9 @@ +--- +stage: Create +group: Ecosystem +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Discord Notifications service > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/22684) in GitLab 11.6. diff --git a/doc/user/project/integrations/emails_on_push.md b/doc/user/project/integrations/emails_on_push.md index b0838690d3b..d8b864e0396 100644 --- a/doc/user/project/integrations/emails_on_push.md +++ b/doc/user/project/integrations/emails_on_push.md @@ -1,3 +1,9 @@ +--- +stage: Create +group: Ecosystem +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Enabling emails on push By enabling this service, you will receive email notifications for every change diff --git a/doc/user/project/integrations/generic_alerts.md b/doc/user/project/integrations/generic_alerts.md index 3490a3f2b9e..dc6aa40ea82 100644 --- a/doc/user/project/integrations/generic_alerts.md +++ b/doc/user/project/integrations/generic_alerts.md @@ -29,8 +29,8 @@ To obtain credentials for setting up a generic alerts integration: - Sign in to GitLab as a user with maintainer [permissions](../../permissions.md) for a project. - Navigate to the **Operations** page for your project, depending on your installed version of GitLab: - - *In GitLab versions 13.1 and greater,* navigate to **{settings}** **Settings > Operations** in your project. - - *In GitLab versions prior to 13.1,* navigate to **{settings}** **Settings > Integrations** in your project. GitLab will display a banner encouraging you to enable the Alerts endpoint in **{settings}** **Settings > Operations** instead. + - *In GitLab versions 13.1 and greater,* navigate to **Settings > Operations** in your project. + - *In GitLab versions prior to 13.1,* navigate to **Settings > Integrations** in your project. GitLab will display a banner encouraging you to enable the Alerts endpoint in **Settings > Operations** instead. - Click **Alerts endpoint**. - Toggle the **Active** alert setting to display the **URL** and **Authorization Key** for the webhook configuration. @@ -101,7 +101,7 @@ After a [project maintainer or owner](#setting-up-generic-alerts) test alert to confirm your integration works properly. 1. Sign in as a user with Developer or greater [permissions](../../../user/permissions.md). -1. Navigate to **{settings}** **Settings > Operations** in your project. +1. Navigate to **Settings > Operations** in your project. 1. Click **Alerts endpoint** to expand the section. 1. Enter a sample payload in **Alert test payload** (valid JSON is required). 1. Click **Test alert payload**. @@ -116,7 +116,7 @@ In GitLab versions 13.2 and greater, GitLab groups alerts based on their payload When an incoming alert contains the same payload as another alert (excluding the `start_time` and `hosts` attributes), GitLab groups these alerts together and displays a counter on the -[Alert Management List](../operations/alert_management.md#alert-management-list) +[Alert Management List](../../../operations/incident_management/incidents.md) and details pages. If the existing alert is already `resolved`, then a new alert will be created instead. diff --git a/doc/user/project/integrations/github.md b/doc/user/project/integrations/github.md index 416996fb629..29818e862e0 100644 --- a/doc/user/project/integrations/github.md +++ b/doc/user/project/integrations/github.md @@ -1,3 +1,9 @@ +--- +stage: Create +group: Ecosystem +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # GitHub project integration **(PREMIUM)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/3836) in GitLab Premium 10.6. @@ -14,7 +20,7 @@ and is automatically configured on [GitHub import](../../../integration/github.m ### Complete these steps on GitHub -This integration requires a [GitHub API token](https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token) +This integration requires a [GitHub API token](https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token) with `repo:status` access granted: 1. Go to your "Personal access tokens" page at <https://github.com/settings/tokens> diff --git a/doc/user/project/integrations/gitlab_slack_application.md b/doc/user/project/integrations/gitlab_slack_application.md index 7a827364d41..62fccb22d36 100644 --- a/doc/user/project/integrations/gitlab_slack_application.md +++ b/doc/user/project/integrations/gitlab_slack_application.md @@ -1,3 +1,9 @@ +--- +stage: Create +group: Ecosystem +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # GitLab Slack application **(FREE ONLY)** > - Introduced in GitLab 9.4. @@ -36,7 +42,7 @@ docs on [Adding an app to your workspace](https://slack.com/help/articles/202035 To enable GitLab's service for your Slack team: -1. Go to your project's **{settings}** **Settings > Integration > Slack application** (only +1. Go to your project's **Settings > Integration > Slack application** (only visible on GitLab.com). 1. Click **Add to Slack**. @@ -47,7 +53,7 @@ That's all! You can now start using the Slack slash commands. To create a project alias on GitLab.com for Slack integration: 1. Go to your project's home page. -1. Navigate to **{settings}** **Settings > Integrations** (only visible on GitLab.com) +1. Navigate to **Settings > Integrations** (only visible on GitLab.com) 1. On the **Integrations** page, click **Slack application**. 1. The current **Project Alias**, if any, is displayed. To edit this value, click **Edit**. diff --git a/doc/user/project/integrations/hangouts_chat.md b/doc/user/project/integrations/hangouts_chat.md index f65b31150a9..54f9bd8d622 100644 --- a/doc/user/project/integrations/hangouts_chat.md +++ b/doc/user/project/integrations/hangouts_chat.md @@ -1,3 +1,9 @@ +--- +stage: Create +group: Ecosystem +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Hangouts Chat service > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/43756) in GitLab 11.2. diff --git a/doc/user/project/integrations/hipchat.md b/doc/user/project/integrations/hipchat.md index 2ed7f13db9b..718f00273bd 100644 --- a/doc/user/project/integrations/hipchat.md +++ b/doc/user/project/integrations/hipchat.md @@ -1,3 +1,9 @@ +--- +stage: Create +group: Ecosystem +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Atlassian HipChat GitLab provides a way to send HipChat notifications upon a number of events, diff --git a/doc/user/project/integrations/img/actions_menu_create_new_dashboard_v13_2.png b/doc/user/project/integrations/img/actions_menu_create_new_dashboard_v13_2.png Binary files differdeleted file mode 100644 index 5d530a80421..00000000000 --- a/doc/user/project/integrations/img/actions_menu_create_new_dashboard_v13_2.png +++ /dev/null diff --git a/doc/user/project/integrations/img/metrics_settings_button_v13_2.png b/doc/user/project/integrations/img/metrics_settings_button_v13_2.png Binary files differdeleted file mode 100644 index d649f77eded..00000000000 --- a/doc/user/project/integrations/img/metrics_settings_button_v13_2.png +++ /dev/null diff --git a/doc/user/project/integrations/img/panel_context_menu_v13_0.png b/doc/user/project/integrations/img/panel_context_menu_v13_0.png Binary files differdeleted file mode 100644 index 2d7cb923981..00000000000 --- a/doc/user/project/integrations/img/panel_context_menu_v13_0.png +++ /dev/null diff --git a/doc/user/project/integrations/img/project_integrations_v13_3.png b/doc/user/project/integrations/img/project_integrations_v13_3.png Binary files differnew file mode 100644 index 00000000000..9c925d32441 --- /dev/null +++ b/doc/user/project/integrations/img/project_integrations_v13_3.png diff --git a/doc/user/project/integrations/img/project_services.png b/doc/user/project/integrations/img/project_services.png Binary files differdeleted file mode 100644 index 5fed38a349c..00000000000 --- a/doc/user/project/integrations/img/project_services.png +++ /dev/null diff --git a/doc/user/project/integrations/img/prometheus_add_metric.png b/doc/user/project/integrations/img/prometheus_add_metric.png Binary files differdeleted file mode 100644 index 9afeb535123..00000000000 --- a/doc/user/project/integrations/img/prometheus_add_metric.png +++ /dev/null diff --git a/doc/user/project/integrations/img/prometheus_alert.png b/doc/user/project/integrations/img/prometheus_alert.png Binary files differdeleted file mode 100644 index ffa1008ff51..00000000000 --- a/doc/user/project/integrations/img/prometheus_alert.png +++ /dev/null diff --git a/doc/user/project/integrations/img/prometheus_dashboard_environments_v12_8.png b/doc/user/project/integrations/img/prometheus_dashboard_environments_v12_8.png Binary files differdeleted file mode 100644 index 467deb86881..00000000000 --- a/doc/user/project/integrations/img/prometheus_dashboard_environments_v12_8.png +++ /dev/null diff --git a/doc/user/project/integrations/img/prometheus_dashboard_select_v_13_0.png b/doc/user/project/integrations/img/prometheus_dashboard_select_v_13_0.png Binary files differdeleted file mode 100644 index 2f0309ce664..00000000000 --- a/doc/user/project/integrations/img/prometheus_dashboard_select_v_13_0.png +++ /dev/null diff --git a/doc/user/project/integrations/img/prometheus_monitoring_dashboard_v13_1.png b/doc/user/project/integrations/img/prometheus_monitoring_dashboard_v13_1.png Binary files differdeleted file mode 100644 index 56a0a508a1d..00000000000 --- a/doc/user/project/integrations/img/prometheus_monitoring_dashboard_v13_1.png +++ /dev/null diff --git a/doc/user/project/integrations/img/webex_teams_configuration.png b/doc/user/project/integrations/img/webex_teams_configuration.png Binary files differindex 66993e0887d..493b3ea50a0 100644 --- a/doc/user/project/integrations/img/webex_teams_configuration.png +++ b/doc/user/project/integrations/img/webex_teams_configuration.png diff --git a/doc/user/project/integrations/index.md b/doc/user/project/integrations/index.md index 75565dd2750..0a1db5da61d 100644 --- a/doc/user/project/integrations/index.md +++ b/doc/user/project/integrations/index.md @@ -1,3 +1,9 @@ +--- +stage: Create +group: Ecosystem +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Project integrations You can find the available integrations under your project's diff --git a/doc/user/project/integrations/irker.md b/doc/user/project/integrations/irker.md index 2d807d4302b..f2e769dcfc0 100644 --- a/doc/user/project/integrations/irker.md +++ b/doc/user/project/integrations/irker.md @@ -1,3 +1,9 @@ +--- +stage: Create +group: Ecosystem +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Irker IRC Gateway GitLab provides a way to push update messages to an Irker server. When diff --git a/doc/user/project/integrations/jira.md b/doc/user/project/integrations/jira.md index 541c65041ad..f11cd4d9539 100644 --- a/doc/user/project/integrations/jira.md +++ b/doc/user/project/integrations/jira.md @@ -1,36 +1,39 @@ +--- +stage: Create +group: Ecosystem +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # GitLab Jira integration -GitLab Issues are a powerful tool for discussing ideas and planning and tracking work. -However, many organizations have been using Jira for these purposes and have -extensive data and business processes built into it. +If you need to use Jira to track work that's implemented in GitLab, GitLab's Jira integrations make the process of working across systems more efficent. + +This page is about the GitLab Jira integration, which is available in every GitLab project by default, allowing you to connect it to any Jira instance, whether Cloud or self-managed. To compare features with the complementary Jira Development Panel integration, see [Jira integrations](jira_integrations.md). -While you can always migrate content and process from Jira to GitLab Issues, -you can also opt to continue using Jira and use it together with GitLab through -our integration. +After you set up this integration, you can cross-reference activity in the GitLab project with any of your projects in Jira. This includes the ability to close or transition Jira issues when work is completed in GitLab. -For a video demonstration of integration with Jira, watch [GitLab workflow with Jira issues and Jenkins pipelines](https://youtu.be/Jn-_fyra7xQ). +Features include: -Once you integrate your GitLab project with your Jira instance, you can automatically -detect and cross-reference activity between the GitLab project and any of your projects -in Jira. This includes the ability to close or transition Jira issues when the work -is completed in GitLab. +- **Mention a Jira issue ID** in a commit message or MR (merge request) and + - GitLab links to the Jira issue. + - The Jira issue adds a comment with details and a link back to the activity in GitLab. +- **Mention that a commit or MR resolves or closes a specific Jira issue** and when it's merged to the default branch: + - GitLab's MR displays a note that it closed the Jira issue. Prior to the merge, MRs indicate which issue they will close. + - The Jira issue shows the activity and is closed or otherwise transitioned as specified in your GitLab settings. +- **View a list of Jira issues directly in GitLab** **(PREMIUM)** -Here's how the integration responds when you take the following actions in GitLab: +For additional features, you can install the [Jira Development Panel integration](../../../integration/jira_development_panel.md). This enables you to: -- **Mention a Jira issue ID** in a commit message or MR (merge request). - - GitLab hyperlinks to the Jira issue. - - The Jira issue adds an issue link to the commit/MR in GitLab. - - The Jira issue adds a comment reflecting the comment made in GitLab, the comment author, and a link to the commit/MR in GitLab, unless this commenting to Jira is [disabled](#disabling-comments-on-jira-issues). -- **Mention that a commit or MR 'closes', 'resolves', or 'fixes' a Jira issue ID**. When the commit is made on the project's default branch (usually master) or the change is merged to the default branch: - - GitLab's merge request page displays a note that it "Closed" the Jira issue, with a link to the issue. (Note: Before the merge, an MR will display that it "Closes" the Jira issue.) - - The Jira issue shows the activity and the Jira issue is closed, or otherwise transitioned. +- In a Jira issue, display relevant GitLab information in the [development panel](https://support.atlassian.com/jira-software-cloud/docs/view-development-information-for-an-issue/), including related branches, commits, and merge requests. +- Use Jira [Smart Commits](https://confluence.atlassian.com/fisheye/using-smart-commits-960155400.html) in GitLab to add Jira comments, log time spent on the issue, or apply any issue transition. -You can also use [Jira's Smart Commits](https://confluence.atlassian.com/fisheye/using-smart-commits-960155400.html) -directly from GitLab, as covered in the article -[How and why to integrate GitLab with Jira](https://www.programmableweb.com/news/how-and-why-to-integrate-gitlab-jira/how-to/2017/04/25). +See the [feature comparison](jira_integrations.md#feature-comparison) for more details. ## Configuration +<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> +For an overview, see [Agile Management - GitLab-Jira Basic Integration](https://www.youtube.com/watch?v=fWvwkx5_00E&feature=youtu.be). + Each GitLab project can be configured to connect to an entire Jira instance. That means one GitLab project can interact with _all_ Jira projects in that instance, once configured. Therefore, you will not have to explicitly associate @@ -68,7 +71,7 @@ Select **Enable integration**. Select a **Trigger** action. This determines whether a mention of a Jira issue in GitLab commits, merge requests, or both, should link the Jira issue back to that source commit/MR and transition the Jira issue, if indicated. -To include a comment on the Jira issue when the above referene is made in GitLab, check **Enable comments**. +To include a comment on the Jira issue when the above reference is made in GitLab, check **Enable comments**. Enter the further details on the page as described in the following table. @@ -80,7 +83,9 @@ Enter the further details on the page as described in the following table. | `Password/API token` |Created in [configuring Jira](#configuring-jira) step. Use `password` for **Jira Server** or `API token` for **Jira Cloud**. | | `Transition ID` | Required for closing Jira issues via commits or merge requests. This is the ID of a transition in Jira that moves issues to a desired state. (See [Obtaining a transition ID](#obtaining-a-transition-id).) If you insert multiple transition IDs separated by `,` or `;`, the issue is moved to each state, one after another, using the given order. | -To enable users to view Jira issues inside GitLab, select **Enable Jira issues** and enter a project key. **(PREMIUM)** +To enable users to view Jira issues inside the GitLab project, select **Enable Jira issues** and enter a Jira project key. **(PREMIUM)** + +You can only display issues from a single Jira project within a given GitLab project. CAUTION: **Caution:** If you enable Jira issues with the setting above, all users that have access to this GitLab project will be able to view all issues from the specified Jira project. diff --git a/doc/user/project/integrations/jira_cloud_configuration.md b/doc/user/project/integrations/jira_cloud_configuration.md index c7157b6bd0e..14999734c00 100644 --- a/doc/user/project/integrations/jira_cloud_configuration.md +++ b/doc/user/project/integrations/jira_cloud_configuration.md @@ -1,3 +1,9 @@ +--- +stage: Create +group: Ecosystem +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Creating an API token in Jira Cloud An API token is needed when integrating with Jira Cloud, follow the steps diff --git a/doc/user/project/integrations/jira_integrations.md b/doc/user/project/integrations/jira_integrations.md new file mode 100644 index 00000000000..90cd9bf3acb --- /dev/null +++ b/doc/user/project/integrations/jira_integrations.md @@ -0,0 +1,35 @@ +--- +stage: Create +group: Ecosystem +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Jira integrations + +## Introduction + +GitLab Issues are a tool for discussing ideas and planning and tracking work. However, your organization may already use Jira for these purposes, with +extensive, established data and business processes they rely on. + +Although you can [migrate](../../../user/project/import/jira.md) your Jira issues and work exclusively in GitLab, you also have the option of continuing to use Jira by using GitLab's Jira integrations. + +## Integrations + +The following Jira integrations allow different types of cross-referencing between GitLab activity and Jira issues, with additional features: + +- [**Jira integration**](jira.md) - This is built in to GitLab. In a given GitLab project, it can be configured to connect to any Jira instance, self-managed or Cloud. +- [**Jira development panel integration**](../../../integration/jira_development_panel.md) **(PREMIUM)** - This connects all GitLab projects under a specified group or personal namespace. + - If you're using Jira Cloud and GitLab.com, install the [GitLab for Jira](https://marketplace.atlassian.com/apps/1221011/gitlab-for-jira) app in the Atlassian Marketplace and see its [documentation](../../../integration/jira_development_panel.md#gitlab-for-jira-app). + - For all other environments, use the [Jira DVCS Connector configuration instructions](../../../integration/jira_development_panel.md#configuration). + +### Feature comparison + +| Capability | Jira integration | Jira Development Panel integration | +|-----------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------| +| Mention of Jira issue ID in GitLab is automatically linked to that issue | Yes | No | +| Mention of Jira issue ID in GitLab issue/MR is reflected in the Jira issue | Yes, as a Jira comment with the GitLab issue/MR title and a link back to it. Its first mention also adds the GitLab page to the Jira issue under “Web links”. | Yes, in the issue’s Development panel | +| Mention of Jira issue ID in GitLab commit message is reflected in the issue | Yes. The entire commit message is added to the Jira issue as a comment and under “Web links”, each with a link back to the commit in GitLab. | Yes, in the issue’s Development panel and optionally with a custom comment on the Jira issue using Jira Smart Commits. | +| Mention of Jira issue ID in GitLab branch names is reflected in Jira issue | No | Yes, in the issue’s Development panel | +| Record Jira time tracking info against an issue | No | Yes. Time can be specified via Jira Smart Commits. | +| Transition or close a Jira issue with a Git commit or merge request | Yes. Only a single transition type, typically configured to close the issue by setting it to Done. | Yes. Transition to any state using Jira Smart Commits. | +| Display a list of Jira issues | Yes **(PREMIUM)** | No | diff --git a/doc/user/project/integrations/jira_server_configuration.md b/doc/user/project/integrations/jira_server_configuration.md index c8278a0f083..38098d7d15b 100644 --- a/doc/user/project/integrations/jira_server_configuration.md +++ b/doc/user/project/integrations/jira_server_configuration.md @@ -1,3 +1,9 @@ +--- +stage: Create +group: Ecosystem +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Creating a username and password for Jira Server We need to create a user in Jira which will have access to all projects that diff --git a/doc/user/project/integrations/mattermost.md b/doc/user/project/integrations/mattermost.md index 67d60984c22..c12a969ca3c 100644 --- a/doc/user/project/integrations/mattermost.md +++ b/doc/user/project/integrations/mattermost.md @@ -1,3 +1,9 @@ +--- +stage: Create +group: Ecosystem +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Mattermost Notifications Service The Mattermost Notifications Service allows your GitLab project to send events (e.g., `issue created`) to your existing Mattermost team as notifications. This requires configurations in both Mattermost and GitLab. diff --git a/doc/user/project/integrations/mattermost_slash_commands.md b/doc/user/project/integrations/mattermost_slash_commands.md index 6a202c9a130..5e08767d3f0 100644 --- a/doc/user/project/integrations/mattermost_slash_commands.md +++ b/doc/user/project/integrations/mattermost_slash_commands.md @@ -1,3 +1,9 @@ +--- +stage: Create +group: Ecosystem +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Mattermost slash commands > Introduced in GitLab 8.14 @@ -42,9 +48,9 @@ the administrator console.  -1. Click **Custom integrations** and set **Enable Custom Slash Commands**, - **Enable custom integrations to override usernames**, and **Override - custom integrations to override profile picture icons** to true +1. Click **Integration Management** and set **Enable Custom Slash Commands**, + **Enable integrations to override usernames**, and **Enable + integrations to override profile picture icons** to true  diff --git a/doc/user/project/integrations/microsoft_teams.md b/doc/user/project/integrations/microsoft_teams.md index 611ae1a01af..b2a2f1c3e7b 100644 --- a/doc/user/project/integrations/microsoft_teams.md +++ b/doc/user/project/integrations/microsoft_teams.md @@ -1,3 +1,9 @@ +--- +stage: Create +group: Ecosystem +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Microsoft Teams service ## On Microsoft Teams diff --git a/doc/user/project/integrations/mock_ci.md b/doc/user/project/integrations/mock_ci.md index b06ccda8287..4567d345336 100644 --- a/doc/user/project/integrations/mock_ci.md +++ b/doc/user/project/integrations/mock_ci.md @@ -1,3 +1,9 @@ +--- +stage: Create +group: Ecosystem +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Mock CI Service **NB: This service is only listed if you are in a development environment!** diff --git a/doc/user/project/integrations/overview.md b/doc/user/project/integrations/overview.md index 79c55e2d140..f179cd6b98e 100644 --- a/doc/user/project/integrations/overview.md +++ b/doc/user/project/integrations/overview.md @@ -1,3 +1,9 @@ +--- +stage: Create +group: Ecosystem +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Integrations Integrations allow you to integrate GitLab with other applications. They @@ -12,7 +18,7 @@ You can find the available integrations under your project's There are more than 20 integrations to integrate with. Click on the one that you want to configure. - + ## Integrations listing @@ -69,10 +75,18 @@ The number of branches or tags supported can be changed via ## Service templates -Service templates are a way to set predefined values for an integration across +Service templates are a way to set predefined values for a project integration across all new projects on the instance. -Read more about [Service templates in this document](services_templates.md). +Read more about [Service templates](services_templates.md). + +## Project integration management + +Project integraton management lets you control integration settings across all projects +of an instance. On the project level, administrators you can choose whether to inherit the +instance configuraton or provide custom settings. + +Read more about [Project integration management](../../admin_area/settings/project_integration_management.md). ## Troubleshooting integrations diff --git a/doc/user/project/integrations/prometheus.md b/doc/user/project/integrations/prometheus.md index f1567208a8f..a19b819c823 100644 --- a/doc/user/project/integrations/prometheus.md +++ b/doc/user/project/integrations/prometheus.md @@ -88,7 +88,7 @@ service account can be found at Google's documentation for [Authenticating from a service account](https://cloud.google.com/iap/docs/authentication-howto#authenticating_from_a_service_account). 1. Navigate to the [Integrations page](overview.md#accessing-integrations) at - **{settings}** **Settings > Integrations**. + **Settings > Integrations**. 1. Click the **Prometheus** service. 1. For **API URL**, provide the domain name or IP address of your server, such as `http://prometheus.example.com/` or `http://192.0.2.1/`. @@ -121,7 +121,8 @@ one of them will be used: [Prometheus manual configuration](#manual-configuration-of-prometheus) and a [managed Prometheus on Kubernetes](#managed-prometheus-on-kubernetes), the manual configuration takes precedence and is used to run queries from - [dashboards](../../../operations/metrics/dashboards/index.md#defining-custom-dashboards-per-project) and [custom metrics](../../../operations/metrics/index.md#adding-custom-metrics). + [custom dashboards](../../../operations/metrics/dashboards/index.md) and + [custom metrics](../../../operations/metrics/index.md#adding-custom-metrics). - If you have managed Prometheus applications installed on Kubernetes clusters at **different** levels (project, group, instance), the order of precedence is described in [Cluster precedence](../../instance/clusters/index.md#cluster-precedence). diff --git a/doc/user/project/integrations/prometheus_library/cloudwatch.md b/doc/user/project/integrations/prometheus_library/cloudwatch.md index 911493cdae9..e278c7eb664 100644 --- a/doc/user/project/integrations/prometheus_library/cloudwatch.md +++ b/doc/user/project/integrations/prometheus_library/cloudwatch.md @@ -4,7 +4,7 @@ group: APM info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# Monitoring AWS Resources +# Monitoring AWS resources > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/12621) in GitLab 9.4 diff --git a/doc/user/project/integrations/redmine.md b/doc/user/project/integrations/redmine.md index c92ddf38ad2..2a85dd9b79b 100644 --- a/doc/user/project/integrations/redmine.md +++ b/doc/user/project/integrations/redmine.md @@ -1,3 +1,9 @@ +--- +stage: Create +group: Ecosystem +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Redmine Service 1. To enable the Redmine integration in a project, navigate to the diff --git a/doc/user/project/integrations/services_templates.md b/doc/user/project/integrations/services_templates.md index bc2bdde2f64..688643a85a7 100644 --- a/doc/user/project/integrations/services_templates.md +++ b/doc/user/project/integrations/services_templates.md @@ -1,3 +1,9 @@ +--- +stage: Create +group: Ecosystem +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Service templates Using a service template, GitLab administrators can provide default values for configuring integrations at the project level. diff --git a/doc/user/project/integrations/slack.md b/doc/user/project/integrations/slack.md index 6c5dc787c5e..03ff5f845b6 100644 --- a/doc/user/project/integrations/slack.md +++ b/doc/user/project/integrations/slack.md @@ -1,3 +1,9 @@ +--- +stage: Create +group: Ecosystem +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Slack Notifications Service The Slack Notifications Service allows your GitLab project to send events @@ -19,7 +25,7 @@ separately configured [Slack slash commands](slack_slash_commands.md). 1. Open your project's page, and navigate to your project's [Integrations page](overview.md#accessing-integrations) at - **{settings}** **Settings > Integrations**. + **Settings > Integrations**. 1. Select the **Slack notifications** integration to configure it. 1. Click **Enable integration**. 1. In **Trigger**, select the checkboxes for each type of GitLab event to send to Slack as a diff --git a/doc/user/project/integrations/slack_slash_commands.md b/doc/user/project/integrations/slack_slash_commands.md index d25a367bd1f..7c2413fce81 100644 --- a/doc/user/project/integrations/slack_slash_commands.md +++ b/doc/user/project/integrations/slack_slash_commands.md @@ -1,3 +1,9 @@ +--- +stage: Create +group: Ecosystem +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Slack slash commands **(CORE ONLY)** > Introduced in GitLab 8.15. diff --git a/doc/user/project/integrations/unify_circuit.md b/doc/user/project/integrations/unify_circuit.md index 98dc6f298d5..c4959a8711b 100644 --- a/doc/user/project/integrations/unify_circuit.md +++ b/doc/user/project/integrations/unify_circuit.md @@ -1,3 +1,9 @@ +--- +stage: Create +group: Ecosystem +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Unify Circuit service The Unify Circuit service sends notifications from GitLab to the conversation for which the webhook was created. diff --git a/doc/user/project/integrations/webex_teams.md b/doc/user/project/integrations/webex_teams.md index 10735e33746..39daa14407f 100644 --- a/doc/user/project/integrations/webex_teams.md +++ b/doc/user/project/integrations/webex_teams.md @@ -1,3 +1,9 @@ +--- +stage: Create +group: Ecosystem +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Webex Teams service You can configure GitLab to send notifications to a Webex Teams space. diff --git a/doc/user/project/integrations/webhooks.md b/doc/user/project/integrations/webhooks.md index 5a0ca03a646..800eb1d3359 100644 --- a/doc/user/project/integrations/webhooks.md +++ b/doc/user/project/integrations/webhooks.md @@ -1,23 +1,10 @@ -# Webhooks +--- +stage: Create +group: Ecosystem +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- -> **Note:** -> Starting from GitLab 8.5: -> -> - the `repository` key is deprecated in favor of the `project` key -> - the `project.ssh_url` key is deprecated in favor of the `project.git_ssh_url` key -> - the `project.http_url` key is deprecated in favor of the `project.git_http_url` key -> -> **Note:** -> Starting from GitLab 11.1, the logs of webhooks are automatically removed after -> one month. -> -> **Note:** -> Starting from GitLab 11.2: -> -> - The `description` field for issues, merge requests, comments, and wiki pages -> is rewritten so that simple Markdown image references (like -> ``) have their target URL changed to an absolute URL. See -> [image URL rewriting](#image-url-rewriting) for more details. +# Webhooks Project webhooks allow you to trigger a URL if for example new code is pushed or a new issue is created. You can configure webhooks to listen for specific events @@ -48,7 +35,25 @@ Navigate to the webhooks page by going to your project's **Settings ➔ Webhooks**. NOTE: **Note:** -On GitLab.com, the [maximum number of webhooks](../../../user/gitlab_com/index.md#maximum-number-of-webhooks) per project, and per group, is limited. +On GitLab.com, the [maximum number of webhooks and their size](../../../user/gitlab_com/index.md#webhooks) per project, and per group, is limited. + +## Version history + +Starting from GitLab 8.5: + +- the `repository` key is deprecated in favor of the `project` key +- the `project.ssh_url` key is deprecated in favor of the `project.git_ssh_url` key +- the `project.http_url` key is deprecated in favor of the `project.git_http_url` key + +Starting from GitLab 11.1, the logs of webhooks are automatically removed after +one month. + +Starting from GitLab 11.2: + +- The `description` field for issues, merge requests, comments, and wiki pages + is rewritten so that simple Markdown image references (like + ``) have their target URL changed to an absolute URL. See + [image URL rewriting](#image-url-rewriting) for more details. ## Use-cases @@ -722,7 +727,7 @@ X-Gitlab-Event: Note Hook "type": "ProjectLabel", "group_id": null } - ], + ] } } ``` @@ -1296,6 +1301,58 @@ X-Gitlab-Event: Job Hook Note that `commit.id` is the ID of the pipeline, not the ID of the commit. +### Deployment events + +Triggered when deployment is finished/failed/canceled. + +**Request Header**: + +```plaintext +X-Gitlab-Event: Deployment Hook +``` + +**Request Body**: + +```json +{ + "object_kind": "deployment", + "status": "success", + "deployable_id": 796, + "deployable_url": "http://10.126.0.2:3000/root/test-deployment-webhooks/-/jobs/796", + "environment": "staging", + "project": { + "id": 30, + "name": "test-deployment-webhooks", + "description": "", + "web_url": "http://10.126.0.2:3000/root/test-deployment-webhooks", + "avatar_url": null, + "git_ssh_url": "ssh://vlad@10.126.0.2:2222/root/test-deployment-webhooks.git", + "git_http_url": "http://10.126.0.2:3000/root/test-deployment-webhooks.git", + "namespace": "Administrator", + "visibility_level": 0, + "path_with_namespace": "root/test-deployment-webhooks", + "default_branch": "master", + "ci_config_path": "", + "homepage": "http://10.126.0.2:3000/root/test-deployment-webhooks", + "url": "ssh://vlad@10.126.0.2:2222/root/test-deployment-webhooks.git", + "ssh_url": "ssh://vlad@10.126.0.2:2222/root/test-deployment-webhooks.git", + "http_url": "http://10.126.0.2:3000/root/test-deployment-webhooks.git" + }, + "short_sha": "279484c0", + "user": { + "name": "Administrator", + "username": "root", + "avatar_url": "https://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon", + "email": "admin@example.com" + }, + "user_url": "http://10.126.0.2:3000/root", + "commit_url": "http://10.126.0.2:3000/root/test-deployment-webhooks/-/commit/279484c09fbe69ededfced8c1bb6e6d24616b468", + "commit_title": "Add new file" +} +``` + +Note that `deployable_id` is the ID of the CI job. + ## Image URL rewriting From GitLab 11.2, simple image references are rewritten to use an absolute URL @@ -1339,7 +1396,8 @@ On this page, you can see data that GitLab sends (request headers and body) and From this page, you can repeat delivery with the same data by clicking `Resend Request` button. -> **Note:** If URL or secret token of the webhook were updated, data will be delivered to the new address. +NOTE: **Note:** +If URL or secret token of the webhook were updated, data will be delivered to the new address. ### Receiving duplicate or multiple webhook requests triggered by one event diff --git a/doc/user/project/integrations/youtrack.md b/doc/user/project/integrations/youtrack.md index e067ab6071e..d243ffc7a37 100644 --- a/doc/user/project/integrations/youtrack.md +++ b/doc/user/project/integrations/youtrack.md @@ -1,3 +1,9 @@ +--- +stage: Create +group: Ecosystem +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # YouTrack Service JetBrains [YouTrack](https://www.jetbrains.com/help/youtrack/standalone/YouTrack-Documentation.html) is a web-based issue tracking and project management platform. diff --git a/doc/user/project/issue_board.md b/doc/user/project/issue_board.md index 1831563332f..7be58ce4ecb 100644 --- a/doc/user/project/issue_board.md +++ b/doc/user/project/issue_board.md @@ -69,7 +69,7 @@ multiple issue boards within the same project. ## Use cases -There are many ways to use GitLab issue boards tailored to your own preferred workflow. +You can tailor GitLab issue boards to your own preferred workflow. Here are some common use cases for issue boards. ### Use cases for a single issue board @@ -324,8 +324,9 @@ As in other list types, click the trash icon to remove a list. > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/11403) in GitLab 12.7 -You can set Work In Progress (WIP) limits per issues list. When a limit is set, the list's header -shows the number of issues in the list and the soft limit of issues. +You can set a Work In Progress (WIP) limit for each issue list on an issue board. When a limit is +set, the list's header shows the number of issues in the list and the soft limit of issues. +You cannot set a WIP limit on the default lists (**Open** and **Closed**). Examples: @@ -337,7 +338,7 @@ Examples: To set a WIP limit for a list: 1. Navigate to a Project or Group board of which you're a member. -1. Click the Settings icon (**{settings}**) in a list's header. +1. Click the settings icon in a list's header. 1. Next to **Work In Progress Limit**, click **Edit**. 1. Enter the maximum number of issues. 1. Press <kbd>Enter</kbd> to save. diff --git a/doc/user/project/issues/confidential_issues.md b/doc/user/project/issues/confidential_issues.md index 602a6d79848..5bb8805159a 100644 --- a/doc/user/project/issues/confidential_issues.md +++ b/doc/user/project/issues/confidential_issues.md @@ -46,7 +46,8 @@ system note in the issue's comments. ## Indications of a confidential issue ->**Note:** If you don't have [enough permissions](#permissions-and-access-to-confidential-issues), +NOTE: **Note:** +If you don't have [enough permissions](#permissions-and-access-to-confidential-issues), you won't be able to see the confidential issues at all. There are a few things that visually separate a confidential issue from a diff --git a/doc/user/project/issues/design_management.md b/doc/user/project/issues/design_management.md index 618506ea7ee..5e456c7986c 100644 --- a/doc/user/project/issues/design_management.md +++ b/doc/user/project/issues/design_management.md @@ -1,7 +1,8 @@ # Design Management > - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/660) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.2. -> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/212566) to GitLab Core in 13.0. +> - Support for SVGs was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12771) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.4. +> - Design Management was [moved](https://gitlab.com/gitlab-org/gitlab/-/issues/212566) to GitLab Core in 13.0. ## Overview @@ -41,10 +42,9 @@ If the requirements are not met, the **Designs** tab displays a message to the u ## Supported files Files uploaded must have a file extension of either `png`, `jpg`, `jpeg`, -`gif`, `bmp`, `tiff` or `ico`. +`gif`, `bmp`, `tiff`, `ico`, or `svg`. -Support for [SVG files](https://gitlab.com/gitlab-org/gitlab/-/issues/12771) -and [PDFs](https://gitlab.com/gitlab-org/gitlab/-/issues/32811) is planned for a future release. +Support for [PDF](https://gitlab.com/gitlab-org/gitlab/issues/32811) is planned for a future release. ## Limitations @@ -110,7 +110,7 @@ instead of directly on the issue description. To upload Design images, drag files from your computer and drop them in the Design Management section, or click **upload** to select images from your file browser: - + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/34353) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.9, you can drag and drop designs onto the dedicated drop zone to upload them. @@ -197,11 +197,19 @@ Once selected, click the **Delete selected** button to confirm the deletion:  -**Note:** +NOTE: **Note:** Only the latest version of the designs can be deleted. Deleted designs are not permanently lost; they can be viewed by browsing previous versions. +## Reordering designs + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/34382) in GitLab 13.3. + +You can change the order of designs by dragging them to a new position: + + + ## Starting discussions on designs When a design is uploaded, you can start a discussion by clicking on @@ -268,21 +276,21 @@ This will be rendered as: ### Enable or disable design references **(CORE ONLY)** -Design reference parsing is under development and not ready for production use. It is -deployed behind a feature flag that is **disabled by default**. +Design reference parsing is +deployed behind a feature flag that is **enabled by default**. [GitLab administrators with access to the GitLab Rails console](../../../administration/feature_flags.md) -can enable it for your instance. +can disable it for your instance. -To enable it: +To disable it: ```ruby -Feature.enable(:design_management_reference_filter_gfm_pipeline) +Feature.disable(:design_management_reference_filter_gfm_pipeline) ``` -To disable it: +To re-enable it: ```ruby -Feature.disable(:design_management_reference_filter_gfm_pipeline) +Feature.enable(:design_management_reference_filter_gfm_pipeline) ``` ## Design activity records diff --git a/doc/user/project/issues/img/design_drag_and_drop_uploads_v13_2.png b/doc/user/project/issues/img/design_drag_and_drop_uploads_v13_2.png Binary files differindex d60f1234b6d..25a02eef406 100644 --- a/doc/user/project/issues/img/design_drag_and_drop_uploads_v13_2.png +++ b/doc/user/project/issues/img/design_drag_and_drop_uploads_v13_2.png diff --git a/doc/user/project/issues/img/design_management_upload_v13.2.png b/doc/user/project/issues/img/design_management_upload_v13.2.png Binary files differdeleted file mode 100644 index 1d4b10307fc..00000000000 --- a/doc/user/project/issues/img/design_management_upload_v13.2.png +++ /dev/null diff --git a/doc/user/project/issues/img/design_management_upload_v13.3.png b/doc/user/project/issues/img/design_management_upload_v13.3.png Binary files differnew file mode 100644 index 00000000000..f7b3f79fb22 --- /dev/null +++ b/doc/user/project/issues/img/design_management_upload_v13.3.png diff --git a/doc/user/project/issues/img/design_management_v13_2.png b/doc/user/project/issues/img/design_management_v13_2.png Binary files differindex 0a6e2be17ab..6d7e03d6f20 100644 --- a/doc/user/project/issues/img/design_management_v13_2.png +++ b/doc/user/project/issues/img/design_management_v13_2.png diff --git a/doc/user/project/issues/img/designs_reordering_v13_3.gif b/doc/user/project/issues/img/designs_reordering_v13_3.gif Binary files differnew file mode 100644 index 00000000000..496eea532e2 --- /dev/null +++ b/doc/user/project/issues/img/designs_reordering_v13_3.gif diff --git a/doc/user/project/issues/img/epic_tree_health_status_v12_10.png b/doc/user/project/issues/img/epic_tree_health_status_v12_10.png Binary files differindex c6d0117b1c4..1a603656fd8 100644 --- a/doc/user/project/issues/img/epic_tree_health_status_v12_10.png +++ b/doc/user/project/issues/img/epic_tree_health_status_v12_10.png diff --git a/doc/user/project/issues/img/resolve_design-discussion_checkbox_v13_1.png b/doc/user/project/issues/img/resolve_design-discussion_checkbox_v13_1.png Binary files differindex bd9d1f7a77c..8791419b919 100644 --- a/doc/user/project/issues/img/resolve_design-discussion_checkbox_v13_1.png +++ b/doc/user/project/issues/img/resolve_design-discussion_checkbox_v13_1.png diff --git a/doc/user/project/issues/img/resolve_design-discussion_icon_v13_1.png b/doc/user/project/issues/img/resolve_design-discussion_icon_v13_1.png Binary files differindex 5cd005f0799..fc1fff321ba 100644 --- a/doc/user/project/issues/img/resolve_design-discussion_icon_v13_1.png +++ b/doc/user/project/issues/img/resolve_design-discussion_icon_v13_1.png diff --git a/doc/user/project/issues/index.md b/doc/user/project/issues/index.md index 9113f5344ba..a6911d183c1 100644 --- a/doc/user/project/issues/index.md +++ b/doc/user/project/issues/index.md @@ -109,6 +109,15 @@ Key actions for Issues include: On an issue's page, you can view [all aspects of the issue](issue_data_and_actions.md), and modify them if you have the necessary [permissions](../../permissions.md). +#### Real-time sidebar **(CORE ONLY)** + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/17589) in GitLab 13.3. +> - It cannot be enabled or disabled per-project. +> - It's not recommended for production use. + +Assignees in the sidebar are updated in real time. This feature is **disabled by default**. +To enable, you need to enable [ActionCable in-app mode](https://docs.gitlab.com/omnibus/settings/actioncable.html). + ### Issues list  diff --git a/doc/user/project/issues/issue_data_and_actions.md b/doc/user/project/issues/issue_data_and_actions.md index 7f236d04fb6..77c50f9178c 100644 --- a/doc/user/project/issues/issue_data_and_actions.md +++ b/doc/user/project/issues/issue_data_and_actions.md @@ -316,4 +316,4 @@ Read more how to [add or remove a zoom meeting](associate_zoom_meeting.md). > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/30906) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.1. -If a status page application is associated with the project, you can use the `/publish` [quick action](../quick_actions.md) to publish the issue. Refer to [GitLab Status Page](../status_page/index.md) for more information. +If a status page application is associated with the project, you can use the `/publish` [quick action](../quick_actions.md) to publish the issue. Refer to [GitLab Status Page](../../../operations/incident_management/status_page.md) for more information. diff --git a/doc/user/project/issues/managing_issues.md b/doc/user/project/issues/managing_issues.md index bcf2ab66978..0e0731528be 100644 --- a/doc/user/project/issues/managing_issues.md +++ b/doc/user/project/issues/managing_issues.md @@ -287,3 +287,7 @@ To add an issue to an [iteration](../../group/iterations/index.md): 1. In an issue sidebar, click **Edit** next to **Iteration**. A dropdown appears. 1. Click an iteration you'd like to associate this issue with. + +You can also use the `/iteration` +[quick action](../quick_actions.md#quick-actions-for-issues-merge-requests-and-epics) +in a comment or description field. diff --git a/doc/user/project/issues/related_issues.md b/doc/user/project/issues/related_issues.md index 445c28492df..954e3771722 100644 --- a/doc/user/project/issues/related_issues.md +++ b/doc/user/project/issues/related_issues.md @@ -14,32 +14,39 @@ and projects. The relationship only shows up in the UI if the user can see both issues. +When you try to close an issue that has open blockers, a warning is displayed. + +TIP: **Tip:** +To manage related issues through our API, see the [API documentation](../../../api/issue_links.md). + ## Adding a related issue > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/2035) in [GitLab Starter](https://about.gitlab.com/pricing/) 12.8. -> - [Improved](https://gitlab.com/gitlab-org/gitlab/-/issues/34239) in [GitLab Starter](https://about.gitlab.com/pricing/) 13.0. +> - [Improved](https://gitlab.com/gitlab-org/gitlab/-/issues/34239) to warn when attempting to close an issue that is blocked by others in [GitLab Starter](https://about.gitlab.com/pricing/) 13.0. > When you try to close an issue with open blockers, you'll see a warning that you can dismiss. -You can relate one issue to another by clicking the related issues "+" button -in the header of the related issue block. Then, input the issue reference number -or paste in the full URL of the issue. +1. Relate one issue to another by clicking the related issues "+" button +in the header of the related issue block. + +1. Input the issue reference number or paste in the full URL of the issue. -Additionally, you can select whether the current issue relates to, blocks, or is blocked by the issues being entered. +1. Select whether the current issue relates to, blocks, or is blocked by the issues being entered. - +  -Issues of the same project can be specified just by the reference number. -Issues from a different project require additional information like the -group and the project name. For example: + Issues of the same project can be specified just by the reference number. + Issues from a different project require additional information like the + group and the project name. For example: -- same project: `#44` -- same group: `project#44` -- different group: `group/project#44` + - same project: `#44` + - same group: `project#44` + - different group: `group/project#44` -Valid references will be added to a temporary list that you can review. -When you have added all the related issues, click **Add** to submit. + Valid references will be added to a temporary list that you can review. -Once you have finished adding all related issues, you will be able to see +1. When you have added all the related issues, click **Add** to submit. + +When you have finished adding all related issues, you will be able to see them categorized so their relationships can be better understood visually.  @@ -47,11 +54,10 @@ them categorized so their relationships can be better understood visually. ## Removing a related issue In the related issues block, click the "x" icon on the right-side of each issue -token that you wish to remove. Due to the bi-directional relationship, it -will no longer appear in either issue. +token that you wish to remove. + +Due to the bi-directional relationship, it will no longer appear in either issue.  Please access our [permissions](../../permissions.md) page for more information. - -Additionally, you are also able to manage related issues through [our API](../../../api/issue_links.md). diff --git a/doc/user/project/labels.md b/doc/user/project/labels.md index 9886ef91f16..0d02b89be7e 100644 --- a/doc/user/project/labels.md +++ b/doc/user/project/labels.md @@ -43,7 +43,7 @@ To assign a label to an issue, merge request or epic: click on them. You can search repeatedly and add more labels. 1. Click **X** or anywhere outside the label section and the labels are applied. -You can also assign a label with the [`/assign @username` quick action](quick_actions.md). +You can also assign a label with the [`/label ~label1 ~label2` quick action](quick_actions.md). ## Label management @@ -82,6 +82,9 @@ Once created, you can edit a label by clicking the pencil (**{pencil}**), or del a label by clicking the three dots (**{ellipsis_v}**) next to the **Subscribe** button and selecting **Delete**. +CAUTION: **Caution:** +If you delete a label, it is permanently deleted. You will not be able to undo the deletion, and all references to the label will be removed from the system. + #### Promote a project label to a group label If you previously created a project label and now want to make it available for other diff --git a/doc/user/project/merge_requests/allow_collaboration.md b/doc/user/project/merge_requests/allow_collaboration.md index f90917d0a8b..60d247ccc19 100644 --- a/doc/user/project/merge_requests/allow_collaboration.md +++ b/doc/user/project/merge_requests/allow_collaboration.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Source Code +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference, howto --- diff --git a/doc/user/project/merge_requests/authorization_for_merge_requests.md b/doc/user/project/merge_requests/authorization_for_merge_requests.md index 0579e3568da..26b3682990e 100644 --- a/doc/user/project/merge_requests/authorization_for_merge_requests.md +++ b/doc/user/project/merge_requests/authorization_for_merge_requests.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Source Code +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: concepts --- diff --git a/doc/user/project/merge_requests/cherry_pick_changes.md b/doc/user/project/merge_requests/cherry_pick_changes.md index 6685c50c1ed..d65c005ee34 100644 --- a/doc/user/project/merge_requests/cherry_pick_changes.md +++ b/doc/user/project/merge_requests/cherry_pick_changes.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Source Code +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference, concepts --- diff --git a/doc/user/project/merge_requests/code_quality.md b/doc/user/project/merge_requests/code_quality.md index 36acba032ff..3c697e22cf5 100644 --- a/doc/user/project/merge_requests/code_quality.md +++ b/doc/user/project/merge_requests/code_quality.md @@ -7,7 +7,8 @@ type: reference, howto # Code Quality -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/1984) in [GitLab Starter](https://about.gitlab.com/pricing/) 9.3. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/1984) in [GitLab Starter](https://about.gitlab.com/pricing/) 9.3. +> - Made [available in all tiers](https://gitlab.com/gitlab-org/gitlab/-/issues/212499) in 13.2. Ensuring your project's code stays simple, readable and easy to contribute to can be problematic. With the help of [GitLab CI/CD](../../../ci/README.md), you can analyze your source code quality using GitLab Code Quality. @@ -66,12 +67,10 @@ For instance, consider the following workflow: ## Example configuration -CAUTION: **Caution:** -The job definition shown below is supported on GitLab 11.11 and later versions. It -also requires the GitLab Runner 11.5 or later. For earlier versions, use the -[previous job definitions](#previous-job-definitions). - This example shows how to run Code Quality on your code by using GitLab CI/CD and Docker. +It requires GitLab 11.11 or later, and GitLab Runner 11.5 or later. If you are using +GitLab 11.4 or ealier, you can view the deprecated job definitions in the +[documentation archive](https://docs.gitlab.com/12.10/ee/user/project/merge_requests/code_quality.html#previous-job-definitions). First, you need GitLab Runner configured: @@ -131,103 +130,65 @@ definition they will be able to execute privileged Docker commands on the Runner host. Having proper access control policies mitigates this attack vector by allowing access only to trusted actors. -### Previous job definitions +### Disabling the code quality job -CAUTION: **Caution:** -Before GitLab 11.5, Code Quality job and artifact had to be named specifically to -automatically extract report data and show it in the merge request widget. While these -old job definitions are still maintained they have been deprecated and are no longer supported on GitLab 12.0 or higher. -You're advised to update your `.gitlab-ci.yml` configuration to reflect that change. +The `code_quality` job will not run if the `$CODE_QUALITY_DISABLED` environment +variable is present. Please refer to the environment variables [documentation](../../../ci/variables/README.md) +to learn more about how to define one. -For GitLab 11.5 and later, the job should look like: +To disable the `code_quality` job, add `CODE_QUALITY_DISABLED` as a custom environment +variable. This can be done: -```yaml -code_quality: - image: docker:stable - variables: - DOCKER_DRIVER: overlay2 - allow_failure: true - services: - - docker:stable-dind - script: - - export SP_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/') - - docker run - --env SOURCE_CODE="$PWD" - --volume "$PWD":/code - --volume /var/run/docker.sock:/var/run/docker.sock - "registry.gitlab.com/gitlab-org/ci-cd/codequality:$SP_VERSION" /code - artifacts: - reports: - codequality: gl-code-quality-report.json -``` +- For the whole project, [in the project settings](../../../ci/variables/README.md#create-a-custom-variable-in-the-ui) + or [CI/CD configuration](../../../ci/variables/README.md#create-a-custom-variable-in-the-ui). +- For a single pipeline run: + + 1. Go to **CI/CD > Pipelines** + 1. Click **Run Pipeline** + 1. Add `CODE_QUALITY_DISABLED` as the variable key, with any value. + +### Using with merge request pipelines + +The configuration provided by the Code Quality template does not let the `code_quality` job +run on [pipelines for merge requests](../../../ci/merge_request_pipelines/index.md). -In GitLab 12.6, Code Quality switched to the -[new versioning scheme](https://gitlab.com/gitlab-org/ci-cd/codequality#versioning-and-release-cycle). -It's highly recommended to include the Code Quality template as shown in the -[example configuration](#example-configuration), which uses the new versioning scheme. -If not using the template, the `SP_VERSION` variable can be hardcoded to use the -new image versions: +If pipelines for merge requests is enabled, the `code_quality:rules` must be redefined. + +The template has these [`rules`](../../../ci/yaml/README.md#rules) for the `code quality` job: ```yaml code_quality: - image: docker:stable - variables: - DOCKER_DRIVER: overlay2 - SP_VERSION: 0.85.6 - allow_failure: true - services: - - docker:stable-dind - script: - - docker run - --env SOURCE_CODE="$PWD" - --volume "$PWD":/code - --volume /var/run/docker.sock:/var/run/docker.sock - "registry.gitlab.com/gitlab-org/ci-cd/codequality:$SP_VERSION" /code - artifacts: - reports: - codequality: gl-code-quality-report.json + rules: + - if: '$CODE_QUALITY_DISABLED' + when: never + - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' ``` -For GitLab 11.4 and earlier, the job should look like: +If you are using merge request pipelines, your `rules` (or [`workflow: rules`](../../../ci/yaml/README.md#workflowrules)) +might look like this example: ```yaml -code_quality: - image: docker:stable - variables: - DOCKER_DRIVER: overlay2 - allow_failure: true - services: - - docker:stable-dind - script: - - export SP_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/') - - docker run - --env SOURCE_CODE="$PWD" - --volume "$PWD":/code - --volume /var/run/docker.sock:/var/run/docker.sock - "registry.gitlab.com/gitlab-org/ci-cd/codequality:$SP_VERSION" /code - artifacts: - paths: [gl-code-quality-report.json] +job1: + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' # Run job1 in merge request pipelines + - if: '$CI_COMMIT_BRANCH == "master"' # Run job1 in pipelines on the master branch (but not in other branch pipelines) + - if: '$CI_COMMIT_TAG' # Run job1 in pipelines for tags ``` -Alternatively the job name could be `codeclimate` or `codequality` and the artifact -name could be `codeclimate.json`. These names have been deprecated with GitLab 11.0 -and may be removed in the next major release, GitLab 12.0. - -For GitLab 10.3 and earlier, the job should look like: +To make these work together, you will need to overwrite the code quality `rules` +so that they match your current `rules`. From the example above, it could look like: ```yaml -codequality: - image: docker:latest - variables: - DOCKER_DRIVER: overlay - services: - - docker:dind - script: - - docker pull codeclimate/codeclimate:0.69.0 - - docker run --env CODECLIMATE_CODE="$PWD" --volume "$PWD":/code --volume /var/run/docker.sock:/var/run/docker.sock --volume /tmp/cc:/tmp/cc codeclimate/codeclimate:0.69.0 init - - docker run --env CODECLIMATE_CODE="$PWD" --volume "$PWD":/code --volume /var/run/docker.sock:/var/run/docker.sock --volume /tmp/cc:/tmp/cc codeclimate/codeclimate:0.69.0 analyze -f json > codeclimate.json || true - artifacts: - paths: [codeclimate.json] +include: + - template: Code-Quality.gitlab-ci.yml + +code_quality: + rules: + - if: '$CODE_QUALITY_DISABLED' + when: never + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' # Run code quality job in merge request pipelines + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' # Run code quality job in pipelines on the master branch (but not in other branch pipelines) + - if: '$CI_COMMIT_TAG' # Run code quality job in pipelines for tags ``` ## Configuring jobs using variables diff --git a/doc/user/project/merge_requests/creating_merge_requests.md b/doc/user/project/merge_requests/creating_merge_requests.md index 50d5decc2cc..0495c864335 100644 --- a/doc/user/project/merge_requests/creating_merge_requests.md +++ b/doc/user/project/merge_requests/creating_merge_requests.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Source Code +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: howto description: "How to create Merge Requests in GitLab." disqus_identifier: 'https://docs.gitlab.com/ee/gitlab-basics/add-merge-request.html' @@ -18,7 +21,7 @@ or through the [GitLab UI](#new-merge-request-from-a-new-branch-created-through- This document describes the several ways to create a merge request. When you start a new merge request, regardless of the method, -you'll be taken to the [**New Merge Request** page](#new-merge-request-page) +you are taken to the [**New Merge Request** page](#new-merge-request-page) to fill it with information about the merge request. If you push a new branch to GitLab, also regardless of the method, @@ -29,8 +32,8 @@ button and start a merge request from there. On the **New Merge Request** page, start by filling in the title and description for the merge request. If there are already -commits on the branch, the title will be prefilled with the first -line of the first commit message, and the description will be +commits on the branch, the title is prefilled with the first +line of the first commit message, and the description is prefilled with any additional lines in the commit message. The title is the only field that is mandatory in all cases. @@ -61,18 +64,18 @@ You can also see the **Create merge request** button in the top-right of the: - **Repository > Files** page. - **Merge Requests** page. -In this case, GitLab will use the most recent branch you pushed +In this case, GitLab uses the most recent branch you pushed changes to as the source branch, and the default branch in the current project as the target. ## New merge request by adding, editing, and uploading a file When you choose to edit, add, or upload a file through the GitLab UI, -at the end of the file you'll see the option to add the **Commit message**, +at the end of the file you see the option to add the **Commit message**, to select the **Target branch** of that commit, and the checkbox to **Start new a merge request with these changes**. -Similarly, if you change files through the Web IDE, when you navigate to **Commit** on the left-hand sidebar, you'll see these same options. +Similarly, if you change files through the Web IDE, when you navigate to **Commit** on the left-hand sidebar, you see these same options. Once you have added, edited, or uploaded the file: @@ -81,23 +84,23 @@ Once you have added, edited, or uploaded the file: 1. Keep the checkbox checked to start a new merge request straightaway, or, uncheck it to add more changes to that branch before starting the merge request. 1. Click **Commit changes**. -If you chose to start a merge request, you'll be taken to the +If you chose to start a merge request, you are taken to the [**New Merge Request** page](#new-merge-request-page), from which you can fill it in with information and submit the merge request. -The merge request will target the default branch of the repository. +The merge request targets the default branch of the repository. If you want to change it, you can do it later by editing the merge request. ## New merge request from a new branch created through the UI To quickly start working on files through the GitLab UI, navigate to your project's **Repository > Branches** and click -**New branch**. A new branch will be created and you can start +**New branch**. A new branch is created and you can start editing files. Once committed and pushed, you can click on the [**Create Merge Request**](#create-merge-request-button) button to open the [**New Merge Request** page](#new-merge-request-page). -A new merge request will be started using the current branch as the source, +A new merge request is started using the current branch as the source, and the default branch in the current project as the target. ## New merge request from your local environment @@ -123,7 +126,7 @@ Once you're done, [push your branch to GitLab](../../../gitlab-basics/start-usin git push origin my-new-branch ``` -In the output, GitLab will prompt you with a direct link for creating +In the output, GitLab prompts you with a direct link for creating a merge request: ```shell @@ -133,7 +136,7 @@ remote: https://gitlab-instance.com/my-group/my-project/merge_requests/new?mer ``` Copy that link and paste it in your browser, and the [**New Merge Request page**](#new-merge-request-page) -will be displayed. +is displayed. There is also a number of [flags you can add to commands when pushing through the command line](../push_options.md) to reduce the need for editing merge requests manually through the UI. @@ -183,10 +186,10 @@ available in GitLab.com. You can create a new merge request by sending an email to a user-specific email address. The address can be obtained on the merge requests page by clicking on -a **Email a new merge request to this project** button. The subject will be +a **Email a new merge request to this project** button. The subject is used as the source branch name for the new merge request and the target branch -will be the default branch for the project. The message body (if not empty) -will be used as the merge request description. You need +is the default branch for the project. The message body (if not empty) +is used as the merge request description. You need ["Reply by email"](../../../administration/reply_by_email.md) enabled to use this feature. If it's not enabled to your instance, you may ask your GitLab administrator to do so. @@ -207,16 +210,16 @@ or contacts to continue working._ You can add commits to the merge request being created by adding patches as attachments to the email. All attachments with a filename -ending in `.patch` will be considered patches and they will be processed +ending in `.patch` are considered patches and they are processed ordered by name. The combined size of the patches can be 2MB. -If the source branch from the subject does not exist, it will be +If the source branch from the subject does not exist, it is created from the repository's HEAD or the specified target branch to apply the patches. The target branch can be specified using the [`/target_branch` quick action](../quick_actions.md). If the source -branch already exists, the patches will be applied on top of it. +branch already exists, the patches are applied on top of it. ## Reviewing and managing Merge Requests diff --git a/doc/user/project/merge_requests/fail_fast_testing.md b/doc/user/project/merge_requests/fail_fast_testing.md index 619a6d04577..60f81159394 100644 --- a/doc/user/project/merge_requests/fail_fast_testing.md +++ b/doc/user/project/merge_requests/fail_fast_testing.md @@ -45,8 +45,9 @@ This template requires: - Use [Pipelines for Merge Requests](../../../ci/merge_request_pipelines/index.md#configuring-pipelines-for-merge-requests) - [Pipelines for Merged Results](../../../ci/merge_request_pipelines/pipelines_for_merged_results/index.md#enable-pipelines-for-merged-results) enabled in the project settings. +- A Docker image with Ruby available. The template uses `image: ruby:2.6` by default, but you [can override](../../../ci/yaml/includes.md#overriding-external-template-values) this. -## Configure Fast RSpec Failure +## Configuring Fast RSpec Failure We'll use the following plain RSpec configuration as a starting point. It installs all the project gems and executes `rspec`, on merge request pipelines only. @@ -69,6 +70,16 @@ include: - template: Verify/FailFast.gitlab-ci.yml ``` +To customize the job, specific options may be set to override the template. For example, to override the default Docker image: + +```yaml +include: + - template: Verify/FailFast.gitlab-ci.yml + +rspec-rails-modified-path-specs: + image: custom-docker-image-with-ruby +``` + ### Example test loads For illustrative purposes, let's say our Rails app spec suite consists of 100 specs per model for ten models. diff --git a/doc/user/project/merge_requests/fast_forward_merge.md b/doc/user/project/merge_requests/fast_forward_merge.md index 2d59e535ce1..b021fa6f336 100644 --- a/doc/user/project/merge_requests/fast_forward_merge.md +++ b/doc/user/project/merge_requests/fast_forward_merge.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Source Code +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference, concepts --- diff --git a/doc/user/project/merge_requests/getting_started.md b/doc/user/project/merge_requests/getting_started.md index 9ac0f3ee42e..c7cabf3c73b 100644 --- a/doc/user/project/merge_requests/getting_started.md +++ b/doc/user/project/merge_requests/getting_started.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Source Code +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: index, reference description: "Getting started with Merge Requests." --- diff --git a/doc/user/project/merge_requests/img/accessibility_mr_widget_v13_0.png b/doc/user/project/merge_requests/img/accessibility_mr_widget_v13_0.png Binary files differindex c52bf9964f1..4ada7e25b65 100644 --- a/doc/user/project/merge_requests/img/accessibility_mr_widget_v13_0.png +++ b/doc/user/project/merge_requests/img/accessibility_mr_widget_v13_0.png diff --git a/doc/user/project/merge_requests/img/approvals_premium_mr_widget_v12_7.png b/doc/user/project/merge_requests/img/approvals_premium_mr_widget_v12_7.png Binary files differdeleted file mode 100644 index 164779a8450..00000000000 --- a/doc/user/project/merge_requests/img/approvals_premium_mr_widget_v12_7.png +++ /dev/null diff --git a/doc/user/project/merge_requests/img/approvals_premium_mr_widget_v13_3.png b/doc/user/project/merge_requests/img/approvals_premium_mr_widget_v13_3.png Binary files differnew file mode 100644 index 00000000000..306bf57354d --- /dev/null +++ b/doc/user/project/merge_requests/img/approvals_premium_mr_widget_v13_3.png diff --git a/doc/user/project/merge_requests/img/approvals_premium_project_edit_v12_5.png b/doc/user/project/merge_requests/img/approvals_premium_project_edit_v12_5.png Binary files differdeleted file mode 100644 index 24c8c8f8c11..00000000000 --- a/doc/user/project/merge_requests/img/approvals_premium_project_edit_v12_5.png +++ /dev/null diff --git a/doc/user/project/merge_requests/img/browser_performance_testing.png b/doc/user/project/merge_requests/img/browser_performance_testing.png Binary files differindex c270462f7a8..016abb89f7c 100644 --- a/doc/user/project/merge_requests/img/browser_performance_testing.png +++ b/doc/user/project/merge_requests/img/browser_performance_testing.png diff --git a/doc/user/project/merge_requests/img/code_quality.png b/doc/user/project/merge_requests/img/code_quality.png Binary files differindex 3c6c92baad2..1e7dd64baff 100644 --- a/doc/user/project/merge_requests/img/code_quality.png +++ b/doc/user/project/merge_requests/img/code_quality.png diff --git a/doc/user/project/merge_requests/img/comment-on-any-diff-line.png b/doc/user/project/merge_requests/img/comment-on-any-diff-line.png Binary files differindex 5b9844bf02f..cff5acb98ef 100644 --- a/doc/user/project/merge_requests/img/comment-on-any-diff-line.png +++ b/doc/user/project/merge_requests/img/comment-on-any-diff-line.png diff --git a/doc/user/project/merge_requests/img/draft_blocked_merge_button_v13_2.png b/doc/user/project/merge_requests/img/draft_blocked_merge_button_v13_2.png Binary files differindex beb12c581d6..f7968772500 100644 --- a/doc/user/project/merge_requests/img/draft_blocked_merge_button_v13_2.png +++ b/doc/user/project/merge_requests/img/draft_blocked_merge_button_v13_2.png diff --git a/doc/user/project/merge_requests/img/load_performance_testing.png b/doc/user/project/merge_requests/img/load_performance_testing.png Binary files differindex 3a58e9c28d4..d5623867ee7 100644 --- a/doc/user/project/merge_requests/img/load_performance_testing.png +++ b/doc/user/project/merge_requests/img/load_performance_testing.png diff --git a/doc/user/project/merge_requests/img/merge_request.png b/doc/user/project/merge_requests/img/merge_request.png Binary files differdeleted file mode 100644 index c0a62bbaba0..00000000000 --- a/doc/user/project/merge_requests/img/merge_request.png +++ /dev/null diff --git a/doc/user/project/merge_requests/img/multiline-comment-highlighted.png b/doc/user/project/merge_requests/img/multiline-comment-highlighted.png Binary files differnew file mode 100644 index 00000000000..1bdcc37e274 --- /dev/null +++ b/doc/user/project/merge_requests/img/multiline-comment-highlighted.png diff --git a/doc/user/project/merge_requests/img/multiline-comment-saved.png b/doc/user/project/merge_requests/img/multiline-comment-saved.png Binary files differnew file mode 100644 index 00000000000..cceab36e62b --- /dev/null +++ b/doc/user/project/merge_requests/img/multiline-comment-saved.png diff --git a/doc/user/project/merge_requests/img/squash_squashed_commit.png b/doc/user/project/merge_requests/img/squash_squashed_commit.png Binary files differindex 458361c5490..7def5339d8a 100644 --- a/doc/user/project/merge_requests/img/squash_squashed_commit.png +++ b/doc/user/project/merge_requests/img/squash_squashed_commit.png diff --git a/doc/user/project/merge_requests/img/versions_compare_head_v12_10.png b/doc/user/project/merge_requests/img/versions_compare_head_v12_10.png Binary files differindex 353cf458243..61cf72e7bf9 100644 --- a/doc/user/project/merge_requests/img/versions_compare_head_v12_10.png +++ b/doc/user/project/merge_requests/img/versions_compare_head_v12_10.png diff --git a/doc/user/project/merge_requests/index.md b/doc/user/project/merge_requests/index.md index f68fc7d7b45..a9ee9d8e507 100644 --- a/doc/user/project/merge_requests/index.md +++ b/doc/user/project/merge_requests/index.md @@ -1,17 +1,16 @@ --- +stage: Create +group: Source Code +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: index, reference --- # Merge requests -Merge requests allow you to visualize and collaborate on the proposed changes -to source code that exist as commits on a given Git branch. +A Merge Request (**MR**) is a _request_ to _merge_ one branch into another. - - -A Merge Request (**MR**) is the basis of GitLab as a code collaboration and version -control platform. It's exactly as the name implies: a _request_ to _merge_ one -branch into another. +Use merge requests to visualize and collaborate on proposed changes +to source code. ## Use cases diff --git a/doc/user/project/merge_requests/load_performance_testing.md b/doc/user/project/merge_requests/load_performance_testing.md index 3239269109d..97f4f202ab3 100644 --- a/doc/user/project/merge_requests/load_performance_testing.md +++ b/doc/user/project/merge_requests/load_performance_testing.md @@ -141,7 +141,8 @@ For example, you can override the duration of the test with a CLI option: GitLab only displays the key performance metrics in the MR widget if k6's results are saved via [summary export](https://k6.io/docs/results-visualization/json#summary-export) as a [Load Performance report artifact](../../../ci/pipelines/job_artifacts.md#artifactsreportsload_performance-premium). -The latest Load Performance artifact available is always used. +The latest Load Performance artifact available is always used, using the +summary values from the test. If [GitLab Pages](../pages/index.md) is enabled, you can view the report directly in your browser. diff --git a/doc/user/project/merge_requests/merge_request_approvals.md b/doc/user/project/merge_requests/merge_request_approvals.md index 4e0e609e59e..407fc5db425 100644 --- a/doc/user/project/merge_requests/merge_request_approvals.md +++ b/doc/user/project/merge_requests/merge_request_approvals.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" type: reference, concepts --- @@ -10,16 +13,16 @@ process, as it clearly communicates the ability to merge the change. ## Optional Approvals **(CORE ONLY)** -> Introduced in [GitLab Core 13.2](https://gitlab.com/gitlab-org/gitlab/-/issues/27426). +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/27426) in GitLab 13.2. -Any user with Developer or greater [permissions](../../permissions.md) can approve a merge request in GitLab Core. -This provides a consistent mechanism for reviewers to provide approval, and makes it easy for +Any user with Developer or greater [permissions](../../permissions.md) can approve a merge request in GitLab Core and higher tiers. +This provides a consistent mechanism for reviewers to approve merge requests, and makes it easy for maintainers to know when a change is ready to merge. Approvals in Core are optional and do not prevent a merge request from being merged when there is no approval. ## Required Approvals **(STARTER)** -> Introduced in [GitLab Enterprise Edition 7.12](https://about.gitlab.com/releases/2015/06/22/gitlab-7-12-released/#merge-request-approvers-ee-only). +> [Introduced](https://about.gitlab.com/releases/2015/06/22/gitlab-7-12-released/#merge-request-approvers-ee-only) in GitLab Enterprise Edition 7.12. Available in [GitLab Starter](https://about.gitlab.com/pricing/) and higher tiers. Required approvals enable enforced code review by requiring specified people to approve a merge request before it can be merged. @@ -30,8 +33,8 @@ Required approvals enable multiple use cases: - Specifying reviewers for a given proposed code change, as well as a minimum number of reviewers, through [Approval rules](#approval-rules). - Specifying categories of reviewers, such as backend, frontend, quality assurance, - database, etc., for all proposed code changes. -- Automatically designating [Code Owners as eligible approvers](#code-owners-as-eligible-approvers), + database, and so on, for all proposed code changes. +- Designating [Code Owners as eligible approvers](#code-owners-as-eligible-approvers), determined by the files changed in a merge request. - [Requiring approval from a security team](#security-approvals-in-merge-requests-ultimate) before merging code that could introduce a vulnerability.**(ULTIMATE)** @@ -47,14 +50,14 @@ be merged, and optionally which users should do the approving. Approvals can be If no approval rules are defined, any user can approve a merge request, though the default minimum number of required approvers can still be set in the [project settings for merge request approvals](#merge-request-approvals-project-settings). -Approval rules define how many approvals a merge request must receive before it can -be merged, and optionally which users should do the approving. Approvals can be defined: +You can opt to define one single rule to approve a merge request among the available rules +or choose more than one. Single approval rules are available in GitLab Starter and higher tiers, +while [multiple approval rules](#multiple-approval-rules-premium) are available in +[GitLab Premium](https://about.gitlab.com/pricing/) and above. -- [As project defaults](#adding--editing-a-default-approval-rule). -- [Per merge request](#editing--overriding-approval-rules-per-merge-request). - -If no approval rules are defined, any user can approve a merge request, though the default -minimum number of required approvers can still be set in the [project settings for merge request approvals](#merge-request-approvals-project-settings). +NOTE: **Note:** +On GitLab.com, you can add a group as an approver if you're a member of that group or the +group is public. #### Eligible Approvers @@ -81,6 +84,11 @@ if [**Prevent author approval**](#allowing-merge-request-authors-to-approve-thei and [**Prevent committers approval**](#prevent-approval-of-merge-requests-by-their-committers) (disabled by default) are enabled on the project settings. +[Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/10294) in [GitLab Starter](https://about.gitlab.com/pricing/) 13.3, +when an eligible approver comments on a merge request, it appears in the **Commented by** column of the Approvals widget, +indicating who has engaged in the merge request review. Authors and reviewers can also easily identify who they should reach out +to if they have any questions or inputs about the content of the merge request. + ##### Implicit Approvers If the number of required approvals is greater than the number of assigned approvers, @@ -116,7 +124,7 @@ Alternatively, you can **require** To add or edit the default merge request approval rule: -1. Navigate to your project's **{settings}** **Settings > General** and expand **Merge request approvals**. +1. Navigate to your project's **Settings > General** and expand **Merge request approvals**. 1. Click **Add approval rule**, or **Edit**. - Add or change the **Rule name**. @@ -151,6 +159,15 @@ settings. When creating or editing a merge request, find the **Approval rules** section, then follow the same steps as [Adding / editing a default approval rule](#adding--editing-a-default-approval-rule). +#### Set up an optional approval rule + +MR approvals can be configured to be optional. +This can be useful if you're working on a team where approvals are appreciated, but not required. + +To configure an approval to be optional, set the number of required approvals in **No. approvals required** to `0`. + +You can also set an optional approval rule through the [Merge requests approvals API](../../../api/merge_request_approvals.md#update-merge-request-level-rule), by setting the `approvals_required` attribute to `0`. + #### Multiple approval rules **(PREMIUM)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/1979) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.10. @@ -171,7 +188,7 @@ a rule is already defined. When an [eligible approver](#eligible-approvers) approves a merge request, it will reduce the number of approvals left for all rules that the approver belongs to. - + #### Scoped to Protected Branch **(PREMIUM)** @@ -221,7 +238,7 @@ or a [failed CI/CD pipeline](merge_when_pipeline_succeeds.md). ### Merge request approvals project settings The project settings for Merge request approvals are found by going to -**{settings}** **Settings > General** and expanding **Merge request approvals**. +**Settings > General** and expanding **Merge request approvals**. #### Prevent overriding default approvals diff --git a/doc/user/project/merge_requests/merge_request_dependencies.md b/doc/user/project/merge_requests/merge_request_dependencies.md index 65f3cb1e0d5..bca9df9ae2b 100644 --- a/doc/user/project/merge_requests/merge_request_dependencies.md +++ b/doc/user/project/merge_requests/merge_request_dependencies.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Source Code +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference, concepts --- diff --git a/doc/user/project/merge_requests/merge_when_pipeline_succeeds.md b/doc/user/project/merge_requests/merge_when_pipeline_succeeds.md index 7d90c9f3cd7..5151b28361a 100644 --- a/doc/user/project/merge_requests/merge_when_pipeline_succeeds.md +++ b/doc/user/project/merge_requests/merge_when_pipeline_succeeds.md @@ -84,7 +84,7 @@ merge-request-pipeline-job: ``` You should avoid configuration like this, and only use branch (`push`) pipelines -or merge request pipelines, when possible. See [`rules` documentation](../../../ci/yaml/README.md#differences-between-rules-and-onlyexcept) +or merge request pipelines, when possible. See [`rules` documentation](../../../ci/yaml/README.md#prevent-duplicate-pipelines) for details on avoiding two pipelines for a single merge request. ### Skipped pipelines diff --git a/doc/user/project/merge_requests/resolve_conflicts.md b/doc/user/project/merge_requests/resolve_conflicts.md index 265e47dd8bb..b6c7ad0ce75 100644 --- a/doc/user/project/merge_requests/resolve_conflicts.md +++ b/doc/user/project/merge_requests/resolve_conflicts.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Source Code +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference, concepts --- diff --git a/doc/user/project/merge_requests/revert_changes.md b/doc/user/project/merge_requests/revert_changes.md index 75d9702ceb9..bc4fee749e8 100644 --- a/doc/user/project/merge_requests/revert_changes.md +++ b/doc/user/project/merge_requests/revert_changes.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Source Code +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference, concepts --- diff --git a/doc/user/project/merge_requests/reviewing_and_managing_merge_requests.md b/doc/user/project/merge_requests/reviewing_and_managing_merge_requests.md index 91ca48f85d5..4e821145339 100644 --- a/doc/user/project/merge_requests/reviewing_and_managing_merge_requests.md +++ b/doc/user/project/merge_requests/reviewing_and_managing_merge_requests.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Source Code +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: index, reference --- @@ -132,7 +135,7 @@ specific commit page.  ->**Tip:** +TIP: **Tip:** You can append `?w=1` while on the diffs page of a merge request to ignore any whitespace changes. @@ -141,10 +144,53 @@ whitespace changes. > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/13950) in GitLab 11.5. GitLab provides a way of leaving comments in any part of the file being changed -in a Merge Request. To do so, click the **...** button in the gutter of the Merge Request diff UI to expand the diff lines and leave a comment, just as you would for a changed line. +in a Merge Request. To do so, click the **{comment}** **comment** icon in the gutter of the Merge Request diff UI to expand the diff lines and leave a comment, just as you would for a changed line.  +### Commenting on multiple lines + +> - [Introduced](https://gitlab.com/gitlab-org/ux-research/-/issues/870) in GitLab 13.2. +> - It's deployed behind a feature flag, enabled by default. +> - [Became enabled by default](https://gitlab.com/gitlab-org/gitlab/-/issues/221268) on GitLab 13.3. +> - It's enabled on GitLab.com. +> - It can be disabled or enabled per-project. +> - It's recommended for production use. +> - For GitLab self-managed instances, GitLab administrators can opt to [disable it](#enable-or-disable-multiline-comments-core-only). **(CORE ONLY)** + +GitLab provides a way to select which lines of code a comment refers to. After starting a comment +a dropdown selector is shown to select the first line that this comment refers to. +The last line is the line that the comment icon was initially clicked on. + +New comments default to single line comments by having the first and last lines +the same. Selecting a different starting line turns this into a multiline comment. + + + +Once a multiline comment is saved the lines of code pertaining to that comment are listed directly +above it. + + + +### Enable or disable multiline comments **(CORE ONLY)** + +The multiline comments feature is under development but ready for production use. +It is deployed behind a feature flag that is **disabled by default**. +[GitLab administrators with access to the GitLab Rails console](../../../administration/feature_flags.md) +can opt to enable it for your instance. + +To disable it: + +```ruby +Feature.disable(:multiline_comments) +``` + +To enable it: + +```ruby +Feature.enable(:multiline_comments) +``` + ## Pipeline status in merge requests widgets If you've set up [GitLab CI/CD](../../../ci/README.md) in your project, @@ -235,7 +281,7 @@ Merge Request again. Here are some tips that will help you be more efficient with merge requests in the command line. -> **Note:** +NOTE: **Note:** This section might move in its own document in the future. ### Checkout merge requests locally diff --git a/doc/user/project/merge_requests/squash_and_merge.md b/doc/user/project/merge_requests/squash_and_merge.md index 79eec059293..7f752b2cc39 100644 --- a/doc/user/project/merge_requests/squash_and_merge.md +++ b/doc/user/project/merge_requests/squash_and_merge.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Source Code +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference, concepts --- @@ -24,6 +27,10 @@ Into a single commit on merge:  +NOTE: **Note:** +The squashed commit in this example is followed by a merge commit, because the merge method for this repository uses a merge commit. You can disable merge commits in +**Project Settings > General > Merge requests > Merge method > Fast-forward merge**. + The squashed commit's commit message will be either: - Taken from the first multi-line commit message in the merge. @@ -36,9 +43,6 @@ It can be customized before merging a merge request.  -NOTE: **Note:** -The squashed commit in this example is followed by a merge commit, as the merge method for this example repository uses a merge commit. - Squashing also works with the fast-forward merge strategy, see [squashing and fast-forward merge](#squash-and-fast-forward-merge) for more details. ## Use cases @@ -88,10 +92,12 @@ squashing can itself be considered equivalent to rebasing. ## Squash Commits Options > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/17613) in GitLab 13.2. -> - It's deployed behind a feature flag, disabled by default. -> - It's disabled on GitLab.com. -> - It's not recommended for production use. -> - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enable-or-disable-squash-commit-options-core-only). **(CORE ONLY)** +> - It was deployed behind a feature flag, disabled by default. +> - [Became enabled by default](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/39382) on GitLab 13.3. +> - It's enabled on GitLab.com. +> - It can be enabled per project. +> - It's recommended for production use. +> - For GitLab self-managed instances, GitLab administrators can opt to [disable it](#enable-or-disable-squash-commit-options-core-only). **(CORE ONLY)** With Squash Commits Options you can configure the behavior of Squash and Merge for your project. To set it up, navigate to your project's **Settings > General** and expand **Merge requests**. @@ -116,10 +122,10 @@ squash commits locally through the command line and force-push to their remote b ### Enable or disable Squash Commit Options **(CORE ONLY)** -Squash Commit Options is under development and not ready for production use. It is -deployed behind a feature flag that is **disabled by default**. +Squash Commit Options is under development but ready for production use. It is +deployed behind a feature flag that is **enabled by default**. [GitLab administrators with access to the GitLab Rails console](../../../administration/feature_flags.md) -can enable it for your instance. Squash Commit Options can be enabled or disabled per-project. +can opt to disable it. To enable it: diff --git a/doc/user/project/merge_requests/test_coverage_visualization.md b/doc/user/project/merge_requests/test_coverage_visualization.md index 12194423a00..6751dde155c 100644 --- a/doc/user/project/merge_requests/test_coverage_visualization.md +++ b/doc/user/project/merge_requests/test_coverage_visualization.md @@ -5,9 +5,13 @@ info: To determine the technical writer assigned to the Stage/Group associated w type: reference, howto --- -# Test Coverage Visualization +# Test Coverage Visualization **(CORE ONLY)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/3708) in GitLab 12.9. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/3708) in GitLab 12.9. +> - It's deployed behind a feature flag, disabled by default. +> - It's disabled on GitLab.com. +> - It can be enabled or disabled per-project. +> - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enabling-the-feature). **(CORE ONLY)** With the help of [GitLab CI/CD](../../../ci/README.md), you can collect the test coverage information of your favorite testing or coverage-analysis tool, and visualize @@ -50,6 +54,10 @@ from any job in any stage in the pipeline. The coverage will be displayed for ea Hovering over the coverage bar will provide further information, such as the number of times the line was checked by tests. +NOTE: **Note:** +The Cobertura XML parser currently does not support the `sources` element and ignores it. It is assumed that +the `filename` of a `class` element contains the full path relative to the project root. + ## Example test coverage configuration The following [`gitlab-ci.yml`](../../../ci/yaml/README.md) example uses [Mocha](https://mochajs.org/) @@ -69,13 +77,25 @@ test: ## Enabling the feature This feature comes with the `:coverage_report_view` feature flag disabled by -default. This feature is disabled due to some performance issues with very large +default. It is disabled on GitLab.com. This feature is disabled due to some performance issues with very large data sets. When [the performance issue](https://gitlab.com/gitlab-org/gitlab/-/issues/211410) -is resolved, the feature will be enabled by default. +is resolved, the feature will be enabled by default. [GitLab administrators with access to the GitLab Rails console](../../../administration/feature_flags.md) +can enable it for your instance. Test coverage visualization can be enabled or disabled per-project. -To enable this feature, ask a GitLab administrator with Rails console access to -run the following command: +To enable it: ```ruby +# Instance-wide Feature.enable(:coverage_report_view) +# or by project +Feature.enable(:coverage_report_view, Project.find(<project id>)) +``` + +To disable it: + +```ruby +# Instance-wide +Feature.disable(:coverage_report_view) +# or by project +Feature.disable(:coverage_report_view, Project.find(<project id>)) ``` diff --git a/doc/user/project/merge_requests/work_in_progress_merge_requests.md b/doc/user/project/merge_requests/work_in_progress_merge_requests.md index ece5e7868dc..d1833318a85 100644 --- a/doc/user/project/merge_requests/work_in_progress_merge_requests.md +++ b/doc/user/project/merge_requests/work_in_progress_merge_requests.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Source Code +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference, concepts --- diff --git a/doc/user/project/milestones/burndown_charts.md b/doc/user/project/milestones/burndown_charts.md index e28ba1d2d5f..0c8bba831a9 100644 --- a/doc/user/project/milestones/burndown_charts.md +++ b/doc/user/project/milestones/burndown_charts.md @@ -35,7 +35,7 @@ Burndown Charts are generally used for tracking and analyzing the completion of a milestone. Therefore, their use cases are tied to the [use you are assigning your milestone to](index.md). -To exemplify, suppose you lead a team of developers in a large company, +For example, suppose you lead a team of developers in a large company, and you follow this workflow: - Your company set the goal for the quarter to deliver 10 new features for your app diff --git a/doc/user/project/operations/alert_management.md b/doc/user/project/operations/alert_management.md index 3e4b94473bc..0feed7dbf40 100644 --- a/doc/user/project/operations/alert_management.md +++ b/doc/user/project/operations/alert_management.md @@ -1,274 +1,5 @@ --- -stage: Monitor -group: Health -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +redirect_to: '../../../operations/incident_management/index.md' --- -# Alert Management - -> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/2877) in GitLab 13.0. - -Alert Management enables developers to easily discover and view the alerts -generated by their application. By surfacing alert information where the code is -being developed, efficiency and awareness can be increased. - -## Enable Alert Management - -NOTE: **Note:** -You will need at least Maintainer [permissions](../../permissions.md) to enable the Alert Management feature. - -There are several ways to accept alerts into your GitLab project, as outlined below. -Enabling any of these methods will allow the Alerts list to display. After configuring -alerts, visit **{cloud-gear}** **Operations > Alerts** in your project's sidebar -to [view the list](#alert-management-list) of alerts. - -### Opsgenie integration **(PREMIUM)** - -> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3066) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.2. - -A new way of monitoring Alerts via a GitLab integration is with -[Opsgenie](https://www.atlassian.com/software/opsgenie). - -NOTE: **Note:** -If you enable the Opsgenie integration, you cannot have other GitLab alert services, -such as [Generic Alerts](../integrations/generic_alerts.md) or -Prometheus alerts, active at the same time. - -To enable Opsgenie integration: - -1. Sign in as a user with Maintainer or Owner [permissions](../../permissions.md). -1. Navigate to **{cloud-gear}** **Operations > Alerts**. -1. In the **Integrations** select box, select Opsgenie. -1. Click the **Active** toggle. -1. In the **API URL**, enter the base URL for your Opsgenie integration, such - as `https://app.opsgenie.com/alert/list`. -1. Click **Save changes**. - -After enabling the integration, navigate to the Alerts list page at **{cloud-gear}** **Operations > Alerts**, and click **View alerts in Opsgenie**. - -### Enable a Generic Alerts endpoint - -GitLab provides the Generic Alerts endpoint so you can accept alerts from a third-party -alerts service. See the -[instructions for toggling generic alerts](../integrations/generic_alerts.md#setting-up-generic-alerts) -to add this option. After configuring the endpoint, the -[Alerts list](#alert-management-list) is enabled. - -To populate the alerts with data, see [Customizing the payload](../integrations/generic_alerts.md#customizing-the-payload) for requests to the alerts endpoint. - -### Enable GitLab-managed Prometheus alerts - -You can install the GitLab-managed Prometheus application on your Kubernetes -cluster. For more information, see -[Managed Prometheus on Kubernetes](../integrations/prometheus.md#managed-prometheus-on-kubernetes). -When GitLab-managed Prometheus is installed, the [Alerts list](#alert-management-list) -is also enabled. - -To populate the alerts with data, see -[GitLab-Managed Prometheus instances](../../../operations/metrics/alerts.md#managed-prometheus-instances). - -### Enable external Prometheus alerts - -You can configure an externally-managed Prometheus instance to send alerts -to GitLab. To set up this configuration, see the [configuring Prometheus](../../../operations/metrics/alerts.md#external-prometheus-instances) documentation. Activating the external Prometheus -configuration also enables the [Alerts list](#alert-management-list). - -To populate the alerts with data, see -[External Prometheus instances](../../../operations/metrics/alerts.md#external-prometheus-instances). - -## Alert Management severity - -Each level of alert contains a uniquely shaped and color-coded icon to help -you identify the severity of a particular alert. These severity icons help you -immediately identify which alerts you should prioritize investigating: - - - -Alerts contain one of the following icons: - -| Severity | Icon | Color (hexadecimal) | -|---|---|---| -| Critical | **{severity-critical}** | `#8b2615` | -| High | **{severity-high}** | `#c0341d` | -| Medium | **{severity-medium}** | `#fca429` | -| Low | **{severity-low}** | `#fdbc60` | -| Info | **{severity-info}** | `#418cd8` | -| Unknown | **{severity-unknown}** | `#bababa` | - -## Alert Management list - -NOTE: **Note:** -You will need at least Developer [permissions](../../permissions.md) to view the Alert Management list. - -You can find the Alert Management list at **{cloud-gear}** **Operations > Alerts** in your project's sidebar. -Each alert contains the following metrics: - - - -- **Severity** - The current importance of a alert and how much attention it should receive. -- **Start time** - How long ago the alert fired. This field uses the standard GitLab pattern of `X time ago`, but is supported by a granular date/time tooltip depending on the user's locale. -- **Alert description** - The description of the alert, which attempts to capture the most meaningful data. -- **Event count** - The number of times that an alert has fired. -- **Issue** - A link to the incident issue that has been created for the alert. -- **Status** - The [current status](#alert-management-statuses) of the alert. - -### Alert Management list sorting - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/217745) in GitLab 13.1. - -The Alert Management list displays alerts sorted by start time, but you can -change the sort order by clicking the headers in the Alert Management list. - -To see if a column is sortable, point your mouse at the header. Sortable columns -display an arrow next to the column name, as shown in this example: - - - -### Searching alerts - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/213884) in GitLab 13.1. - -The alert list supports a simple free text search. - - - -This search filters on the following fields: - -- Title -- Description -- Monitoring tool -- Service - -### Alert Management statuses - -Each alert contains a status dropdown to indicate which alerts need investigation. -Standard alert statuses include `triggered`, `acknowledged`, and `resolved`: - -- **Triggered**: No one has begun investigation. -- **Acknowledged**: Someone is actively investigating the problem. -- **Resolved**: No further work is required. - -## Alert Management details - -NOTE: **Note:** -You will need at least Developer [permissions](../../permissions.md) to view Alert Management details. - -Navigate to the Alert Management detail view by visiting the [Alert Management list](#alert-management-list) and selecting an Alert from the list. - - - - - -### Update an Alert's status - -The Alert Management detail view enables you to update the Alert Status. -See [Alert Management statuses](#alert-management-statuses) for more details. - -### Create an Issue from an Alert - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/217745) in GitLab 13.1. - -The Alert Management detail view enables you to create an issue with a -description automatically populated from an alert. To create the issue, -click the **Create Issue** button. You can then view the issue from the -alert by clicking the **View Issue** button. - -Closing a GitLab issue associated with an alert changes the alert's status to Resolved. -See [Alert Management statuses](#alert-management-statuses) for more details about statuses. - -### Update an Alert's assignee - -> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3066) in GitLab 13.1. - -The Alert Management detail view allows users to update the Alert assignee. - -In large teams, where there is shared ownership of an alert, it can be difficult -to track who is investigating and working on it. The Alert Management detail view -enables you to update the Alert assignee: - -NOTE: **Note:** -GitLab currently only supports a single assignee per alert. - -1. To display the list of current alerts, click - **{cloud-gear}** **Operations > Alerts**: - -  - -1. Select your desired alert to display its **Alert Management Details View**: - -  - -1. If the right sidebar is not expanded, click - **{angle-double-right}** **Expand sidebar** to expand it. -1. In the right sidebar, locate the **Assignee** and click **Edit**. From the - dropdown menu, select each user you want to assign to the alert. GitLab creates - a [To-Do list item](../../todos.md) for each user. - -  - -To remove an assignee, click **Edit** next to the **Assignee** dropdown menu and -deselect the user from the list of assignees, or click **Unassigned**. - -### Alert system notes - -> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3066) in GitLab 13.1. - -When you take action on an alert, this is logged as a system note, -which is visible in the Alert Details view. This gives you a linear -timeline of the alert's investigation and assignment history. - -The following actions will result in a system note: - -- [Updating the status of an alert](#update-an-alerts-status) -- [Creating an issue based on an alert](#create-an-issue-from-an-alert) -- [Assignment of an alert to a user](#update-an-alerts-assignee) - - - -### View an Alert's metrics data - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/217768) in GitLab 13.2. - -To view the metrics for an alert: - - 1. Sign in as a user with Developer or higher [permissions](../../permissions.md). - 1. Navigate to **{cloud-gear}** **Operations > Alerts**. - 1. Click the alert you want to view. - 1. Below the title of the alert, click the **Metrics** tab. - - - -For GitLab-managed Prometheus instances, metrics data is automatically available -for the alert, making it easy to see surrounding behavior. See -[Managed Prometheus instances](../../../operations/metrics/alerts.md#managed-prometheus-instances) -for information on setting up alerts. - -For externally-managed Prometheus instances, you can configure your alerting rules to -display a chart in the alert. See -[Embedding metrics based on alerts in incident issues](../../../operations/metrics/embed.md#embedding-metrics-based-on-alerts-in-incident-issues) -for information on how to appropriately configure your alerting rules. See -[External Prometheus instances](../../../operations/metrics/alerts.md#external-prometheus-instances) -for information on setting up alerts for your self-managed Prometheus instance. - -## Use cases for assigning alerts - -Consider a team formed by different sections of monitoring, collaborating on a -single application. After an alert surfaces, it's extremely important to -route the alert to the team members who can address and resolve the alert. - -Assigning Alerts to multiple assignees eases collaboration and delegation. All -assignees are shown in your team's work-flows, and all assignees receive -notifications, simplifying communication and ownership of the alert. - -After completing their portion of investigating or fixing the alert, users can -unassign their account from the alert when their role is complete. -The [alerts status](#alert-management-statuses) can be updated to -reflect if the alert has been resolved. - -### Slack Notifications - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/216326) in GitLab 13.1. - -You can be alerted via a Slack message when a new alert has been received. - -See the [Slack Notifications Service docs](../integrations/slack.md) for information on how to set this up. +This document was moved to [another location](../../../operations/incident_management/index.md). diff --git a/doc/user/project/operations/dashboard_settings.md b/doc/user/project/operations/dashboard_settings.md index f30ce5024d6..2640f2cf98f 100644 --- a/doc/user/project/operations/dashboard_settings.md +++ b/doc/user/project/operations/dashboard_settings.md @@ -1,52 +1,5 @@ --- -stage: Monitor -group: APM -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +redirect_to: '../../../operations/metrics/dashboards/settings.md' --- -# Metrics dashboard settings - -You can configure your [Monitoring dashboard](../integrations/prometheus.md) to -display the time zone of your choice, and the links of your choice. - -To configure these settings you must have Manage Project -Operations [permissions](../../permissions.md). - -## Change the dashboard time zone - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/214370) in GitLab 13.1. - -By default, your monitoring dashboard displays dates and times in your local -time zone, but you can display dates and times in UTC format. To change the -time zone: - -1. Sign in as a user with Manage Project Operations [permissions](../../permissions.md). -1. Navigate to **{settings}** **Settings > Operations**, and scroll to - **Metrics Dashboard**. -1. In the **Dashboard timezone** select box, select *User's local timezone* - or *UTC*: - -  -1. Click **Save changes**. - -## Link to an external dashboard - -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/57171) in GitLab 12.0. - -You can add a button on your monitoring dashboard that links directly to your -existing external dashboards: - -1. Sign in as a user with Manage Project Operations [permissions](../../permissions.md). -1. Navigate to **{settings}** **Settings > Operations**, and scroll to - **Metrics Dashboard**. -1. In **External dashboard URL**, provide the URL to your external dashboard: - -  - -1. Click **Save changes**. - -GitLab displays a **View full dashboard** button in the top right corner of your -[monitoring dashboard](../../../ci/environments/index.md#monitoring-environments) -which opens the URL you provided: - - +This document was moved to [another location](../../../operations/metrics/dashboards/settings.md). diff --git a/doc/user/project/operations/error_tracking.md b/doc/user/project/operations/error_tracking.md index a7a16de90e0..3c00c4a6c41 100644 --- a/doc/user/project/operations/error_tracking.md +++ b/doc/user/project/operations/error_tracking.md @@ -1,95 +1,5 @@ --- -stage: Monitor -group: Health -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +redirect_to: '../../../operations/error_tracking.md' --- -# Error Tracking - -> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/169) in GitLab 11.8. - -Error tracking allows developers to easily discover and view the errors that their application may be generating. By surfacing error information where the code is being developed, efficiency and awareness can be increased. - -## Sentry error tracking - -[Sentry](https://sentry.io/) is an open source error tracking system. GitLab allows administrators to connect Sentry to GitLab, to allow users to view a list of Sentry errors in GitLab. - -### Deploying Sentry - -You may sign up to the cloud hosted <https://sentry.io>, deploy your own [on-premise instance](https://docs.sentry.io/server/installation/) or use GitLab to [install Sentry to a Kubernetes cluster](../../clusters/applications.md#install-sentry-using-gitlab-cicd). - -### Enabling Sentry - -NOTE: **Note:** -You will need at least Maintainer [permissions](../../permissions.md) to enable the Sentry integration. - -GitLab provides an easy way to connect Sentry to your project: - -1. Sign up to Sentry.io or [deploy your own](#deploying-sentry) Sentry instance. -1. [Create](https://docs.sentry.io/guides/integrate-frontend/create-new-project/) a new Sentry project. For each GitLab project that you want to integrate, we recommend that you create a new Sentry project. -1. [Find or generate](https://docs.sentry.io/api/auth/) a Sentry auth token for your Sentry project. - Make sure to give the token at least the following scopes: `event:read` and `project:read`. -1. Navigate to your project’s **Settings > Operations**. -1. Ensure that the **Active** checkbox is set. -1. In the **Sentry API URL** field, enter your Sentry hostname. For example, enter `https://sentry.example.com` if this is the address at which your Sentry instance is available. For the SaaS version of Sentry, the hostname will be `https://sentry.io`. -1. In the **Auth Token** field, enter the token you previously generated. -1. Click the **Connect** button to test the connection to Sentry and populate the **Project** dropdown. -1. From the **Project** dropdown, choose a Sentry project to link to your GitLab project. -1. Click **Save changes** for the changes to take effect. -1. You can now visit **Operations > Error Tracking** in your project's sidebar to [view a list](#error-tracking-list) of Sentry errors. - -### Enabling GitLab issues links - -You may also want to enable Sentry's GitLab integration by following the steps in the [Sentry documentation](https://docs.sentry.io/workflow/integrations/global-integrations/#gitlab) - -## Error Tracking List - -NOTE: **Note:** -You will need at least Reporter [permissions](../../permissions.md) to view the Error Tracking list. - -You can find the Error Tracking list at **Operations > Error Tracking** in your project's sidebar. -Here, you can filter errors by title or by status (one of Ignored , Resolved, or Unresolved) and sort in descending order by Frequency, First Seen, or Last Seen. By default, the error list is ordered by Last Seen and filtered to Unresolved errors. - - - -## Error Details - -From error list, users can navigate to the error details page by clicking the title of any error. - -This page has: - -- A link to the Sentry issue. -- A link to the GitLab commit if the Sentry [release ID/version](https://docs.sentry.io/workflow/releases/?platform=javascript#configure-sdk) on the Sentry Issue's first release matches a commit SHA in your GitLab hosted project. -- Other details about the issue, including a full stack trace. -- In [GitLab 12.7 and newer](https://gitlab.com/gitlab-org/gitlab/-/issues/36246), language and urgency are displayed. - -By default, a **Create issue** button is displayed: - - - -If you create a GitLab issue from the error, the **Create issue** button will change to a **View issue** button and a link to the GitLab issue will surface within the error detail section: - - - -## Taking Action on errors - -You can take action on Sentry Errors from within the GitLab UI. - -### Ignoring errors - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/39665) in GitLab 12.7. - -From within the [Error Details](#error-details) page you can ignore a Sentry error by simply clicking the **Ignore** button near the top of the page. - -Ignoring an error will prevent it from appearing in the [Error Tracking List](#error-tracking-list), and will silence notifications that were set up within Sentry. - -### Resolving errors - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/39825) in GitLab 12.7. - -From within the [Error Details](#error-details) page you can resolve a Sentry error by -clicking the **Resolve** button near the top of the page. - -Marking an error as resolved indicates that the error has stopped firing events. If a GitLab issue is linked to the error, then the issue will be closed. - -If another event occurs, the error reverts to unresolved. +This document was moved to [another location](../../../operations/error_tracking.md). diff --git a/doc/user/project/operations/linking_to_an_external_dashboard.md b/doc/user/project/operations/linking_to_an_external_dashboard.md index ff609a3720e..2640f2cf98f 100644 --- a/doc/user/project/operations/linking_to_an_external_dashboard.md +++ b/doc/user/project/operations/linking_to_an_external_dashboard.md @@ -1,5 +1,5 @@ --- -redirect_to: 'dashboard_settings.md' +redirect_to: '../../../operations/metrics/dashboards/settings.md' --- -This document was moved to [another location](dashboard_settings.md). +This document was moved to [another location](../../../operations/metrics/dashboards/settings.md). diff --git a/doc/user/project/pages/custom_domains_ssl_tls_certification/dns_concepts.md b/doc/user/project/pages/custom_domains_ssl_tls_certification/dns_concepts.md index 6e48afad96a..735d27ec04d 100644 --- a/doc/user/project/pages/custom_domains_ssl_tls_certification/dns_concepts.md +++ b/doc/user/project/pages/custom_domains_ssl_tls_certification/dns_concepts.md @@ -32,7 +32,7 @@ for the most popular hosting services: - [Amazon](https://docs.aws.amazon.com/AmazonS3/latest/dev/website-hosting-custom-domain-walkthrough.html) - [Bluehost](https://www.bluehost.com/help/article/dns-management-add-edit-or-delete-dns-entries) -- [CloudFlare](https://support.cloudflare.com/hc/en-us/articles/201720164-Creating-a-Cloudflare-account-and-adding-a-website) +- [Cloudflare](https://support.cloudflare.com/hc/en-us/articles/201720164-Creating-a-Cloudflare-account-and-adding-a-website) - [cPanel](https://documentation.cpanel.net/display/84Docs/Edit+DNS+Zone) - [DreamHost](https://help.dreamhost.com/hc/en-us/articles/215414867-How-do-I-add-custom-DNS-records-) - [Go Daddy](https://www.godaddy.com/help/add-an-a-record-19238) diff --git a/doc/user/project/pages/custom_domains_ssl_tls_certification/index.md b/doc/user/project/pages/custom_domains_ssl_tls_certification/index.md index 0425ca56285..8e8f75be82d 100644 --- a/doc/user/project/pages/custom_domains_ssl_tls_certification/index.md +++ b/doc/user/project/pages/custom_domains_ssl_tls_certification/index.md @@ -1,5 +1,5 @@ --- -last_updated: 2019-07-04 +last_updated: 2020-07-25 type: reference, howto disqus_identifier: 'https://docs.gitlab.com/ee/user/project/pages/getting_started_part_three.html' stage: Release @@ -129,11 +129,11 @@ They require: | ------------------------------------------------- | ---------- | ---------------------- | | `example.com` | A | `35.185.44.232` | | `_gitlab-pages-verification-code.example.com` | TXT | `gitlab-pages-verification-code=00112233445566778899aabbccddeeff` | -|--------------------------------------------+--------------------------------------------| +|---------------------------------------------------+------------+------------------------| | `www.example.com` | CNAME | `namespace.gitlab.io` | | `_gitlab-pages-verification-code.www.example.com` | TXT | `gitlab-pages-verification-code=00112233445566778899aabbccddeeff` | -If you're using CloudFlare, check +If you're using Cloudflare, check [Redirecting `www.domain.com` to `domain.com` with Cloudflare](#redirecting-wwwdomaincom-to-domaincom-with-cloudflare). > **Notes**: @@ -236,7 +236,9 @@ To secure your custom domain with GitLab Pages you can opt by: - Manually adding SSL/TLS certificates to GitLab Pages websites by following the steps below. -### Requirements +### Manual addition of SSL/TLS certificates + +You can use any certificate satisfying the following requirements: - A GitLab Pages website up and running accessible via a custom domain. - **A PEM certificate**: it is the certificate generated by the CA, @@ -245,12 +247,15 @@ To secure your custom domain with GitLab Pages you can opt by: the part of the encryption keychain that identifies the CA. Usually it's combined with the PEM certificate, but there are some cases in which you need to add them manually. - [CloudFlare certs](https://about.gitlab.com/blog/2017/02/07/setting-up-gitlab-pages-with-cloudflare-certificates/) + [Cloudflare certs](https://about.gitlab.com/blog/2017/02/07/setting-up-gitlab-pages-with-cloudflare-certificates/) are one of these cases. - **A private key**, it's an encrypted key which validates your PEM against your domain. -### Steps +NOTE: **Note:** +[Cloudflare certificates](https://about.gitlab.com/blog/2017/02/07/setting-up-gitlab-pages-with-cloudflare-certificates/), for example, meet these requirements. + +#### Steps - To add the certificate at the time you add a new domain, go to your project's **Settings > Pages > New Domain**, add the domain name and the certificate. @@ -288,7 +293,7 @@ To enable this setting: 1. Tick the checkbox **Force HTTPS (requires valid certificates)**. NOTE: **Note:** -If you use CloudFlare CDN in front of GitLab Pages, make sure to set the SSL connection setting to `full` instead of `flexible`. For more details, see the [CloudFlare CDN directions](https://support.cloudflare.com/hc/en-us/articles/200170416-End-to-end-HTTPS-with-Cloudflare-Part-3-SSL-options#h_4e0d1a7c-eb71-4204-9e22-9d3ef9ef7fef). +If you use Cloudflare CDN in front of GitLab Pages, make sure to set the SSL connection setting to `full` instead of `flexible`. For more details, see the [Cloudflare CDN directions](https://support.cloudflare.com/hc/en-us/articles/200170416-End-to-end-HTTPS-with-Cloudflare-Part-3-SSL-options#h_4e0d1a7c-eb71-4204-9e22-9d3ef9ef7fef). <!-- ## Troubleshooting diff --git a/doc/user/project/pages/custom_domains_ssl_tls_certification/ssl_tls_concepts.md b/doc/user/project/pages/custom_domains_ssl_tls_certification/ssl_tls_concepts.md index e307b807aaa..f30361e6669 100644 --- a/doc/user/project/pages/custom_domains_ssl_tls_certification/ssl_tls_concepts.md +++ b/doc/user/project/pages/custom_domains_ssl_tls_certification/ssl_tls_concepts.md @@ -56,7 +56,7 @@ reiterating the importance of HTTPS. ## Issuing Certificates -GitLab Pages accepts certificates provided in the [PEM](https://support.quovadisglobal.com/kb/a37/what-is-pem-format.aspx) format, issued by +GitLab Pages accepts certificates provided in the [PEM](https://knowledge.digicert.com/quovadis) format, issued by [Certificate Authorities](https://en.wikipedia.org/wiki/Certificate_authority) or as [self-signed certificates](https://en.wikipedia.org/wiki/Self-signed_certificate). Note that [self-signed certificates are typically not used](https://www.mcafee.com/blogs/other-blogs/mcafee-labs/self-signed-certificates-secure-so-why-ban/) for public websites for security reasons and to ensure that browsers trust your site's certificate. @@ -72,7 +72,7 @@ to everyone. The most popular is [Let's Encrypt](https://letsencrypt.org/), which issues certificates trusted by most of browsers, it's open source, and free to use. See [GitLab Pages integration with Let's Encrypt](../custom_domains_ssl_tls_certification/lets_encrypt_integration.md) to enable HTTPS on your custom domain. -Similarly popular are [certificates issued by CloudFlare](https://www.cloudflare.com/ssl/), +Similarly popular are [certificates issued by Cloudflare](https://www.cloudflare.com/ssl/), which also offers a [free CDN service](https://blog.cloudflare.com/cloudflares-free-cdn-and-you/). Their certs are valid up to 15 years. See the tutorial on -[how to add a CloudFlare Certificate to your GitLab Pages website](https://about.gitlab.com/blog/2017/02/07/setting-up-gitlab-pages-with-cloudflare-certificates/). +[how to add a Cloudflare Certificate to your GitLab Pages website](https://about.gitlab.com/blog/2017/02/07/setting-up-gitlab-pages-with-cloudflare-certificates/). diff --git a/doc/user/project/pages/getting_started/pages_from_scratch.md b/doc/user/project/pages/getting_started/pages_from_scratch.md index 7278c734b07..cabaf734d77 100644 --- a/doc/user/project/pages/getting_started/pages_from_scratch.md +++ b/doc/user/project/pages/getting_started/pages_from_scratch.md @@ -160,8 +160,9 @@ When it succeeds, go to **Settings > Pages** to view the URL where your site is now available. If you want to do more advanced tasks, you can update your `.gitlab-ci.yml` file -with [any of the available settings](../../../../ci/yaml/README.md). You can check -your CI syntax with the [GitLab CI/CD Lint Tool](../../../../ci/yaml/README.md#validate-the-gitlab-ciyml). +with [any of the available settings](../../../../ci/yaml/README.md). See +[Validate the `.gitlab-ci.yml`](../../../../ci/yaml/README.md#validate-the-gitlab-ciyml) +for instructions on validating your YAML file with the Lint tool included with GitLab. The following topics show other examples of other options you can add to your CI/CD file. diff --git a/doc/user/project/pages/getting_started_part_one.md b/doc/user/project/pages/getting_started_part_one.md index c0bdd4b7f0b..949a6c16c1b 100644 --- a/doc/user/project/pages/getting_started_part_one.md +++ b/doc/user/project/pages/getting_started_part_one.md @@ -13,7 +13,7 @@ according to your intended website's URL. ## GitLab Pages default domain names ->**Note:** +NOTE: **Note:** If you use your own GitLab instance to deploy your site with GitLab Pages, check with your sysadmin what's your Pages wildcard domain. This guide is valid for any GitLab instance, diff --git a/doc/user/project/pages/img/change_path_v12_10.png b/doc/user/project/pages/img/change_path_v12_10.png Binary files differindex 9b5c17f1dda..b6ed011c6d9 100644 --- a/doc/user/project/pages/img/change_path_v12_10.png +++ b/doc/user/project/pages/img/change_path_v12_10.png diff --git a/doc/user/project/pages/img/choose_ci_template_v13_1.png b/doc/user/project/pages/img/choose_ci_template_v13_1.png Binary files differindex 84dd9fe2e0f..a0c25ba1642 100644 --- a/doc/user/project/pages/img/choose_ci_template_v13_1.png +++ b/doc/user/project/pages/img/choose_ci_template_v13_1.png diff --git a/doc/user/project/pages/img/pages_project_templates_v13_1.png b/doc/user/project/pages/img/pages_project_templates_v13_1.png Binary files differindex 3f6d1ecd786..d67a82f0623 100644 --- a/doc/user/project/pages/img/pages_project_templates_v13_1.png +++ b/doc/user/project/pages/img/pages_project_templates_v13_1.png diff --git a/doc/user/project/pages/img/remove_fork_relationship_v13_1.png b/doc/user/project/pages/img/remove_fork_relationship_v13_1.png Binary files differindex 1bc7bcd253b..84aa2e571c7 100644 --- a/doc/user/project/pages/img/remove_fork_relationship_v13_1.png +++ b/doc/user/project/pages/img/remove_fork_relationship_v13_1.png diff --git a/doc/user/project/pages/index.md b/doc/user/project/pages/index.md index b116c28d94b..eff80a4c9bd 100644 --- a/doc/user/project/pages/index.md +++ b/doc/user/project/pages/index.md @@ -59,7 +59,6 @@ To update a GitLab Pages website: | [Explore GitLab Pages](introduction.md) | Requirements, technical aspects, specific GitLab CI/CD configuration options, Access Control, custom 404 pages, limitations, FAQ. | | [Custom domains and SSL/TLS Certificates](custom_domains_ssl_tls_certification/index.md) | Custom domains and subdomains, DNS records, and SSL/TLS certificates. | | [Let's Encrypt integration](custom_domains_ssl_tls_certification/lets_encrypt_integration.md) | Secure your Pages sites with Let's Encrypt certificates, which are automatically obtained and renewed by GitLab. | -| [CloudFlare certificates](https://about.gitlab.com/blog/2017/02/07/setting-up-gitlab-pages-with-cloudflare-certificates/) | Secure your Pages site with CloudFlare certificates. | Learn more and see examples: diff --git a/doc/user/project/protected_branches.md b/doc/user/project/protected_branches.md index d0baf57f169..3d0cb1bf3a5 100644 --- a/doc/user/project/protected_branches.md +++ b/doc/user/project/protected_branches.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" type: reference, howto --- diff --git a/doc/user/project/protected_tags.md b/doc/user/project/protected_tags.md index bb5bca39398..5dd2a72c91e 100644 --- a/doc/user/project/protected_tags.md +++ b/doc/user/project/protected_tags.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" type: reference, howto --- diff --git a/doc/user/project/push_options.md b/doc/user/project/push_options.md index ca658c06647..db9e2f270e0 100644 --- a/doc/user/project/push_options.md +++ b/doc/user/project/push_options.md @@ -1,5 +1,8 @@ --- -type: reference +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference, howto --- # Push Options diff --git a/doc/user/project/quick_actions.md b/doc/user/project/quick_actions.md index def05bf94e4..518cf472b50 100644 --- a/doc/user/project/quick_actions.md +++ b/doc/user/project/quick_actions.md @@ -44,7 +44,7 @@ The following quick actions are applicable to descriptions, discussions and thre | `/duplicate <#issue>` | ✓ | | | Close this issue and mark as a duplicate of another issue. **(CORE)** Also, mark both as related. **(STARTER)** | | `/epic <epic>` | ✓ | | | Add to epic `<epic>`. The `<epic>` value should be in the format of `&epic`, `group&epic`, or a URL to an epic. **(PREMIUM)** | | `/estimate <<W>w <DD>d <hh>h <mm>m>` | ✓ | ✓ | | Set time estimate. For example, `/estimate 1w 3d 2h 14m`. | -| `/iteration *iteration:iteration` | ✓ | | | Set iteration ([Introduced in GitLab 13.1](https://gitlab.com/gitlab-org/gitlab/-/issues/196795)) **(STARTER)** | +| `/iteration *iteration:"iteration name"` | ✓ | | | Set iteration. For example, to set the `Late in July` iteration: `/iteration *iteration:"Late in July"` ([introduced in GitLab 13.1](https://gitlab.com/gitlab-org/gitlab/-/issues/196795)). **(STARTER)** | | `/label ~label1 ~label2` | ✓ | ✓ | ✓ | Add one or more labels. Label names can also start without a tilde (`~`), but mixed syntax is not supported. | | `/lock` | ✓ | ✓ | | Lock the thread. | | `/merge` | | ✓ | | Merge changes. Depending on the project setting, this may be [when the pipeline succeeds](merge_requests/merge_when_pipeline_succeeds.md), adding to a [Merge Train](../../ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md), etc. | @@ -52,7 +52,7 @@ The following quick actions are applicable to descriptions, discussions and thre | `/move <path/to/project>` | ✓ | | | Move this issue to another project. | | `/parent_epic <epic>` | | | ✓ | Set parent epic to `<epic>`. The `<epic>` value should be in the format of `&epic`, `group&epic`, or a URL to an epic ([introduced in GitLab 12.1](https://gitlab.com/gitlab-org/gitlab/-/issues/10556)). **(ULTIMATE)** | | `/promote` | ✓ | | | Promote issue to epic. **(PREMIUM)** | -| `/publish` | ✓ | | | Publish issue to an associated [Status Page](status_page/index.md) ([Introduced in GitLab 13.0](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/30906)) **(ULTIMATE)** | +| `/publish` | ✓ | | | Publish issue to an associated [Status Page](../../operations/incident_management/status_page.md) ([Introduced in GitLab 13.0](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/30906)) **(ULTIMATE)** | | `/reassign @user1 @user2` | ✓ | ✓ | | Change assignee. **(STARTER)** | | `/relabel ~label1 ~label2` | ✓ | ✓ | ✓ | Replace existing labels with those specified. | | `/relate #issue1 #issue2` | ✓ | | | Mark issues as related. **(STARTER)** | @@ -60,7 +60,7 @@ The following quick actions are applicable to descriptions, discussions and thre | `/remove_due_date` | ✓ | | | Remove due date. | | `/remove_epic` | ✓ | | | Remove from epic. **(PREMIUM)** | | `/remove_estimate` | ✓ | ✓ | | Remove time estimate. | -| `/remove_iteration` | ✓ | | | Remove iteration ([Introduced in GitLab 13.1](https://gitlab.com/gitlab-org/gitlab/-/issues/196795)) **(STARTER)** | +| `/remove_iteration` | ✓ | | | Remove iteration ([introduced in GitLab 13.1](https://gitlab.com/gitlab-org/gitlab/-/issues/196795)) **(STARTER)** | | `/remove_milestone` | ✓ | ✓ | | Remove milestone. | | `/remove_parent_epic` | | | ✓ | Remove parent epic from epic ([introduced in GitLab 12.1](https://gitlab.com/gitlab-org/gitlab/-/issues/10556)). **(ULTIMATE)** | | `/remove_time_spent` | ✓ | ✓ | | Remove time spent. | diff --git a/doc/user/project/releases/img/deploy_freeze_v13_2.png b/doc/user/project/releases/img/deploy_freeze_v13_2.png Binary files differnew file mode 100644 index 00000000000..27d3a6044a1 --- /dev/null +++ b/doc/user/project/releases/img/deploy_freeze_v13_2.png diff --git a/doc/user/project/releases/img/milestone_list_with_releases_v12_5.png b/doc/user/project/releases/img/milestone_list_with_releases_v12_5.png Binary files differindex efe48058d9a..b9eb18e9279 100644 --- a/doc/user/project/releases/img/milestone_list_with_releases_v12_5.png +++ b/doc/user/project/releases/img/milestone_list_with_releases_v12_5.png diff --git a/doc/user/project/releases/img/release_with_milestone_v12_9.png b/doc/user/project/releases/img/release_with_milestone_v12_9.png Binary files differindex c828e36114a..4a904a854f1 100644 --- a/doc/user/project/releases/img/release_with_milestone_v12_9.png +++ b/doc/user/project/releases/img/release_with_milestone_v12_9.png diff --git a/doc/user/project/releases/img/releases_count_v13_2.png b/doc/user/project/releases/img/releases_count_v13_2.png Binary files differindex 1c0493326d1..d2d77e016bc 100644 --- a/doc/user/project/releases/img/releases_count_v13_2.png +++ b/doc/user/project/releases/img/releases_count_v13_2.png diff --git a/doc/user/project/releases/index.md b/doc/user/project/releases/index.md index 258601574ca..20880d0c4fa 100644 --- a/doc/user/project/releases/index.md +++ b/doc/user/project/releases/index.md @@ -53,25 +53,26 @@ Read more about [Release permissions](../../../user/permissions.md#project-membe You can create a release in the user interface, or by using the [Releases API](../../../api/releases/index.md#create-a-release). -We recommend using the API to add release notes as one of the last steps in your CI/CD release pipeline. +We recommend using the API to create releases as one of the last steps in your +CI/CD pipeline. To create a new release through the GitLab UI: -1. Navigate to **Project overview > Releases** and click the **New release** button. +1. Navigate to **Project overview > Releases** and click the **New release** + button. 1. In the [**Tag name**](#tag-name) box, enter a name. -1. In the **Create from** list, select the branch or enter a tag or commit SHA. -1. In the **Message** box, enter a message associated with the tag. -1. Optionally, in the [**Release notes**](#release-notes-description) - field, enter the release's description. You can use Markdown and drag and drop files to this field. - - If you leave this field empty, only a tag will be created. - - If you populate it, both a tag and a release will be created. -1. Click **Create tag**. -If you created a release, you can view it at **Project overview > Releases**. -If you created a tag, you can view it at **Repository > Tags**. + NOTE: **Note:** + Creating a release based on an existing tag using the user + interface is not yet supported. However, this is possible using the + [Releases API](../../../api/releases/index.md#create-a-release). -You can now edit the release to [add milestones](#associate-milestones-with-a-release) -and [release assets](#release-assets). +1. In the **Create from** list, select a branch, tag, or commit SHA to use when + creating the new tag. +1. Optionally, fill out any additional information about the release, such as its + [title](#title), [milestones](#associate-milestones-with-a-release), + [release notes](#release-notes-description), or [assets links](#links). +1. Click **Create release**. ### Schedule a future release @@ -176,7 +177,7 @@ Prevent unintended production releases during a period of time you specify by setting a [*deploy freeze* period](../../../ci/environments/deployment_safety.md). Deploy freezes help reduce uncertainty and risk when automating deployments. -Use the [Freeze Periods API](../../../api/freeze_periods.md) to set a `freeze_start` and a `freeze_end`, which +A maintainer can set a deploy freeze window in the user interface or by using the [Freeze Periods API](../../../api/freeze_periods.md) to set a `freeze_start` and a `freeze_end`, which are defined as [crontab](https://crontab.guru/) entries. If the job that's executing is within a freeze period, GitLab CI/CD creates an environment @@ -193,6 +194,22 @@ deploy_to_production: - if: $CI_DEPLOY_FREEZE == null ``` +To set a deploy freeze window in the UI, complete these steps: + +1. Sign in to GitLab as a user with project Maintainer [permissions](../../permissions.md). +1. Navigate to **Project overview**. +1. In the left navigation menu, navigate to **Settings > CI / CD**. +1. Scroll to **Deploy freezes**. +1. Click **Expand** to see the deploy freeze table. +1. Click **Add deploy freeze** to open the deploy freeze modal. +1. Enter the start time, end time, and timezone of the desired deploy freeze period. +1. Click **Add deploy freeze** in the modal. + + + +CAUTION: **Caution:** +To edit or delete a deploy freeze, use the [Freeze Periods API](../../../api/freeze_periods.md). + If a project contains multiple freeze periods, all periods apply. If they overlap, the freeze covers the complete overlapping period. @@ -202,6 +219,21 @@ For more information, see [Deployment safety](../../../ci/environments/deploymen The following fields are available when you create or edit a release. +### Title + +The release title can be customized using the **Release title** field when +creating or editing a release. If no title is provided, the release's tag name +is used instead. + +NOTE: **Note:** +Guest users of private projects are allowed to view the **Releases** page +but are _not_ allowed to view details about the Git repository (in particular, +tag names). Because of this, release titles are replaced with a generic +title like "Release-1234" for Guest users to avoid leaking tag name information. + +See the [Permissions](../../permissions.md#project-members-permissions) page for +more information about permissions. + ### Tag name The release tag name should include the release version. GitLab uses [Semantic Versioning](https://semver.org/) @@ -241,12 +273,12 @@ as pre-built packages, compliance/security evidence, or container images. > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/27300) in GitLab 12.9. The assets associated with a release are accessible through a permanent URL. -GitLab will always redirect this URL to the actual asset +GitLab always redirects this URL to the actual asset location, so even if the assets move to a different location, you can continue to use the same URL. This is defined during [link creation](../../../api/releases/links.md#create-a-link) or [updating](../../../api/releases/links.md#update-a-link). Each asset has a name, a URL of the *actual* asset location, and optionally, a -`filepath` parameter, which, if you specify it, will create a URL pointing +`filepath` parameter, which, if you specify it, creates a URL pointing to the asset for the Release. The format of the URL is: ```plaintext @@ -270,7 +302,7 @@ This asset has a direct link of: https://gitlab.com/gitlab-org/gitlab-runner/releases/v11.9.0-rc2/downloads/binaries/gitlab-runner-linux-amd64 ``` -The physical location of the asset can change at any time and the direct link will remain unchanged. +The physical location of the asset can change at any time and the direct link remains unchanged. ### Source code @@ -372,7 +404,7 @@ When a release is created, release evidence is automatically collected. To initi Evidence collection snapshots are visible on the Releases page, along with the timestamp the evidence was collected. -### Include report artifacts as release evidence **(ULTIMATE ONLY)** +### Include report artifacts as release evidence **(ULTIMATE)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/32773) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.2. diff --git a/doc/user/project/repository/branches/index.md b/doc/user/project/repository/branches/index.md index f94ca7ac106..54979d1c4ce 100644 --- a/doc/user/project/repository/branches/index.md +++ b/doc/user/project/repository/branches/index.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" type: concepts, howto --- @@ -63,7 +66,7 @@ By default, when you create a new project in GitLab, the initial branch is calle For self-managed instances, a GitLab administrator can customize the initial branch name to something else. This way, every new project created from then on will start from the custom branch name rather than `master`. To do so: -1. Go to the **{admin}** **Admin Area > Settings > Repository** and expand **Default initial +1. Go to the **Admin Area > Settings > Repository** and expand **Default initial branch name**. 1. Change the default initial branch to a custom name of your choice. 1. **Save Changes**. diff --git a/doc/user/project/repository/file_finder.md b/doc/user/project/repository/file_finder.md index ac10071e578..99319efbb7f 100644 --- a/doc/user/project/repository/file_finder.md +++ b/doc/user/project/repository/file_finder.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Editor +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers disqus_identifier: 'https://docs.gitlab.com/ee/workflow/file_finder.html' --- @@ -37,6 +40,7 @@ the `app/controllers/admin/deploy_keys_controller.rb` file. Using a fuzzy search, we start by typing letters that get us closer to the file. -**Tip:** To narrow down your search, include `/` in your search terms. +TIP: **Tip:** +To narrow down your search, include `/` in your search terms.  diff --git a/doc/user/project/repository/forking_workflow.md b/doc/user/project/repository/forking_workflow.md index 75a84e36169..e90f0a7354c 100644 --- a/doc/user/project/repository/forking_workflow.md +++ b/doc/user/project/repository/forking_workflow.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Source Code +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers disqus_identifier: 'https://docs.gitlab.com/ee/workflow/forking_workflow.html' --- @@ -26,7 +29,7 @@ Forking a project is, in most cases, a two-step process. NOTE: **Note:** The project path must be unique within the namespace. -  +  The fork is created. The permissions you have in the namespace are the permissions you will have in the fork. @@ -50,7 +53,7 @@ Without mirroring, to work locally you'll have to use `git pull` to update your with the upstream project, then push the changes back to your fork to update it. CAUTION: **Caution:** -With mirroring, before approving a merge request, you'll likely be asked to sync; hence automating it is recommend. +With mirroring, before approving a merge request, you'll likely be asked to sync; hence automating it is recommended. Read more about [How to keep your fork up to date with its origin](https://about.gitlab.com/blog/2016/12/01/how-to-keep-your-fork-up-to-date-with-its-origin/). @@ -61,7 +64,7 @@ When you are ready to send your code back to the upstream project, choose your forked project's branch. For **Target branch**, choose the original project's branch. NOTE: **Note:** -When creating a merge request, if the forked project's visibility is more restrictive than the parent project (for example the fork is private, parent is public), the target branch will default to the forked project's default branch. This prevents potentially exposing private code of the forked project. +When creating a merge request, if the forked project's visibility is more restrictive than the parent project (for example the fork is private, the parent is public), the target branch will default to the forked project's default branch. This prevents potentially exposing the private code of the forked project.  diff --git a/doc/user/project/repository/git_blame.md b/doc/user/project/repository/git_blame.md index e63b57747ef..6636463722e 100644 --- a/doc/user/project/repository/git_blame.md +++ b/doc/user/project/repository/git_blame.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Source Code +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference, howto description: "Documentation on Git file blame." --- diff --git a/doc/user/project/repository/git_history.md b/doc/user/project/repository/git_history.md index b7375602a78..6cc05d2192f 100644 --- a/doc/user/project/repository/git_history.md +++ b/doc/user/project/repository/git_history.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Source Code +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference, howto description: "Documentation on Git file history." --- diff --git a/doc/user/project/repository/gpg_signed_commits/index.md b/doc/user/project/repository/gpg_signed_commits/index.md index 68b5c2651e1..5526828c969 100644 --- a/doc/user/project/repository/gpg_signed_commits/index.md +++ b/doc/user/project/repository/gpg_signed_commits/index.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" type: concepts, howto --- diff --git a/doc/user/project/repository/img/file_finder_find_button_v12_10.png b/doc/user/project/repository/img/file_finder_find_button_v12_10.png Binary files differindex e93db946005..51545f63fde 100644 --- a/doc/user/project/repository/img/file_finder_find_button_v12_10.png +++ b/doc/user/project/repository/img/file_finder_find_button_v12_10.png diff --git a/doc/user/project/repository/img/file_finder_find_file_v12_10.png b/doc/user/project/repository/img/file_finder_find_file_v12_10.png Binary files differindex 1404ccc6d0b..e54de6a3065 100644 --- a/doc/user/project/repository/img/file_finder_find_file_v12_10.png +++ b/doc/user/project/repository/img/file_finder_find_file_v12_10.png diff --git a/doc/user/project/repository/img/forking_workflow_choose_namespace.png b/doc/user/project/repository/img/forking_workflow_choose_namespace.png Binary files differdeleted file mode 100644 index eb023ca85f2..00000000000 --- a/doc/user/project/repository/img/forking_workflow_choose_namespace.png +++ /dev/null diff --git a/doc/user/project/repository/img/forking_workflow_choose_namespace_v13_2.png b/doc/user/project/repository/img/forking_workflow_choose_namespace_v13_2.png Binary files differnew file mode 100644 index 00000000000..4843cc671ae --- /dev/null +++ b/doc/user/project/repository/img/forking_workflow_choose_namespace_v13_2.png diff --git a/doc/user/project/repository/index.md b/doc/user/project/repository/index.md index 0cf375009a0..a57806cf3ff 100644 --- a/doc/user/project/repository/index.md +++ b/doc/user/project/repository/index.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Source Code +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: concepts, howto --- diff --git a/doc/user/project/repository/jupyter_notebooks/index.md b/doc/user/project/repository/jupyter_notebooks/index.md index 1948b12aacd..fe432e0d553 100644 --- a/doc/user/project/repository/jupyter_notebooks/index.md +++ b/doc/user/project/repository/jupyter_notebooks/index.md @@ -1,3 +1,9 @@ +--- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference +--- # Jupyter Notebook Files > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/2508/) in GitLab 9.1. diff --git a/doc/user/project/repository/reducing_the_repo_size_using_git.md b/doc/user/project/repository/reducing_the_repo_size_using_git.md index baad5027703..8247f69c61a 100644 --- a/doc/user/project/repository/reducing_the_repo_size_using_git.md +++ b/doc/user/project/repository/reducing_the_repo_size_using_git.md @@ -32,15 +32,24 @@ Git LFS files can only be removed by an Administrator using a ## Purge files from repository history -To make cloning your project faster, rewrite branches and tags to remove unwanted files. +To reduce the size of your repository in GitLab, you must remove references to large files from branches, tags, and +other internal references (refs) that are automatically created by GitLab. These refs include: + +- `refs/merge-requests/*` for merge requests. +- `refs/pipelines/*` for + [pipelines](../../../ci/pipelines/index.md#troubleshooting-fatal-reference-is-not-a-tree). +- `refs/environments/*` for environments. + +Git doesn't usually download these refs to make cloning and fetch faster, but we can use the `--mirror` option to +download all the advertised refs. 1. [Install `git filter-repo`](https://github.com/newren/git-filter-repo/blob/main/INSTALL.md) using a supported package manager or from source. -1. Clone a fresh copy of the repository using `--bare`: +1. Clone a fresh copy of the repository using `--bare` and `--mirror`: ```shell - git clone --bare https://example.gitlab.com/my/project.git + git clone --bare --mirror https://example.gitlab.com/my/project.git ``` 1. Using `git filter-repo`, purge any files from the history of your repository. @@ -74,16 +83,10 @@ To make cloning your project faster, rewrite branches and tags to remove unwante [`git filter-repo` documentation](https://htmlpreview.github.io/?https://github.com/newren/git-filter-repo/blob/docs/html/git-filter-repo.html#EXAMPLES) for more examples and the complete documentation. -1. Running `git filter-repo` removes all remotes. To restore the remote for your project, run: - - ```shell - git remote add origin https://example.gitlab.com/<namespace>/<project_name>.git - ``` - 1. Force push your changes to overwrite all branches on GitLab: ```shell - git push origin --force --all + git push origin --force 'refs/heads/*' ``` [Protected branches](../protected_branches.md) will cause this to fail. To proceed, you must @@ -92,13 +95,21 @@ To make cloning your project faster, rewrite branches and tags to remove unwante 1. To remove large files from tagged releases, force push your changes to all tags on GitLab: ```shell - git push origin --force --tags + git push origin --force 'refs/tags/*' ``` [Protected tags](../protected_tags.md) will cause this to fail. To proceed, you must remove tag protection, push, and then re-enable protected tags. -1. Manually run [project housekeeping](../../../administration/housekeeping.md#manual-housekeeping) +1. To prevent dead links to commits that no longer exist, push the `refs/replace` created by `git filter-repo`. + + ```shell + git push origin --force 'refs/replace/*' + ``` + + Refer to the Git [`replace`](https://git-scm.com/book/en/v2/Git-Tools-Replace) documentation for information on how this works. + +1. Run a [repository cleanup](#repository-cleanup). NOTE: **Note:** Project statistics are cached for performance. You may need to wait 5-10 minutes @@ -106,26 +117,9 @@ to see a reduction in storage utilization. ## Purge files from GitLab storage -To reduce the size of your repository in GitLab, you must remove GitLab internal references to -commits that contain large files. Before completing these steps, -[purge files from your repository history](#purge-files-from-repository-history). +In addition to the refs mentioned above, GitLab also creates hidden `refs/keep-around/*`to prevent commits being deleted. Hidden refs are not advertised, which means we can't download them using Git, but these refs are included in a project export. -As well as [branches](branches/index.md) and tags, which are a type of Git ref, GitLab automatically -creates other refs. These refs prevent dead links to commits, or missing diffs when viewing merge -requests. [Repository cleanup](#repository-cleanup) can be used to remove these from GitLab. - -The following internal refs are not advertised: - -- `refs/merge-requests/*` for merge requests. -- `refs/pipelines/*` for - [pipelines](../../../ci/pipelines/index.md#troubleshooting-fatal-reference-is-not-a-tree). -- `refs/environments/*` for environments. - -This means they are not usually included when fetching, which makes fetching faster. In addition, -`refs/keep-around/*` are hidden refs to prevent commits with discussion from being deleted and -cannot be fetched at all. - -However, these refs can be accessed from the Git bundle inside a project export. +To purge files from GitLab storage: 1. [Install `git filter-repo`](https://github.com/newren/git-filter-repo/blob/main/INSTALL.md) using a supported package manager or from source. @@ -173,6 +167,39 @@ However, these refs can be accessed from the Git bundle inside a project export. [`git filter-repo` documentation](https://htmlpreview.github.io/?https://github.com/newren/git-filter-repo/blob/docs/html/git-filter-repo.html#EXAMPLES) for more examples and the complete documentation. +1. Because cloning from a bundle file sets the `origin` remote to the local bundle file, delete this `origin` remote, and set it to the URL to your repository: + + ```shell + git remote remove origin + git remote add origin https://gitlab.example.com/<namespace>/<project_name>.git + ``` + +1. Force push your changes to overwrite all branches on GitLab: + + ```shell + git push origin --force 'refs/heads/*' + ``` + + [Protected branches](../protected_branches.md) will cause this to fail. To proceed, you must + remove branch protection, push, and then re-enable protected branches. + +1. To remove large files from tagged releases, force push your changes to all tags on GitLab: + + ```shell + git push origin --force 'refs/tags/*' + ``` + + [Protected tags](../protected_tags.md) will cause this to fail. To proceed, you must remove tag + protection, push, and then re-enable protected tags. + +1. To prevent dead links to commits that no longer exist, push the `refs/replace` created by `git filter-repo`. + + ```shell + git push origin --force 'refs/replace/*' + ``` + + Refer to the Git [`replace`](https://git-scm.com/book/en/v2/Git-Tools-Replace) documentation for information on how this works. + 1. Run a [repository cleanup](#repository-cleanup). ## Repository cleanup @@ -187,16 +214,27 @@ references to these objects. You can use To clean up a repository: 1. Go to the project for the repository. -1. Navigate to **{settings}** **Settings > Repository**. -1. Upload a list of objects. For example, a `commit-map` file. +1. Navigate to **Settings > Repository**. +1. Upload a list of objects. For example, a `commit-map` file created by `git filter-repo` which is located in the + `filter-repo` directory. + + If your `commit-map` file is larger than 10MB, the file can be split and uploaded piece by piece: + + ```shell + split -l 100000 filter-repo/commit-map filter-repo/commit-map- + ``` + 1. Click **Start cleanup**. This will: - Remove any internal Git references to old commits. -- Run `git gc` against the repository. +- Run `git gc` against the repository to remove unreferenced objects. Repacking your repository will temporarily + cause the size of your repository to increase significantly, because the old pack files are not removed until the + new pack files have been created. +- Recalculate the size of your repository on disk. -You will receive an email once it has completed. +You will receive an email notification with the recalculated repository size after the cleanup has completed. When using repository cleanup, note: diff --git a/doc/user/project/repository/repository_mirroring.md b/doc/user/project/repository/repository_mirroring.md index bdf13100a6a..bbb673b74b0 100644 --- a/doc/user/project/repository/repository_mirroring.md +++ b/doc/user/project/repository/repository_mirroring.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Source Code +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers disqus_identifier: 'https://docs.gitlab.com/ee/workflow/repository_mirroring.html' --- @@ -114,14 +117,92 @@ After the mirror is created, this option can currently only be modified via the To set up a mirror from GitLab to GitHub, you need to follow these steps: -1. Create a [GitHub personal access token](https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token) with the `public_repo` box checked. +1. Create a [GitHub personal access token](https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token) with the `public_repo` box checked. 1. Fill in the **Git repository URL** field using this format: `https://<your_github_username>@github.com/<your_github_group>/<your_github_project>.git`. 1. Fill in **Password** field with your GitHub personal access token. 1. Click the **Mirror repository** button. The mirrored repository will be listed. For example, `https://*****:*****@github.com/<your_github_group>/<your_github_project>.git`. -The repository will push soon. To force a push, click the appropriate button. +The repository will push soon. To force a push, click the **Update now** (**{retry}**) button. + +## Setting up a push mirror from GitLab to AWS CodeCommit + +AWS CodeCommit push mirroring is currently the best way to connect GitLab repositories to AWS CodePipeline, as GitLab is not yet supported as one of their Source Code Management (SCM) providers. + +Each new AWS Codepipeline needs significant AWS infrastructure setup. It also requires an individual pipeline per branch. + +If AWS CodeDeploy is the final step of a CodePipeline, you can, instead, leverage GitLab CI/CD pipelines and simply use the AWS CLI in the final job in `.gitlab-ci.yml` to deploy to CodeDeploy. + +NOTE: **Note:** +GitLab-to-AWS-CodeCommit push mirroring cannot use SSH authentication until [GitLab issue 34014](https://gitlab.com/gitlab-org/gitlab/-/issues/34014) is resolved. + +To set up a mirror from GitLab to AWS CodeCommit: + +1. In the AWS IAM console, create an IAM user. +1. Add the following least privileges permissions for repository mirroring as an "inline policy". + + The Amazon Resource Names (ARNs) must explicitly include the region and account. The IAM policy + below grants privilege for mirroring access to two sample repositories. These permissions have + been tested to be the minimum (least privileged) required for mirroring: + + ```json + { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "MinimumGitLabPushMirroringPermissions", + "Effect": "Allow", + "Action": [ + "codecommit:GitPull", + "codecommit:GitPush" + ], + "Resource": [ + "arn:aws:codecommit:us-east-1:111111111111:MyDestinationRepo", + "arn:aws:codecommit:us-east-1:111111111111:MyDemo*" + ] + } + ] + } + ``` + +1. After the user was created, click the AWS IAM user name. +1. Click the **Security credentials** tab. +1. Under **HTTPS Git credentials for AWS CodeCommit** click **Generate credentials**. + + NOTE: **Note:** + This Git user ID and password is specific to communicating with CodeCommit. Do + not confuse it with the IAM user ID or AWS keys of this user. + +1. Copy or download special Git HTTPS user ID and password. +1. In the AWS CodeCommit console, create a new repository to mirror from your GitLab repo. +1. Open your new repository and click **Clone URL > Clone HTTPS** (not **Clone HTTPS (GRC)**). +1. In GitLab, open the repository to be push-mirrored. +1. Click **Settings > Repository** and expand **Mirroring repositories**. +1. Fill in the **Git repository URL** field using this format: + + ```plaintext + https://<your_aws_git_userid>@git-codecommit.<aws-region>.amazonaws.com/v1/repos/<your_codecommit_repo> + ``` + + Replace `<your_aws_git_userid>` with the AWS **special HTTPS Git user ID** from the IAM Git + credentials created earlier. Replace `<your_codecommit_repo>` with the name of your repo in CodeCommit. + +1. For **Mirror direction**, select **Push**. +1. For **Authentication method**, select **Password** and fill in the **Password** field with the special IAM Git clone user ID **password** created earlier in AWS. +1. The option **Only mirror protected branches** should be good for CodeCommit as it pushes more + frequently (from every five minutes to every minute). + CodePipeline requires individual pipeline setups for named branches you wish to have a AWS CI setup for. Since feature branches that have dynamic names will not be supported anyway, configuring **Only mirror protected branches** does not cause flexibility problems with CodePipeline integration as long as you are also willing to protect all the named branches you want to build CodePipelines for. + +1. Click **Mirror repository**. You should see the mirrored repository appear: + + ```plaintext + https://*****:*****@git-codecommit.<aws-region>.amazonaws.com/v1/repos/<your_codecommit_repo> + ``` + +To test mirroring by forcing a push, click the half-circle arrows button (hover text is **Update now**). +If **Last successful update** shows a date, you have configured mirroring correctly. +If it is not working correctly a red `error` tag appears and shows the error message as hover text. ## Setting up a push mirror to another GitLab instance with 2FA activated @@ -232,7 +313,7 @@ fingerprints in the open for you to check: - [AWS CodeCommit](https://docs.aws.amazon.com/codecommit/latest/userguide/regions.html#regions-fingerprints) - [Bitbucket](https://support.atlassian.com/bitbucket-cloud/docs/configure-ssh-and-two-step-verification/) -- [GitHub](https://help.github.com/en/github/authenticating-to-github/githubs-ssh-key-fingerprints) +- [GitHub](https://docs.github.com/en/github/authenticating-to-github/githubs-ssh-key-fingerprints) - [GitLab.com](../../gitlab_com/index.md#ssh-host-keys-fingerprints) - [Launchpad](https://help.launchpad.net/SSHFingerprints) - [Savannah](http://savannah.gnu.org/maintenance/SshAccess/) diff --git a/doc/user/project/repository/web_editor.md b/doc/user/project/repository/web_editor.md index 59fedacb2ca..452955b327c 100644 --- a/doc/user/project/repository/web_editor.md +++ b/doc/user/project/repository/web_editor.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Editor +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: howto --- @@ -46,7 +49,7 @@ has already been created, which creates a link to the license itself.  ->**Note:** +NOTE: **Note:** The **Set up CI/CD** button will not appear on an empty repository. You have to at least add a file in order for the button to show up. diff --git a/doc/user/project/repository/x509_signed_commits/index.md b/doc/user/project/repository/x509_signed_commits/index.md index ffbad4e0989..972a46ee7a3 100644 --- a/doc/user/project/repository/x509_signed_commits/index.md +++ b/doc/user/project/repository/x509_signed_commits/index.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" type: concepts, howto --- diff --git a/doc/user/project/requirements/img/requirements_archived_list_view_v13_1.png b/doc/user/project/requirements/img/requirements_archived_list_view_v13_1.png Binary files differindex aafc8543bae..04cb011ff89 100644 --- a/doc/user/project/requirements/img/requirements_archived_list_view_v13_1.png +++ b/doc/user/project/requirements/img/requirements_archived_list_view_v13_1.png diff --git a/doc/user/project/requirements/img/requirements_list_v13_1.png b/doc/user/project/requirements/img/requirements_list_v13_1.png Binary files differindex 3d2adfac074..0ebda571928 100644 --- a/doc/user/project/requirements/img/requirements_list_v13_1.png +++ b/doc/user/project/requirements/img/requirements_list_v13_1.png diff --git a/doc/user/project/service_desk.md b/doc/user/project/service_desk.md index cb6f8e2221d..e9b07f54b91 100644 --- a/doc/user/project/service_desk.md +++ b/doc/user/project/service_desk.md @@ -111,7 +111,7 @@ The **Thank you email** is the email sent to a user after they submit an issue. The file name of the template has to be `thank_you.md`. You can use `%{ISSUE_ID}` placeholder which will be replaced by an issue IID in the email and `%{ISSUE_PATH}` placeholder which will be replaced by project path and the issue IID. -As the service desk issues are created as confidential (only project members can see them) +As the Service Desk issues are created as confidential (only project members can see them) the response email does not provide the issue link. #### New note email diff --git a/doc/user/project/settings/import_export.md b/doc/user/project/settings/import_export.md index cb9f0491b44..478d9b3b948 100644 --- a/doc/user/project/settings/import_export.md +++ b/doc/user/project/settings/import_export.md @@ -1,3 +1,10 @@ +--- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference, howto +--- + # Project import/export > - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/3050) in GitLab 8.9. @@ -17,7 +24,7 @@ See also: To set up a project import/export: - 1. Navigate to **{admin}** **Admin Area >** **{settings}** **Settings > Visibility and access controls**. + 1. Navigate to **Admin Area > Settings > Visibility and access controls**. 1. Scroll to **Import sources** 1. Enable desired **Import sources** @@ -34,7 +41,7 @@ Note the following: - Group members are exported as project members, as long as the user has maintainer or admin access to the group where the exported project lives. - Project members with owner access will be imported as maintainers. -- Using an admin account to import will map users by email address (self-managed only). +- Using an admin account to import will map users by primary email address (self-managed only). Otherwise, a supplementary comment is left to mention that the original author and the MRs, notes, or issues will be owned by the importer. - If an imported project contains merge requests originating from forks, @@ -124,7 +131,7 @@ For more details on the specific data persisted in a project export, see the 1. Go to your project's homepage. -1. Click **{settings}** **Settings** in the sidebar. +1. Click **Settings** in the sidebar. 1. Scroll down to find the **Export project** button: diff --git a/doc/user/project/settings/index.md b/doc/user/project/settings/index.md index 7fe6e702d1c..a65e48d5d56 100644 --- a/doc/user/project/settings/index.md +++ b/doc/user/project/settings/index.md @@ -1,3 +1,10 @@ +--- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference, index, howto +--- + # Project settings NOTE: **Note:** @@ -57,7 +64,7 @@ Use the switches to enable or disable the following features: | **Pipelines** | ✓ | Enables [CI/CD](../../../ci/README.md) functionality | | **Container Registry** | | Activates a [registry](../../packages/container_registry/) for your Docker images | | **Git Large File Storage** | | Enables the use of [large files](../../../topics/git/lfs/index.md#git-large-file-storage-lfs) | -| **Packages** | | Supports configuration of a [package registry](../../../administration/packages/index.md#gitlab-package-registry-administration-premium-only) functionality | +| **Packages** | | Supports configuration of a [package registry](../../../administration/packages/index.md#gitlab-package-registry-administration) functionality | | **Wiki** | ✓ | Enables a separate system for [documentation](../wiki/) | | **Snippets** | ✓ | Enables [sharing of code and text](../../snippets.md) | | **Pages** | ✓ | Allows you to [publish static websites](../pages/) | @@ -128,13 +135,13 @@ no longer actively maintained. Projects that have been archived can also be unarchived. Only project Owners and Admin users have the [permissions](../../permissions.md#project-members-permissions) to archive a project. -When a project is archived, the repository, issues, merge requests, and all +When a project is archived, the repository, packages, issues, merge requests, and all other features are read-only. Archived projects are also hidden in project listings. To archive a project: -1. Navigate to your project's **{settings}** **Settings > General**. +1. Navigate to your project's **Settings > General**. 1. Under **Advanced**, click **Expand**. 1. In the **Archive project** section, click the **Archive project** button. 1. Confirm the action when asked to. @@ -158,7 +165,7 @@ To find an archived project: Next, to unarchive the project: -1. Navigate to your project's **{settings}** **Settings > General**. +1. Navigate to your project's **Settings > General**. 1. Under **Advanced**, click **Expand**. 1. In the **Unarchive project** section, click the **Unarchive project** button. 1. Confirm the action when asked to. @@ -175,7 +182,7 @@ project via a browser) and its place on the file disk where GitLab is installed. To rename a repository: -1. Navigate to your project's **{settings}** **Settings > General**. +1. Navigate to your project's **Settings > General**. 1. Under **Advanced**, click **Expand**. 1. Under "Rename repository", change the "Path" to your liking. 1. Hit **Rename project**. @@ -198,7 +205,7 @@ You can transfer an existing project into a [group](../../group/index.md) if: To transfer a project: -1. Navigate to your project's **{settings}** **Settings > General**. +1. Navigate to your project's **Settings > General**. 1. Under **Advanced**, click **Expand**. 1. Under "Transfer project", choose the namespace you want to transfer the project to. @@ -212,25 +219,25 @@ NOTE: **Note:** GitLab administrators can use the admin interface to move any project to any namespace if needed. -#### Remove a project +#### Delete a project NOTE: **Note:** -Only project owners and admins have [permissions](../../permissions.md#project-members-permissions) to remove a project. +Only project owners and admins have [permissions](../../permissions.md#project-members-permissions) to delete a project. -To remove a project: +To delete a project: -1. Navigate to your project, and select **{settings}** **Settings > General > Advanced**. -1. In the Remove project section, click the **Remove project** button. +1. Navigate to your project, and select **Settings > General > Advanced**. +1. In the "Delete project" section, click the **Delete project** button. 1. Confirm the action when asked to. This action: -- Removes a project including all associated resources (issues, merge requests etc). +- Deletes a project including all associated resources (issues, merge requests etc). - From [GitLab 13.2](https://gitlab.com/gitlab-org/gitlab/-/issues/220382) on [Premium or Silver](https://about.gitlab.com/pricing/) or higher tiers, group admins can [configure](../../group/index.md#enabling-delayed-project-removal-premium) projects within a group to be deleted after a delayed period. When enabled, actual deletion happens after number of days -specified in [instance settings](../../admin_area/settings/visibility_and_access_controls.md#default-deletion-adjourned-period-premium-only). +specified in [instance settings](../../admin_area/settings/visibility_and_access_controls.md#default-deletion-delay-premium-only). CAUTION: **Warning:** The default behavior of [Delayed Project deletion](https://gitlab.com/gitlab-org/gitlab/-/issues/32935) in GitLab 12.6 was changed to @@ -242,7 +249,7 @@ The default behavior of [Delayed Project deletion](https://gitlab.com/gitlab-org To restore a project marked for deletion: -1. Navigate to your project, and select **{settings}** **Settings > General > Advanced**. +1. Navigate to your project, and select **Settings > General > Advanced**. 1. In the Restore project section, click the **Restore project** button. #### Removing a fork relationship @@ -270,7 +277,7 @@ to remove a fork relationship. ### Error Tracking -Configure Error Tracking to discover and view [Sentry errors within GitLab](../operations/error_tracking.md). +Configure Error Tracking to discover and view [Sentry errors within GitLab](../../../operations/error_tracking.md). ### Jaeger tracing **(ULTIMATE)** @@ -278,5 +285,5 @@ Add the URL of a Jaeger server to allow your users to [easily access the Jaeger ### Status Page -[Add Storage credentials](../status_page/#syncing-incidents-to-the-status-page) -to enable the syncing of public Issues to a [deployed status page](../status_page/#status-page-project). +[Add Storage credentials](../../../operations/incident_management/status_page.md#sync-incidents-to-the-status-page) +to enable the syncing of public Issues to a [deployed status page](../../../operations/incident_management/status_page.md#create-a-status-page-project). diff --git a/doc/user/project/settings/project_access_tokens.md b/doc/user/project/settings/project_access_tokens.md index 42ba2654b42..cbc4895f014 100644 --- a/doc/user/project/settings/project_access_tokens.md +++ b/doc/user/project/settings/project_access_tokens.md @@ -1,17 +1,24 @@ -# Project access tokens (Alpha) **(CORE ONLY)** +--- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference, howto +--- -CAUTION: **Warning:** -This is an [Alpha](https://about.gitlab.com/handbook/product/#alpha) feature, and it is subject to change at any time without -prior notice. +# Project access tokens **(CORE ONLY)** > - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/2587) in GitLab 13.0. -> - It's deployed behind a feature flag, disabled by default. +> - It was [deployed](https://gitlab.com/groups/gitlab-org/-/epics/2587) behind a feature flag, disabled by default. +> - [Became enabled by default](https://gitlab.com/gitlab-org/gitlab/-/issues/218722) in GitLab 13.3. > - It's disabled on GitLab.com. -> - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enable-or-disable-project-access-tokens). +> - It can be enabled or disabled by project. +> - It's recommended for production use. +> - For GitLab self-managed instances, GitLab administrators can [disable it](#enable-or-disable-project-access-tokens). Project access tokens are scoped to a project and can be used to authenticate with the [GitLab API](../../../api/README.md#personalproject-access-tokens). -You can also use project access tokens with Git to authenticate over HTTP or SSH. +<!-- Commented out until https://gitlab.com/gitlab-org/gitlab/-/issues/219551 is fixed --> +<!-- You can also use project access tokens with Git to authenticate over HTTP or SSH. --> Project access tokens expire on the date you define, at midnight UTC. @@ -21,7 +28,7 @@ For examples of how you can use a project access token to authenticate with the 1. Log in to GitLab. 1. Navigate to the project you would like to create an access token for. -1. In the **{settings}** **Settings** menu choose **Access Tokens**. +1. In the **Settings** menu choose **Access Tokens**. 1. Choose a name and optional expiry date for the token. 1. Choose the [desired scopes](#limiting-scopes-of-a-project-access-token). 1. Click the **Create project access token** button. @@ -31,8 +38,14 @@ For examples of how you can use a project access token to authenticate with the ## Project bot users For each project access token created, a bot user will also be created and added to the project with -["Maintainer" level permissions](../../permissions.md#project-members-permissions). API calls made with a -project access token will be associated to the corresponding bot user. +["Maintainer" level permissions](../../permissions.md#project-members-permissions). + +For the bot: + +- The name is set to the name of the token. +- The username is set to `project_{project_id}_bot`, such as `project_123_bot`. + +API calls made with a project access token are associated with the corresponding bot user. These users will appear in **Members** but can not be modified. Furthermore, the bot user can not be added to any other project. @@ -41,10 +54,12 @@ When the project access token is [revoked](#revoking-a-project-access-token) the records will be moved to a system-wide user with the username "Ghost User". For more information, see [Associated Records](../../profile/account/delete_account.md#associated-records). +Project bot users are a [GitLab-created service account](../../../subscriptions/index.md#self-managed) and do not count as a licensed seat. + ## Revoking a project access token At any time, you can revoke any project access token by clicking the -respective **Revoke** button in **{settings}** **Settings > Access Tokens**. +respective **Revoke** button in **Settings > Access Tokens**. ## Limiting scopes of a project access token @@ -63,19 +78,30 @@ the following table. ### Enable or disable project access tokens -Project access tokens is an [Alpha](https://about.gitlab.com/handbook/product/#alpha) feature and is not recommended for production use. -It is deployed behind a feature flag that is **disabled by default**. +Project access tokens are deployed behind a feature flag that is **enabled by default**. [GitLab administrators with access to the GitLab Rails console](../../../administration/feature_flags.md) -can enable it for your instance. +can disable it for your instance, globally or by project. + +To disable it globally: + +```ruby +Feature.disable(:resource_access_token) +``` -To enable it: +To disable it for a specific project: + +```ruby +Feature.disable(:resource_access_token, project) +``` + +To enable it globally: ```ruby Feature.enable(:resource_access_token) ``` -To disable it: +To enable it for a specific project: ```ruby -Feature.disable(:resource_access_token) +Feature.enable(:resource_access_token, project) ``` diff --git a/doc/user/project/static_site_editor/img/wysiwyg_editor_v13_0.png b/doc/user/project/static_site_editor/img/wysiwyg_editor_v13_0.png Binary files differdeleted file mode 100644 index a2f5248bf39..00000000000 --- a/doc/user/project/static_site_editor/img/wysiwyg_editor_v13_0.png +++ /dev/null diff --git a/doc/user/project/static_site_editor/img/wysiwyg_editor_v13_3.png b/doc/user/project/static_site_editor/img/wysiwyg_editor_v13_3.png Binary files differnew file mode 100644 index 00000000000..52776c6a290 --- /dev/null +++ b/doc/user/project/static_site_editor/img/wysiwyg_editor_v13_3.png diff --git a/doc/user/project/static_site_editor/index.md b/doc/user/project/static_site_editor/index.md index 8653bb5001a..4e401014122 100644 --- a/doc/user/project/static_site_editor/index.md +++ b/doc/user/project/static_site_editor/index.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Static Site Editor +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" type: reference, how-to description: "The static site editor enables users to edit content on static websites without prior knowledge of the underlying templating language, site architecture or Git commands." --- @@ -10,6 +13,7 @@ description: "The static site editor enables users to edit content on static web > - Support for adding images through the WYSIWYG editor [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/216640) in GitLab 13.1. > - Markdown front matter hidden on the WYSIWYG editor [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/216834) in GitLab 13.1. > - Support for `*.md.erb` files [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/223171) in GitLab 13.2. +> - Non-Markdown content blocks uneditable on the WYSIWYG mode [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/216836) in GitLab 13.3. DANGER: **Danger:** In GitLab 13.0, we [introduced breaking changes](https://gitlab.com/gitlab-org/gitlab/-/issues/213282) @@ -50,15 +54,15 @@ the bottom-left corner of its pages:  -When clicking it, GitLab will open up an editor window from which the content +When you click it, GitLab opens up an editor window from which the content can be directly edited. When you're ready, you can submit your changes in a click of a button: - + When an editor submits their changes, in the background, GitLab automatically creates a new branch, commits their changes, and opens a merge request. The -editor will land directly on the merge request, and then they can assign it to +editor lands directly on the merge request, and then they can assign it to a colleague for review. ## Getting started @@ -73,7 +77,7 @@ easily edit your content. template. You can either [fork it](../repository/forking_workflow.md#creating-a-fork) or [create a new project from a template](../../../gitlab-basics/create-project.md#built-in-templates). 1. Edit the `data/config.yml` file adding your project's path. -1. Editing the file will trigger a CI/CD pipeline to deploy your project with GitLab Pages. +1. Editing the file triggers a CI/CD pipeline to deploy your project with GitLab Pages. 1. When the pipeline finishes, from your project's left-side menu, go to **Settings > Pages** to find the URL of your new website. 1. Visit your website and look at the bottom-left corner of the screen to see the new **Edit this page** button. diff --git a/doc/user/project/status_page/index.md b/doc/user/project/status_page/index.md index 1c885ae043f..20bb23f1d6b 100644 --- a/doc/user/project/status_page/index.md +++ b/doc/user/project/status_page/index.md @@ -1,134 +1,5 @@ --- -stage: Monitor -group: Health -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +redirect_to: '../../../operations/incident_management/status_page.md' --- -# GitLab Status Page **(ULTIMATE)** - -> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/2479) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.10. - -GitLab Status Page allows you to create and deploy a static website to communicate efficiently to users during an incident. - -## How to set up - -NOTE: **Note:** -Only AWS S3 is supported as a deploy target. - -```mermaid -graph TB - subgraph GitLab Instance - issues(issue updates) -- trigger --> middleware(Background job: JSON generation) - end - subgraph Cloud Provider - middleware --saves data --> c1(Cloud Bucket stores JSON file) - end - subgraph Status Page - d(Static Site on CDN) -- fetches data --> c1 - end -``` - -Setting up a Status Page is pretty painless but there are a few things you need to do. - -### Cloud account set up - -To use GitLab Status Page you first need to set up your account details for your cloud provider in the operations settings page. Today, only AWS is supported. - -#### AWS Setup - -1. Within your AWS acccout, create two new IAM policies. - - [Create bucket](https://gitlab.com/gitlab-org/status-page/-/blob/master/deploy/etc/s3_create_policy.json). - - [Update bucket contents](https://gitlab.com/gitlab-org/status-page/-/blob/master/deploy/etc/s3_update_bucket_policy.json) (Remember replace `S3_BUCKET_NAME` with your bucket name). -1. Create a new AWS access key with the permissions policies created in the first step. - -### Status Page project - -To deploy the Status Page to AWS S3 you need to add the Status Page project & configure the necessary CI variables. - -1. Fork the [Status Page](https://gitlab.com/gitlab-org/status-page) project. This can also be done via [Repository Mirroring](https://gitlab.com/gitlab-org/status-page#repository-mirroring) which will ensure you get the up-to-date Status Page features. -1. Add the following variables in **Settings > CI/CD > Variables**. (To get these variables from Amazon, use your Amazon Console): - - `S3_BUCKET_NAME` - name of the Amazon S3 bucket (If a bucket with the provided name doesn't exist, the first pipeline run will create one and configure it for [static website hosting](https://docs.aws.amazon.com/AmazonS3/latest/dev/HostingWebsiteOnS3Setup.html)) - - `AWS_DEFAULT_REGION` - the AWS region - - `AWS_ACCESS_KEY_ID` - the AWS access key ID - - `AWS_SECRET_ACCESS_KEY` - the AWS secret -1. Run the pipeline to deploy the Status Page to S3. - -### Syncing incidents to the Status Page - -Once the CI/CD variables are set, you'll need to set up the Project you want to use for Incident issues: - -1. To view the [Operations Settings](../settings/#operations-settings) page, navigate to **{settings}** **Settings > Operations > Status Page**. -1. Fill in your cloud provider's credentials and make sure the **Active** checkbox is checked. -1. Click **Save changes**. - -## Status Page UI - -The Status Page landing page shows you an overview of the recent incidents. Clicking on an incident will take you to the incident's detail page. - - - -### Incident detail page - -The incident detail page shows detailed information about a particular incident. For example: - -- Status on the incident, including when the incident was last updated. -- The incident title, including any emojis. -- The description of the incident, including emojis. -- Any file attachments provided in the incident description or comments with a valid image extension. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/205166) in GitLab 13.1. -- A chronological ordered list of updates to the incident. - - - -## How it works - -### Publishing Incidents - -To publish an Incident, you first need to create an issue in the Project you enabled the Status Page settings in. - -Issues are not published to the Status Page by default. Use the `/publish` [quick action](../quick_actions.md) in an issue to publish the issue. Only [project or group owners](../../permissions.md) are permitted to publish issues. - -After the quick action is used, a background worker publishes the issue onto the Status Page using the credentials you provided during setup. - -Since all incidents are published publicly, user and group mentions are anonymized with `Incident Responder`, -and titles of non-public [GitLab references](../../markdown.md#special-gitlab-references) are removed. - -When an Incident is published in the GitLab project, you can access the -details page of the Incident by clicking the **Published on status page** button -displayed under the Incident's title. - - - -NOTE: **Note:** -Confidential issues can't be published. If you make a published issue confidential, it will be unpublished. - -### Publishing updates - -To publish an update to the Incident, update the incident issue's description. - -CAUTION: **Caution:** -When referenced issues are changed (e.g. title, confidentiality) the incident they were referenced in are not updated automatically. - -### Adding comments - -To add comments to the Status Page Incident, create a comment on the incident issue. - -When you're ready to publish the comment, add a microphone [award emoji](../../../user/award_emojis.md) reaction (`:microphone` 🎤) to the comment. This marks the comment as one which should be deployed to the Status Page. - -CAUTION: **Caution:** -Anyone with access to view the Issue can add an Emoji Award to a comment, so you may want to keep your Issues limited to team members only. - -### Changing the Incident status - -To change the incident status from `open` to `closed`, close the incident issue within GitLab. This will then be updated shortly on the Status Page website. - -## Attachment storage - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/205166) in GitLab 13.1. - -Beginning with GitLab 13.1, files attached to incident issue descriptions or -comments are published and unpublished to the status page storage as part of -the [publication flow](#how-it-works). - -### Limit - -Only 5000 attachments per issue will be transferred to the status page. +This document was moved to [status_page.md](../../../operations/incident_management/status_page.md). diff --git a/doc/user/project/web_ide/index.md b/doc/user/project/web_ide/index.md index abead8afdc8..12ba55cafdc 100644 --- a/doc/user/project/web_ide/index.md +++ b/doc/user/project/web_ide/index.md @@ -1,3 +1,10 @@ +--- +stage: Create +group: Editor +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference, how-to +--- + # Web IDE > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/4539) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 10.4. @@ -24,8 +31,6 @@ file path fragments to start seeing results. ## Syntax highlighting -> Support for `.gitlab-ci.yml` validation [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/218472) in GitLab 13.2. - As expected from an IDE, syntax highlighting for many languages within the Web IDE will make your direct editing even easier. @@ -37,14 +42,6 @@ The Web IDE currently provides: - IntelliSense and validation support (displaying errors and warnings, providing smart completions, formatting, and outlining) for some languages. For example: TypeScript, JavaScript, CSS, LESS, SCSS, JSON, and HTML. -- Validation support for certain JSON and YAML files using schemas based on the - [JSON Schema Store](https://www.schemastore.org/json/). This feature - is only supported for the `.gitlab-ci.yml` file. - - NOTE: **Note:** - Validation support based on schemas is hidden behind - the feature flag `:schema_linting` on self-managed installations. To enable the - feature, you can [turn on the feature flag in Rails console](../../../administration/feature_flags.md#how-to-enable-and-disable-features-behind-flags). Because the Web IDE is based on the [Monaco Editor](https://microsoft.github.io/monaco-editor/), you can find a more complete list of supported languages in the @@ -56,6 +53,37 @@ If you are missing Syntax Highlighting support for any language, we prepared a s NOTE: **Note:** Single file editing is based on the [Ace Editor](https://ace.c9.io). +### Schema based validation + +> - Support for `.gitlab-ci.yml` validation [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/218472) in GitLab 13.2. +> - It was deployed behind a feature flag, disabled by default. +> - It's enabled on GitLab.com. +> - It cannot be enabled or disabled per-project. +> - For GitLab self-managed instances, GitLab administrators can opt to [enable it](#enable-or-disable-schema-based-validation-core-only). + +The Web IDE provides validation support for certain JSON and YAML files using schemas +based on the [JSON Schema Store](https://www.schemastore.org/json/). This feature is +only supported for the `.gitlab-ci.yml` file. + +#### Enable or disable Schema based validation **(CORE ONLY)** + +Schema based validation is under development and not ready for production use. It is +deployed behind a feature flag that is **disabled by default** for self-managed instances, +[GitLab administrators with access to the GitLab Rails console](../../../administration/feature_flags.md) +can enable it for your instance. + +To enable it: + +```ruby +Feature.enable(:schema_linting) +``` + +To disable it: + +```ruby +Feature.disable(:schema_linting) +``` + ### Themes > - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/2389) in GitLab in 13.0. diff --git a/doc/user/project/wiki/index.md b/doc/user/project/wiki/index.md index 9044ee0765f..5503cd97628 100644 --- a/doc/user/project/wiki/index.md +++ b/doc/user/project/wiki/index.md @@ -1,3 +1,10 @@ +--- +stage: Create +group: Knowledge +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference, how-to +--- + # Wiki A separate system for documentation called Wiki, is built right into each diff --git a/doc/user/search/advanced_global_search.md b/doc/user/search/advanced_global_search.md index 0613e9b8e34..820d66e4cd6 100644 --- a/doc/user/search/advanced_global_search.md +++ b/doc/user/search/advanced_global_search.md @@ -1,3 +1,10 @@ +--- +stage: Create +group: Editor +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference +--- + # Advanced Global Search **(STARTER)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/109) in GitLab [Starter](https://about.gitlab.com/pricing/) 8.4. @@ -15,20 +22,20 @@ visit the [administrator documentation](../../integration/elasticsearch.md). The Advanced Global Search in GitLab is a powerful search service that saves you time. Instead of creating duplicate code and wasting time, you can -now search for code within other teams that can help your own project. +now search for code within other projects that can help your own project. GitLab leverages the search capabilities of [Elasticsearch](https://www.elastic.co/elasticsearch/) and enables it when searching in: - Projects -- Repositories -- Commits - Issues - Merge requests - Milestones -- Notes (comments) -- Snippets +- Comments +- Code +- Commits - Wiki +- Users ## Use cases diff --git a/doc/user/search/advanced_search_syntax.md b/doc/user/search/advanced_search_syntax.md index 5dc1f8fe779..d8fce3223ed 100644 --- a/doc/user/search/advanced_search_syntax.md +++ b/doc/user/search/advanced_search_syntax.md @@ -1,3 +1,10 @@ +--- +stage: Create +group: Editor +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference +--- + # Advanced Syntax Search **(STARTER)** > - Introduced in [GitLab Enterprise Starter](https://about.gitlab.com/pricing/) 9.2 @@ -66,3 +73,20 @@ Examples: - Finding the text 'def create' inside files with the `.rb` extension: `def create extension:rb` - Finding the text `sha` inside files in a folder called `encryption`: `sha path:encryption` - Finding any file starting with `hello` containing `world` and with the `.js` extension: `world filename:hello* extension:js` + +#### Excluding filters + +[Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/31684) in GitLab Starter 13.3. + +Filters can be inversed to **filter out** results from the result set, by prefixing the filter name with a `-` (hyphen) character, such as: + +- `-filename` +- `-path` +- `-extension` + +Examples: + +- Finding `rails` in all files but `Gemfile.lock`: `rails -filename:Gemfile.lock` +- Finding `success` in all files excluding `.po|pot` files: `success -filename:*.po*` +- Finding `import` excluding minified JavaScript (`.min.js`) files: `import -extension:min.js` +- Finding `docs` for all files outside the `docs/` folder: `docs -path:docs/` diff --git a/doc/user/search/img/filter_approved_by_merge_requests_v13_0.png b/doc/user/search/img/filter_approved_by_merge_requests_v13_0.png Binary files differindex ef4c65aea4b..5020da90297 100644 --- a/doc/user/search/img/filter_approved_by_merge_requests_v13_0.png +++ b/doc/user/search/img/filter_approved_by_merge_requests_v13_0.png diff --git a/doc/user/search/index.md b/doc/user/search/index.md index b616b606b64..98b34fe8383 100644 --- a/doc/user/search/index.md +++ b/doc/user/search/index.md @@ -1,3 +1,10 @@ +--- +stage: Create +group: Editor +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: index, reference, howto +--- + # Search through GitLab ## Issues and merge requests diff --git a/doc/user/snippets.md b/doc/user/snippets.md index 4d8f47ee3be..15391f034a8 100644 --- a/doc/user/snippets.md +++ b/doc/user/snippets.md @@ -1,3 +1,10 @@ +--- +stage: Create +group: Editor +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +type: reference +--- + # Snippets With GitLab Snippets you can store and share bits of code and text with other users. diff --git a/doc/user/todos.md b/doc/user/todos.md index ef0e75bc197..02fef07a89a 100644 --- a/doc/user/todos.md +++ b/doc/user/todos.md @@ -43,9 +43,15 @@ A To Do appears on your To-Do List when: - Commit - Design - The CI/CD pipeline for your merge request failed -- An open merge request becomes unmergeable due to conflict, and you are either: - - The author - - Have set it to automatically merge once the pipeline succeeds +- An open merge request becomes unmergeable due to conflict, and one of the following is true: + - You are the author + - You are the user that set it to automatically merge once the pipeline succeeds +- [Since GitLab 13.2](https://gitlab.com/gitlab-org/gitlab/-/issues/12136), a merge request + is removed from a [merge train](../ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md) + and you are the user that added it. **(PREMIUM)** + +When multiple trigger actions occur for the same user on the same object (for example, an issue) +only the first is displayed as a single to-do on their To-Do List. To-do triggers are not affected by [GitLab Notification Email settings](profile/notifications.md). diff --git a/doc/user/upgrade_email_bypass.md b/doc/user/upgrade_email_bypass.md new file mode 100644 index 00000000000..027f7337228 --- /dev/null +++ b/doc/user/upgrade_email_bypass.md @@ -0,0 +1,123 @@ +# Updating to GitLab 13.2: Email confirmation issues + +In the [GitLab 13.0.1 security release](https://about.gitlab.com/releases/2020/05/27/security-release-13-0-1-released/), +we described a security issue that allowed users to bypass the email verification process. +In that notice, we strongly recommended that you upgrade all affected installations to the +latest version as soon as possible. + +There is a chance that users with multiple email addresses on a self-managed instance may +be unable to commit code and sign in. For more information, see the following resolved and closed +[security issue](https://gitlab.com/gitlab-org/gitlab/-/issues/121664). + +This page can help you identify the users at risk, as well as potential issues of the update. + +## The risk: users get emails that require confirmation + +During the update process to GitLab 13.2, a background migration is run for accounts that meet the +conditions for the security issue. Such users are marked as _unconfirmed_. + +An initial email is sent to _unconfirmed_ users to describe the issue. A second email is then +sent within five minutes, with a link for users to re-confirm the subject email address. + +## Do the confirmation emails expire? + +The links in these re-confirmation emails expire after one day by default. Users who click an expired link will be asked to request a new re-confirmation email. Any user can request a new re-confirmation email from `http://gitlab.example.com/users/confirmation/new`. + +## Generate a list of affected users + +You can generate this list before and after the upgrade using different methods. + +### Before an upgrade to GitLab 13.2 + +Use the following code to search for users who: + +- Are currently confirmed. +- Include identical `confirmed_at` times. +- Also have a secondary email address. + +```ruby +emails_and_users_that_will_be_unconfirmed = Email.joins(:user).merge(User.active).where('emails.confirmed_at IS NOT NULL').where('emails.confirmed_at = users.confirmed_at').where('emails.email <> users.email') +``` + +### After an upgrade to GitLab 13.2 + +Use the following code to search for users who: + +- Are currently **not** confirmed. +- Are also pending confirmation on or after the date of upgrade. + +```ruby +User.where(confirmed_at: nil).where('LENGTH(confirmation_token) = 32') +``` + +## What does it look like when a user is blocked? + +A regular user might receive a message that says "You have to confirm your email address before continuing". This message could includes a 404 or 422 error code, when the user tries to sign in. + +NOTE: **Note:** +We hope to improve the [sign-in experience for an unverified user](https://gitlab.com/gitlab-org/gitlab/-/issues/29279) in a future release. + +When an affected user commits code to a Git repository, that user may see the following message: + +```shell +Your account has been blocked. Fatal: Could not read from remote repository + +# or + +Your primary email address is not confirmed. +``` + +You can assure your users that they have not been [Blocked](admin_area/blocking_unblocking_users.md) by an administrator. +When affected users see this message, they must confirm their email address before they can commit code. + +## What do I need to know as an administrator of a GitLab self-managed Instance? + +You have the following options to help your users: + +- They can confirm their address through the email that they received. +- They can confirm the subject email address themselves by navigating to `https://gitlab.example.com/users/confirmation/new`. + +As an administrator, you may also confirm a user in the [Admin Area](admin_area/#administering-users). + +## What do I do if I am an administrator and I am locked out? + +If you are an administrator and cannot otherwise verify your email address, sign in to your GitLab +instance with a [Rails console session](../administration/troubleshooting/navigating_gitlab_via_rails_console.md#starting-a-rails-console-session). +Once connected, run the following commands to confirm your administrator account: + +```ruby +admin = User.find_by_username "root" # replace with your admin username +admin.confirmed_at = Time.zone.now +admin.save! +``` + +## How do I force-confirm all users on my self-managed instance? + +If you are an administrator and would like to force-confirm all users on your system, sign in to your GitLab +instance with a [Rails console session](../administration/troubleshooting/navigating_gitlab_via_rails_console.md#starting-a-rails-console-session). +Once connected, run the following commands to confirm all user accounts: + +```ruby +User.where('LENGTH(confirmation_token) = 32').where(confirmed_at: nil).find_each { |u| u.confirmed_at = Time.now; u.save } +``` + +CAUTION: **Caution:** +The command described in this section may activate users who have not properly confirmed their email addresses. + +## What about LDAP users? + +LDAP Users will remain confirmed if all of the following conditions are met: + +- The ["User email confirmation at sign-up" option](../security/user_email_confirmation.md) is set to false. +- The first sign-in is based on user LDAP credentials. +- The user has added and verified [a secondary email address](profile/index.md#profile-settings) some time later. + +NOTE: **Note:** +Confirmation timestamps (primary vs. secondary) will be different. + +Users will be unconfirmed by the background migration if any of the following conditions are met: + +- They [create an account through GitLab](profile/account/create_accounts.md). +- They [swap their primary email address](profile/index.md#profile-settings) and verify it. +- If they have two email addresses with the same `confirmed_at` timestamp due to the linked [security issue](https://gitlab.com/gitlab-org/gitlab/-/issues/121664). +- [LDAP is introduced](../administration/auth/ldap/index.md), and users' primary email address matches that in LDAP. |
